ch03-决策树/03-使用信息熵寻找最优划分.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 模拟使用信息熵寻找最优划分"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### sklearn中决策树模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import datasets\n",
    "\n",
    "iris = datasets.load_iris()\n",
    "X = iris.data[:, 2:]\n",
    "y = iris.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(class_weight=None, criterion='entropy', max_depth=2,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
       "            splitter='best')"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "dt_clf = DecisionTreeClassifier(max_depth=2, criterion=\"entropy\")\n",
    "dt_clf.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_decision_boundary(model, axis):\n",
    "    \n",
    "    x0, x1 = np.meshgrid(\n",
    "        np.linspace(axis[0], axis[1], int((axis[1]-axis[0])*100)).reshape(-1, 1),\n",
    "        np.linspace(axis[2], axis[3], int((axis[3]-axis[2])*100)).reshape(-1, 1),\n",
    "    )\n",
    "    X_new = np.c_[x0.ravel(), x1.ravel()]\n",
    "\n",
    "    y_predict = model.predict(X_new)\n",
    "    zz = y_predict.reshape(x0.shape)\n",
    "\n",
    "    from matplotlib.colors import ListedColormap\n",
    "    custom_cmap = ListedColormap(['#EF9A9A','#FFF59D','#90CAF9'])\n",
    "    \n",
    "    plt.contourf(x0, x1, zz, linewidth=5, cmap=custom_cmap)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mgbook/.pyenv/versions/3.6.4rc1/lib/python3.6/site-packages/matplotlib/contour.py:967: UserWarning: The following kwargs were not used by contour: 'linewidth'\n",
      "  s)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x11da78160>"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD6CAYAAAC1W2xyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAGp1JREFUeJzt3XmMnPV9x/H3d3d9sVkfGBwbQuMgckAoh9hYKM6FW1zAkCZQKjdBoQmK/2gTkYNUom4SkoZGkVAqK6SR3Cht1KJE5aqSAOF0DpyAYyfYcjlEuIqLsXGM7fUJu/vtHzOzO/PsMzvPPPM88xzzeUmrzDz+zbNfHPnj8fN89jfm7oiISHn0ZT2AiIgkS8EuIlIyCnYRkZJRsIuIlIyCXUSkZBTsIiIl0zLYzWzAzG41s41m9r0ma2ab2U/MbKuZ/YeZWfKjiohIFFHesX8I2Oruy4ElZnZOyJqrgB3ufjawALgwwRlFRKQNUYL9p8A3zWwAmA8cCFmzAri/+vgh4IJkxhMRkXYNtFrg7gcBzOxRYKe7PxuybCGwv/r4APD24AIzWwOsARicNeu8ty5eHHdmieEPcxZmPYKIdOjFJ3+7x91PbLWuZbCb2ULgIPBu4CEzu8DdNwSW7QHmVR/Pqz5v4O7rgfUA5y5d6hvWrm31rSVB3z/n6qxHEJEOfWbZrBeirItyKebzwJXuPgYcBuaErHkQWFl9vAIIBr+IiHRJlGD/NvAJM/s18AfgKTO7KbDmFuBkM9sG7KUS9CIikoEo19j/j8q78HrXBdYcAy5NcC4REYlJP6AkIlIyCnYRkZJRsIuIlIyCXUSkZBTsIiIlo2AXESkZBbuISMko2EVESkbBLiJSMgp2EZGSUbCLiJSMgl1EpGQU7CIiJaNgFxEpGQW7iEjJKNhFREpGwS4iUjIKdhGRklGwi4iUjIJdRKRkFOwiIiWjYBcRKRkFu4hIySjYRURKRsEuIlIyCnYRkZJRsIuIlIyCXUSkZFoGu1V838weMbMfmdlAyJqLzGyHmT1c/Xp7OuOKiEgrU0I6xHJgwN3PN7OfASuBu0PWfcfdb0xyOBEplt/vu4fNu2/m4Ogu3jDwRoYXfYrT5l8c63VArHNJtGDfBayrPn5tmnVXmNmfAy8Cf+Hu3ulwIlIcv993D7/c+TXG/CgAB0df5pc7vwYwbSCHve7nL92AmTHur7d1LqloeSnG3Z92901m9mFgJnBvyLJngC+6+zJgCfD+ZMcUkbzbvPvmiXCuGfOjbN59c9uvc0YnQr2dc0lFpJunZvZB4FrgMncfC1myF3ig+vh5YFHIOdaY2WYz27xnZCTmuCKSVwdHd7V1POqvx13by6LcPF0MfAFY5e7NEvlzwGoz6wPOBLYHF7j7encfdvfhE4aGOplZRHLoDQNvbOt41F+Pu7aXRXnHfjWVyyv3Vhsv15jZTYE1NwMfBx4F7nT3xxOeU0RybnjRp+i32Q3H+m32xI3Qdl5nDNBnM9o+l1S0vHnq7t8AvtFizU7gAwnNJCIFVLup2W6Tpdnr4pxLKqK0YkSkgOJWD6Pa+NLXeXLfHTjjGH28Y/7lLD/p+ljf47T5F4e+TkEej4JdpITiVg+j2vjS13li320Tz53xiefLT7q+4/NLZ7SlgEgJxa0eRvXkvjvaOi7dpWAXKaG41cOonPG2jkt3KdhFSihu9TAqaxIdzY5Ld+n/BZESils9jOod8y9v67h0l26eipRQ3OphVLUbpGGtGMmegl2kh0TdRTHsWPAvheUnXT8lyKNULKPWMNOua8aV17nqKdhFSijqjom/2PkV3B1ndNp1cXdpDL4uag0z7bpmXHmdK0jX2EVKKOqOieP++kSoT7cu7i6NwddFrWGmXdeMK69zBSnYRUoojV0Q4+7SWH88ag0z7bpmXHmdK0jBLlJCaeyCGHeXxvrjUWuYadc148rrXEEKdpESirpjYp/NwAK32uLurBilYhm1hpl2XTOuvM4VpJunIiXUzo6JUY/F3aWx/nVRa5hp1zXjyutcQZbFR5Oeu3Spb1i7tuvft5d9/5yrsx5BSqbotcUweZ/1M8tmbXH34Vbr9I5dRNpW9NpimCLN2oqusYtI24peWwxTpFlbUbCLSNuKXlsMU6RZW1Gwi0jbil5bDFOkWVtRsItI24peWwxTpFlb0c1TEWlb0WuLYYo0aysKdhGJpdkHUMddlwdFmnU6CnYRmSLY5z7lDe/hxYMPJ/KDTXnpiudljjQo2EWkQVif+4l9t038ejtbAMfdtjdteZkjLbp5KiINwvrcQVG3AI67bW/a8jJHWhTsItIg6d52nG1705aXOdKiYBeRBkn3tuNs25u2vMyRFgW7iDQI63MHRd0COO62vWnLyxxp0c1TEWkQ1udOqhWTl654XuZIS8tte83MgH8H3g7sBi5399HAmtnAbcApwDbgYz7NibVtb/dp295ySbOOWHSd1BjzXoFMctve5cCAu59vZj8DVgJ3B9ZcBexw90vN7CfAhcB9bc4sIhFEqSOGVQ/DKoplqvhBZzXGMlUgo1xj3wWsqz5+rcmaFcD91ccPARd0OJeINBGljhhWPQyrKJap4ged1RjLVIFs+Y7d3Z8GMLMPAzOBe0OWLQT2Vx8foHLZpoGZrQHWALzp+ONjjisiadYRi66TGmOZKpCRWjFm9kHgWuAydx8LWbIHmFd9PK/6vIG7r3f3YXcfPmFoKO68Ij0vzTpi0XVSYyxTBbJlsJvZYuALwCp3H2my7EEq196hcllmQzLjiUhQlDpiWPUwrKJYpoofdFZjLFMFMsrN06uBJcC9lYIM/wac7u7X1a25BbjczLYBW6kEvYikIM06YtF1UmMsUwUyyjX2bwDfaLHmGHBpUkOJyPSC28v+ft89vHjw4YY1uw4/xqHR3YBzaHQ3uw4/xvKTrs/tbotBcefqZOtdbdsrIrkQVtP7+Utfxpm8HeaMT1Qil590/bSvzUPFL69zFYW2FBApuLCaXn2o13ty3x0tX5uHil9e5yoKBbtIwbVTx3PGI70264pfXucqCgW7SMG1U8ezwB/5vFb88jpXUSjYRQourKZn9Ieufcf8y1u+Ng8Vv7zOVRS6eSpScM1qersOP8aT++7AGcfo4x3zL2+4cTrda7O+QZnXuYpCwS6Sc9uf/iSbXtvCOJV/Yi+beR77B0+dEtqr33bXlNfWuu2DA4t443HnhJ4/SsVv40tfn/L93njcOZGCN0ptsdmaKEGe17pmlhTsIjm2/elP8shrW6Dyw4GMQ+X567+dWBNWZUyyLrjxpa837B5Z+35P7Lsd8GnPH2UO7ciYPF1jF8mxTXWhPiH4vKq+yphkXTBYkZzU+JELYeePMod2ZEyegl0kx8ZbL5lQX2VMsi4YrEhOJ3j+KHNoR8bkKdhFcqydP6D1VcYk64LBiuR0guePMod2ZEyegl0kx5bNPA+CnzLZ5FMn66uMSdYFgxXJSY2XhMLOH2UO7ciYPN08FcmxM9/6rxCxFVNfZUyyLlg7b5xWTJQ5tCNj8lp+mHUa9GHW3acPsxYpviQ/zFpK4Np3/mfWI/S8u154gnXbN/Ly4REWHzfEtWcuZ9WbT0/m5I9vhl/cDQdehbkL4H2XwBkt//xLwXwm4joFu0gX3PXCE9yw5QGOjlU+YHrn4RFu2PIAQOfh/vhm+Ol/wWj1g6oPvFp5Dgr3HqWbpyJdsG77xolQrzk6Nsq67Rs7P/kv7p4M9ZrR1yvHpScp2EW64OXD4R8X3Ox4Ww682t5xKT0Fu0gXLD5uqK3jbZm7oL3jUnoKdpEuuPbM5czub7ylNbt/gGvPXN75yd93CQzMaDw2MKNyXHqSbp6KdEHtBmkqrZjaDVK1YqRKwS7SJasOHWLViy9Nhu9bDk1dFLe2eMbw1HU5qUCmWvOUUAp2kW6IUklMsraYkwpkqjVPaUrX2EW6IUolMcnaYk4qkKnWPKUpBbtIN0SpJCZZW8xJBTLVmqc0pWAX6YYolcQka4s5qUCmWvOUphTsIt0QpZKYZG0xJxXIVGue0pRunop0Q5RKYpK1xZxUIFOteUpTkYLdzGYAd7j7ZU1+/SLgu8Dz1UPXuPtTiUwokiedVAg3/BgOHag8PvAq3Hf71HNFdd+tsPUR8HGwPu46/Y9ZZ6MN4cngIOtOOYmXDw9Vjg0Osqr9/+KOrXrz6QryLmsZ7GY2B3gUeFuLpd9x9xsTmUokjzqpEH77y5OhXvPa0cpX7Vx3/QD6DMbGpj//fbfCY7+aeHrXcbO54dAfONpXubK68/AI//CbezEzXh8fnzimmmHvaHmN3d2PuPtZwI4WS68ws01mdrtZk49RFymyTiqEwVAP4+OToT7d+bc+0vB03YL5E6E+8TL3iVCvUc2wdyR18/QZ4IvuvgxYArw/uMDM1pjZZjPbvGdEVScpoKwqhMHze2NgvzzQH/lUqhn2hqSCfS/wQPXx88Ci4AJ3X+/uw+4+fMKQqk5SQFlVCIPnt8Y/totHA+/yp6GaYW9IKtg/B6w2sz7gTGB7QucVyY9OKoSDc1uvsT7oD7z7Djv/2ec3PL321X3MDlx2GTBjRuDyjGqGvaPtYDezt5jZTYHDNwMfp3KT9U53fzyJ4URy5YxhuOgvJ99Bz11QeR6lFfO3X5ka7jNnN55r1V/Bxatbn3/llXDOuyfeua86fJQbBhey5LghDFhy3BBfe9ef8Y/DKxuO3XDen+rGaY+I3GN399Oq//sccF3g13YCH0h0MpE0JbmLYphAHZGzz6+Ee9gM9XY8ByP7K49H9leew9RZV15Z+apaVf0KSjPItWtjfukHlKT3pL3zYaCOiI9PPq+FcdgMd/2g8cZo7XW1vyDSmDUm7dqYb9pSQHpP2jsfBuqIocfDZgi0XZoez8EHVWvXxnxTsEvvSbu2GCWgO/1eGX9QtXZtzDcFu/SetGuL1uSPVf3xTr9Xxh9UrV0b803BLr0n7Z0PA3XE0ONhM0T5CwFy8UHV2rUx33TzVHpP2jsf1m6QBlsxdS2WpjPseG7q6970lsx3aQzSro35Zu7e9W967tKlvmHt2q5/3142/6PRf+xcRPLJBq/Z4u4t/1bXO3Yprk620A3zw3+B/3168vngXDh8sPHdM0x9Rx12LOxdNjQeO/V0ePaJXL0Tl3LQO/YeUbp37MEeOFSuPUf9SdCgYKh3ygzq/2z198O4N2/MQGfzS0+I+o5dN0+lmJLuoicZ6tAY6lDZjne6UIdc9NOlHBTsUkxZbaGbtqLPL7mgYJdiymoL3bQVfX7JBQW7FFPSXfQ/emvnM9ULfohYf3/znnpNDvrpUg4KdimmTrbQDbP6b6aG++DcyTC2vspWuXXb5U57bNVHGme7eHVlW976Y+e8O7n5Reqo7ijFFXUL3TBhVcmzlsG+PdPXDx/fPFlRHJpXqTWeMdz4w0e1dXHmTbrCKT1JwS69p9mWuX02+WHSYdvjRt3uN+62wGlvJyw9Q5dipPc02zJ3LPDZocH6YdSKZdwqZtrbCUvPULBL72mnUli/NmrFMm4Vs6wVTuk6Bbv0nnYqhfVro1Ys41Yxy1rhlK5TsEvvabZlbn9g24Vg/TBqxTJuFTPt7YSlZ+jmqfSeZlvmhh2rv2kZdbvfuNsCp72dsPQMBbskJ69VvbC5wkSpI0atWMatYnZS4RSpUrBLMvJa1Qub654fNu60mJdZRRKia+ySjLxW9cLmCttpMQ+ziiREwS7JyGtVL261UaTAFOySjLxW9eJWG0UKTMEuychrVS9srrCdFvMwq0hCdPNUkpHXql7caqNIgUUKdjObAdzh7pc1+fXZwG3AKcA24GOexYepSrbiVvXuu3Xqh0EHd0uMuq5Z5TJsrrCdG/Vh01ICLS/FmNkcYAtw4TTLrgJ2uPvZwIIWa0Um3XcrPParyZaKj1ee33dr++tq1cbaTdBajTFsC92gsNc+9qt45xLJWMtgd/cj7n4WsGOaZSuA+6uPHwIuSGA26QVbH4l2PMq6TiqXYa8NUiVSCiKpm6cLgf3VxweA44MLzGyNmW02s817RkYS+rZSeME+ebPjUdZ1UrmMWnVUJVIKIKlg3wPMqz6eV33ewN3Xu/uwuw+fMDSU0LeVwmv2OaDB41HWdVK5jFp1VCVSCiCpYH8QWFl9vALYkNB5pezOPj/a8SjrOqlchr02SJVIKYi2g93M3mJmNwUO3wKcbGbbgL1Ugl6ktZVXhn8YdLDtEmVdJx9wHfZafdi0FJRl0Uo8d+lS37B2bde/by+b/9H+1otEJNds8Jot7t7y3YV+8lREpGQU7CIiJaNgFxEpGQW7iEjJKNhFREpGwS4iUjIKdhGRklGwi4iUjIJdRKRkFOwiIiWjYBcRKRkFu4hIySjYRURKRsEuIlIyCnYRkZJRsIuIlIyCXUSkZBTsIiIlo2AXESkZBbuISMko2EVESmYg6wGK7u6ROXxr7xAvj/azeGCMTx8/wiVDR7IeS0R6mIK9A3ePzOGrr8zjqFf+4bNzdICvvjIPQOEuIpnRpZgOfGvv0ESo1xz1Pr61dyijiUREFOwdeXm0v63jIiLdoGDvwOKBsbaOi4h0g4K9A58+foTZNt5wbLaN8+njRzKaSEREN087UrtBqlaMiOTJtMFuZrOB24BTgG3Ax9zdA2suAr4LPF89dI27P5X8qPl0ydARBbmI5EqrSzFXATvc/WxgAXBhk3Xfcff3VL96JtSbuXtkDhe/sIhzn1nCxS8s4u6ROVmPJCI9pFWwrwDurz5+CLigyborzGyTmd1uZpbYdAVU67bvHB3AsYluu8JdRLqlVbAvBPZXHx8Ajg9Z8wzwRXdfBiwB3p/ceMWjbruIZK1VsO8B5lUfz6s+D9oLPFB9/DywKOxEZrbGzDab2eY9I+VtjajbLiJZaxXsDwIrq49XABtC1nwOWG1mfcCZwPawE7n7encfdvfhE4bK++5V3XYRyVqrYL8FONnMtlF5Z/6Mmd0UWHMz8HHgUeBOd388+TGLQ912EcnatHVHdz8GXBo4fF1gzU7gA8mOVVzqtotI1vQDSkTfenfNjuPZdGzWxPNls47xoXlHprw2ye8pItKung/2qFvvTob6ZJtz07FZ/Gb3LLx6bOfoAF/ePR/HGaX5+bTdr4ikqef3iolaTwyGeoVNhHrN69hEqDc7nyqRIpKmng/2btUT68+nSqSIpKnng71b9cT686kSKSJp6vlgj1pPXDbrGNCw/xngWODYDJwBpj+fKpEikqaeD/ZLho7wpRP3s2RgFMNZMjDKl07cP+Um5vo37a0L98rXslnHuHHRvobXfmXRPr66aPrzRf2eIiJx9HwrBuB3R2awa7QfB3aN9vO7IzP47/1zQquNL+4dmKgofmjekdBte6Ns+KXtfkUkLT0f7DfunsutI4PUGi/jUH0OraqNYRVFVRlFJGs9fynm9rpQn2Shx4LVxrCKoqqMIpK1ng/28dZLphWsKKrKKCJZ6/lg7/Q3IFhRVJVRRLLW88F+xdAhwmqMUaqNYRVFVRlFJGs9H+xrFx3gyqFD9FXDvA/nyqFDkaqNYRVFVRlFJGulasVE3THxxt1zuX1kkHEqf7MNMj5xrX0c2HJ4Ji+MNf7WbDk2k627Z3JsohXTz1d2z+OfXxniFZ+8fn6ijfHZE1u/O9fujiKSltIEe9SaYVi9cYQ+6lswz47NqD6aPDZW/ao/dgyqoT557BXv5+93z584pt0dRaTbSnMpJmrNsHm9Mfg8uWPa3VFEuqk0wR61ZthpvTEu7e4oIt1SmmCPWjPM6j9YuzuKSLeUJtij1gyb1xuDz5M7pt0dRaSbShPsUWuGYfXGIcaprzae2v86/XXPwenHmRU4NgvnRBtrOHaijfFPLWqRqkSKSJrMPfiOM33nLl3qG9au7fr37WXzP6rr9yJFZ4PXbHH34VbrSlN3bCZuXzzYda9cwmHKsbWLDqQ5vohI20od7HH74lG28q0/pnAXkTwpzTX2MHH74u1s5Xv7ROCLiORDqYM9bl+8na57Vr14EZFmSh3scfvi7fymlPo3UEQKqdS5FLcv3s5WvrWbqiIieVHqm6e1G6TttmJqN0PVihGRIpo22M1sNnAbcAqwDfiYB4rvUdZk6ZKhI7F+8GftogOhoa0gF5G8a3Up5ipgh7ufDSwALoy5RkREuqRVsK8A7q8+fgi4IOYaERHpklbX2BcC+6uPDwBvj7kGM1sDrKk+PbhgzZqn2hu1LScAe1I8f9qSn39N6yUJ0u9/doo8O2j+Vt4cZVGrYN8DzKs+nkf4wFHW4O7rgfVRhuqUmW2Osp9CXmn+bBV5/iLPDpo/Ka0uxTwIrKw+XgFsiLlGRES6pFWw3wKcbGbbgL3AM2Z2U4s1DyY/poiIRDXtpRh3PwZcGjh8XYQ1WevKJZ8Uaf5sFXn+Is8Omj8RmezHLiIi6Sn1lgIiIr2olMFuZjPM7MdZz9Euq/i+mT1iZj8ys0Jt+WBmA2Z2q5ltNLPvZT1PHGb2WTN7IOs52mVmF5nZDjN7uPoVWjvOMzP7OzP7pZndY2Yzs56nHWb2gbrf+xfN7Oos5yldsJvZHGALxfwJ2OXAgLufD8xlsm1UFB8Ctrr7cmCJmZ2T9UDtMLM3A3+d9Rwd+I67v6f6lebPiSTOzE4F3unu7wXuAd6U8Uhtcfef1X7vqWyt8rss5yldsLv7EXc/C9iR9Swx7ALWVR+/luUgMf0U+Gb1XxrzqfzAWpGsA67PeogOXGFmm8zsdjMLfipM3v0JsMDMfgG8F3gu43liMbPjgNPcfVuWc5Qu2IvM3Z92901m9mFgJnBv1jO1w90PuvthYCOwy92fzXqmqMzsI8BW4PGsZ4npGeCL7r4MWAK8P+N52nUi8Iq7v4/Ku/X3ZDxPXBeSg8q3gj1nzOyDwLXAZe4+/SeC5IyZLTSzWcC7qbz7KtK+QZdSedf4Q+A8M/tUxvO0ay9QuzfwPLAou1FiOQDULh89C5yc4SyduAz4SdZDKNhzxMwWA18AVrn79J8Gkk+fB66s/oV0GJiT8TyRuftHqtdHVwNb3P3mrGdq0+eA1WbWB5wJbM94nnZtAd5VfXwalXAvlOrlrwuobIaYKQV7vlxN5Z/R91bvrn8i64Ha9G3gE2b2a+APFOxSUsHdDHwceBS4090LdUnJ3X8N7DGz3wBPufumrGeK4V3A/7j70awH0Q8oiYiUjN6xi4iUjIJdRKRkFOwiIiWjYBcRKRkFu4hIySjYRURKRsEuIlIy/w/3YZRXTIfS+QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11d92c0b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_decision_boundary(dt_clf, axis=[0.5, 7.5, 0, 3])\n",
    "plt.scatter(X[y==0, 0], X[y==0, 1])\n",
    "plt.scatter(X[y==1, 0], X[y==1, 1])\n",
    "plt.scatter(X[y==2, 0], X[y==2, 1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模拟使用信息熵进行划分\n",
    "\n",
    "`inf` 正无穷的值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 切分函数\n",
    "def split(X, y, d, value):\n",
    "    index_a = (X[:, d] <= value)\n",
    "    index_b = (X[:, d] > value)\n",
    "    return X[index_a], X[index_b], y[index_a], y[index_b]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "def entropy(y):\n",
    "    counter = Counter(y)\n",
    "    res = 0.0\n",
    "    for num in counter.values():\n",
    "        p = num / len(y)\n",
    "        res += -p * np.log(p)\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 在数据上寻找_a、_b\n",
    "def try_split(X, y):\n",
    "    # 每次找到最小的熵，更新\n",
    "    best_entropy = float('inf')\n",
    "    best_d, best_v = -1, -1\n",
    "    # 穷搜\n",
    "    for d in range(X.shape[1]):\n",
    "        sorted_index = np.argsort(X[:,d])\n",
    "        for i in range(1, len(X)):\n",
    "            if X[sorted_index[i-1], d] != X[sorted_index[i], d]:\n",
    "                v = (X[sorted_index[i-1], d] + X[sorted_index[i], d]) / 2\n",
    "                X_l, X_r, y_l, y_r = split(X, y, d, v)\n",
    "                e = entropy(y_l) + entropy(y_r)\n",
    "                if e < best_entropy:\n",
    "                    best_entropy, best_d, best_v = e, d, v\n",
    "\n",
    "    return best_entropy, best_d, best_v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best_entropy =  0.6931471805599453\n",
      "best_d =  0\n",
      "best_v =  2.45\n"
     ]
    }
   ],
   "source": [
    "best_entropy, best_d, best_v = try_split(X, y)\n",
    "print('best_entropy = ', best_entropy)\n",
    "print('best_d = ', best_d)\n",
    "print('best_v = ', best_v)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 第一次最佳划分，在第0 维度，2.45左右"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "X1_l, X1_r, y1_l, y1_r = split(X, y, best_d, best_v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "entropy(y1_l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6931471805599453"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "entropy(y1_r)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 对右边再一次划分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best_entropy =  0.4132278899361904\n",
      "best_d =  1\n",
      "best_v =  1.75\n"
     ]
    }
   ],
   "source": [
    "best_entropy2, best_d2, best_v2 = try_split(X1_r, y1_r)\n",
    "print('best_entropy = ', best_entropy2)\n",
    "print('best_d = ', best_d2)\n",
    "print('best_v = ', best_v2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "X2_l, X2_r, y2_l, y2_r = split(X1_r, y1_r, best_d2, best_v2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.30849545083110386"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "entropy(y2_l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.10473243910508653"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "entropy(y2_r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4rc1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}