ch06-梯度下降法/05-sklearn中梯度下降法.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# sklearn中随机梯度下降\n",
    "参见 自己封装[代码](playML/LinearRegression.py)\n",
    "\n",
    "科学方法，把所有的样本都看1遍，把所有的信息都考虑进来了\n",
    "\n",
    "\n",
    "`np.random.permutation(m)` 生成随机序列，乱序，重新排列\n",
    "\n",
    "### 具体实现也是挺复杂的\n",
    "\n",
    "计算每1个 $\\theta$ 都是取固定的1项，这个前提不变\n",
    "\n",
    "改进：每一轮迭代，取不同样本，更新1次 $\\theta$，直到把所有的样本都用到，更新m次 $\\theta$\n",
    "\n",
    "考虑到所有的`样本`信息\n",
    "\n",
    "迭代次数，要整体`看样本`看几遍\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = 10000\n",
    "\n",
    "x = np.random.normal(size=m)\n",
    "X = x.reshape(-1, 1)\n",
    "y = 4.*x + 3. + np.random.normal(0, 3, size=m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression()"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from playML.LinearRegression import LinearRegression\n",
    "\n",
    "lin_reg = LinearRegression()\n",
    "lin_reg.fit_sgd(X, y, n_iters=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([4.03882587])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lin_reg.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.0337469167769564"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lin_reg.intercept_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 随机梯度下降\n",
    "def fit_sgd(self, X_train, y_train, n_iters=50, t0=5, t1=50):\n",
    "    \"\"\"根据训练数据集X_train, y_train, 使用梯度下降法训练Linear Regression模型\"\"\"\n",
    "    assert X_train.shape[0] == y_train.shape[0], \\\n",
    "        \"the size of X_train must be equal to the size of y_train\"\n",
    "    assert n_iters >= 1\n",
    "\n",
    "    def dJ_sgd(theta, X_b_i, y_i):\n",
    "        return X_b_i * (X_b_i.dot(theta) - y_i) * 2.\n",
    "\n",
    "    def sgd(X_b, y, initial_theta, n_iters=5, t0=5, t1=50):\n",
    "\n",
    "        def learning_rate(t):\n",
    "            return t0 / (t + t1)\n",
    "\n",
    "        theta = initial_theta\n",
    "        m = len(X_b)\n",
    "\n",
    "        # 科学方法，把所有的样本都看1遍，把所有的信息都考虑进来了\n",
    "        for i_iter in range(n_iters):\n",
    "            indexes = np.random.permutation(m) # 随机打乱序列\n",
    "            X_b_new = X_b[indexes,:] # 新样本序列\n",
    "            y_new = y[indexes]\n",
    "            for i in range(m):\n",
    "                gradient = dJ_sgd(theta, X_b_new[i], y_new[i])\n",
    "                theta = theta - learning_rate(i_iter * m + i) * gradient\n",
    "\n",
    "        return theta\n",
    "\n",
    "    X_b = np.hstack([np.ones((len(X_train), 1)), X_train]) # 组合数组\n",
    "    initial_theta = np.random.randn(X_b.shape[1])\n",
    "    self._theta = sgd(X_b, y_train, initial_theta, n_iters, t0, t1)\n",
    "\n",
    "    self.intercept_ = self._theta[0]\n",
    "    self.coef_ = self._theta[1:]\n",
    "\n",
    "    return self"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br><br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 真实数据使用我们自己的SGD（波士顿房价）\n",
    "\n",
    "迭代次数小，很具有随机性\n",
    "\n",
    "增加迭代次数，结果越稳定"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import datasets\n",
    "\n",
    "boston = datasets.load_boston()\n",
    "X = boston.data\n",
    "y = boston.target\n",
    "\n",
    "X = X[y<50.0]\n",
    "y = y[y<50.0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from playML.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, seed=666)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "standardScaler = StandardScaler()\n",
    "\n",
    "standardScaler.fit(X_train)\n",
    "X_train_standard = standardScaler.transform(X_train)\n",
    "X_test_standard = standardScaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 10.7 ms, sys: 1.31 ms, total: 12 ms\n",
      "Wall time: 12.5 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.7538905076039513"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from playML.LinearRegression import LinearRegression\n",
    "\n",
    "lin_reg = LinearRegression()\n",
    "%time lin_reg.fit_sgd(X_train_standard, y_train, n_iters=2)\n",
    "lin_reg.score(X_test_standard, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 141 ms, sys: 15.1 ms, total: 156 ms\n",
      "Wall time: 145 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.8106749214200866"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from playML.LinearRegression import LinearRegression\n",
    "\n",
    "lin_reg = LinearRegression()\n",
    "%time lin_reg.fit_sgd(X_train_standard, y_train, n_iters=50)\n",
    "lin_reg.score(X_test_standard, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 245 ms, sys: 7.8 ms, total: 253 ms\n",
      "Wall time: 252 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.813282692448816"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from playML.LinearRegression import LinearRegression\n",
    "\n",
    "lin_reg = LinearRegression()\n",
    "%time lin_reg.fit_sgd(X_train_standard, y_train, n_iters=100)\n",
    "lin_reg.score(X_test_standard, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br><br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## scikit-learn中的SGD\n",
    "\n",
    "做了很多优化，实现也更复杂\n",
    "\n",
    "参数`n_iter`：浏览样本多少遍，默认值是5\n",
    "\n",
    "开源工程：[linear_model/stochastic_gradient.py](https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/linear_model/stochastic_gradient.py)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 只能解决线性模型\n",
    "from sklearn.linear_model import SGDRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3.04 ms, sys: 4.13 ms, total: 7.16 ms\n",
      "Wall time: 4.69 ms\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mgbook/.pyenv/versions/3.6.4rc1/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:144: FutureWarning: max_iter and tol parameters have been added in SGDRegressor in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.805097826144864"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_reg = SGDRegressor()\n",
    "%time sgd_reg.fit(X_train_standard, y_train)\n",
    "sgd_reg.score(X_test_standard, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 7.98 ms, sys: 3.44 ms, total: 11.4 ms\n",
      "Wall time: 44.7 ms\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mgbook/.pyenv/versions/3.6.4rc1/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:130: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.812996392314848"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_reg = SGDRegressor(n_iter=100)\n",
    "%time sgd_reg.fit(X_train_standard, y_train)\n",
    "sgd_reg.score(X_test_standard, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4rc1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}