Baseline model evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "07a9cfe7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import matthews_corrcoef\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn.metrics import mean_squared_error  \n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "cc8c25bd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     AAC_A  AAC_C  AAC_D  AAC_E  AAC_F  AAC_G  AAC_H  AAC_I  AAC_K  AAC_L  \\\n",
      "0     3.51  10.53   0.00   1.75  10.53  10.53   1.75   1.75   8.77   1.75   \n",
      "1    11.86  10.17   0.00   3.39   6.78   8.47   5.08   3.39  10.17   5.08   \n",
      "2    10.53  10.53   0.00   5.26   8.77   7.02   0.00   5.26   5.26   1.75   \n",
      "3     7.02  10.53   1.75   3.51   5.26   8.77   0.00   5.26  14.04   5.26   \n",
      "4     1.67  10.00   0.00   5.00   5.00   8.33   1.67   6.67  11.67   5.00   \n",
      "..     ...    ...    ...    ...    ...    ...    ...    ...    ...    ...   \n",
      "384   6.82   0.00   6.82  15.91   2.27   2.27   0.00   6.82  18.18   4.55   \n",
      "385   4.35   0.00   0.00  17.39   0.00   0.00   0.00   4.35  13.04  13.04   \n",
      "386  12.50   0.00   0.00   6.25   6.25  12.50   0.00   0.00  18.75   0.00   \n",
      "387  12.50   0.00   0.00   6.25   6.25  12.50   0.00   0.00  18.75   0.00   \n",
      "388  11.76   0.00   0.00   5.88   8.82   8.82   0.00   5.88   5.88  17.65   \n",
      "\n",
      "     ...  AAC_N  AAC_P  AAC_Q  AAC_R  AAC_S  AAC_T  AAC_V  AAC_W  AAC_Y  \\\n",
      "0    ...   8.77   7.02   1.75   7.02   8.77   7.02   3.51   0.00   5.26   \n",
      "1    ...   5.08   6.78   3.39   5.08   3.39   3.39   1.69   1.69   5.08   \n",
      "2    ...   5.26   5.26   0.00   8.77   5.26   5.26   5.26   0.00   8.77   \n",
      "3    ...   3.51   7.02   1.75   8.77   3.51   3.51   1.75   1.75   7.02   \n",
      "4    ...   6.67   5.00   6.67  11.67   3.33   5.00   0.00   1.67   5.00   \n",
      "..   ...    ...    ...    ...    ...    ...    ...    ...    ...    ...   \n",
      "384  ...   2.27   4.55   4.55   2.27   6.82  11.36   0.00   0.00   0.00   \n",
      "385  ...  13.04   8.70   8.70   0.00   0.00   4.35   8.70   0.00   0.00   \n",
      "386  ...   0.00   6.25   6.25   0.00   6.25  12.50  12.50   0.00   0.00   \n",
      "387  ...   0.00   6.25   6.25   0.00   6.25  12.50  12.50   0.00   0.00   \n",
      "388  ...   0.00   2.94   0.00   2.94   0.00   5.88  14.71   2.94   2.94   \n",
      "\n",
      "     Activity  \n",
      "0           1  \n",
      "1           1  \n",
      "2           1  \n",
      "3           1  \n",
      "4           1  \n",
      "..        ...  \n",
      "384         0  \n",
      "385         0  \n",
      "386         0  \n",
      "387         0  \n",
      "388         0  \n",
      "\n",
      "[389 rows x 21 columns]\n"
     ]
    }
   ],
   "source": [
    "#import data and split\n",
    "peptide = pd.read_excel(r'imput.xlsx',sheet_name='Sheet1')\n",
    "print (peptide)\n",
    "X=peptide.drop(['Activity'], axis=1)\n",
    "y=peptide['Activity']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "345dea91",
   "metadata": {},
   "outputs": [],
   "source": [
    "from numpy import mean\n",
    "from numpy import std\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import RepeatedStratifiedKFold\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from matplotlib import pyplot\n",
    "# get a list of models to evaluate\n",
    "def get_models():\n",
    "    models = dict()\n",
    "    models['SGD'] = SGDClassifier(loss = 'log') \n",
    "    models['KNN'] = KNeighborsClassifier()\n",
    "    models['DT'] = DecisionTreeClassifier()\n",
    "    models['SVM'] = SVC()\n",
    "    models['NB'] = GaussianNB()\n",
    "    return models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "222384bf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">SGD 0.937 (0.033)\n",
      ">KNN 0.960 (0.025)\n",
      ">DT 0.953 (0.037)\n",
      ">SVM 0.976 (0.022)\n",
      ">NB 0.964 (0.020)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAay0lEQVR4nO3df3Dc9Z3f8ecrsrGC+SUXhbnYBjs9D0j1gZNuHZL4crjkGpOkoeU6PftC03gMrqaxQoGbM7VoIZPaoZ2DCyc4dlxMOHqpuJTChdy4IZ0USj1NAzKWASOTU+wcFuYO+ezgxI7PNn73j+8XvKxX2q+tlXb3q9djRjP6fj+f7+77+93dl7767PeHIgIzM8uv99W7ADMzm1gOejOznHPQm5nlnIPezCznHPRmZjk3rd4FVHLhhRfGvHnz6l2GmVnT2Lp1676IaK/U1pBBP2/ePPr7++tdhplZ05D0l6O1eejGzCznHPRmZjnnoDczyzkHvZlZzjnozcxyrmrQS3pI0puSXh6lXZL+UNKQpBclfaSkbZmkV9O222pZ+ETp6+tj4cKFtLS0sHDhQvr6+updkpnZuGTZo38YWDZG+zXAgvRnNfAAgKQW4P60vRNYIalzPMVOtL6+Pnp6eujt7eXIkSP09vbS09PjsDezplY16CPiWWD/GF2uBR6JxP8DLpD0K8BiYCgidkXEUeDRtG/DWr9+PZs2bWLp0qVMnz6dpUuXsmnTJtavX1/v0szMzlgtTpiaDewpmR5O51Wa/9HRHkTSapL/CLj44otrUNbpGxwcZMmSJe+Zt2TJEgYHB+tSz2SRNO7HyMt9DabKtpg1axYHDhyodxm0tbWxf/9Y+5GNodnfF7X4MrbSFogx5lcUERsjohARhfb2imfxTriOjg62bNnynnlbtmyho6OjLvVMlogY8ydrnzyYKtviwIEDVddjMn4a4Y9NFs3+vqhF0A8Dc0um5wB7x5jfsHp6eli1ahVPP/00x44d4+mnn2bVqlX09PTUuzQzszNWi6GbJ4E1kh4lGZp5KyLekDQCLJA0H3gdWA78Tg2eb8KsWLECgO7ubgYHB+no6GD9+vXvzjcza0ZVg15SH3AVcKGkYeAOYDpARBSBzcBngCHgMLAybTsuaQ3wFNACPBQROyZgHWpqxYoVDnYzy5WqQR8RY6ZeJINPXx6lbTPJHwIzM6sTnxlrZpZzDnozs5xz0NuUMWvWLCSN6wcY92PMmjWrzlvCppqGvMOU2UR459jxeqvFyTdmp8N79GZmOeegNzPLOQe9mVnOOejNzHLOQW9mlnMOejOznHPQm5nlnIPezCznHPRmZjnnoDczyzkHvZlZzjnozcxyzkFvZpZzmYJe0jJJr0oaknRbhfY2SU9IelHSc5IWlrTdLGmHpJcl9UlqreUKmJnZ2KoGvaQW4H7gGqATWCGps6zbOmAgIi4Hvgjcmy47G/gKUIiIhST3jl1eu/LNzKyaLHv0i4GhiNgVEUeBR4Fry/p0Aj8AiIidwDxJF6Vt04D3S5oGnA3srUnlZmaWSZagnw3sKZkeTueV2g5cByBpMXAJMCciXgd+H3gNeAN4KyK+X+lJJK2W1C+pf2Rk5PTWwszMRpUl6CvdDqf8Nj13AW2SBoBuYBtwXFIbyd7/fOCDwExJ11d6kojYGBGFiCi0t7dnrd/MzKrIEvTDwNyS6TmUDb9ExMGIWBkRi0jG6NuB3cCngN0RMRIRx4DHgY/XonAzq5+RwyN86XtfYt8v99W7FMsgS9A/DyyQNF/SWSRfpj5Z2kHSBWkbwA3AsxFxkGTI5kpJZyu5UebVwGDtyjezeii+WOSFv36B4vZivUuxDKoGfUQcB9YAT5GE9LcjYoekLkldabcOYIeknSRH59yULvsj4DHgBeCl9Pk21nwtzGzSjBwe4TtD3yEI/mzoz7xX3wSmZekUEZuBzWXziiW//xBYMMqydwB3jKNGM2sgxReLnIgTAJyIExS3F7n9ytvrXJWNxWfGmllm7+zNHztxDIBjJ455r74JZNqjz6PkK4PxiSg/+KjxzJo1iwMHDoz7cca7vdra2ti/f/+467DaiDvOgzvPP+3lin+njRPnnAPvO/l+OHHsCMUHC9z+N6f/Pos7zjvtZWptKnxGpmzQVwtpSU0R5NUcOHCgIdajFn9YrXb01YNntNzf/Wo77z/vva/lsfeJhw9M49+dwWO2tbWx/84zKqVm9n/lbaD+f3Dg7Ql75Ckb9GZTWS3++J+yM/Tvx/2QdaGvHmyYnaG4c2Ie22P0ZmY556A3M8s5B72ZWc456M3Mcs5Bb2aWcw56M7Occ9CbmeWcg97MbBya4ZLNDnozs3Fohks2O+jNzM5Qs1yy2UFvZnaGKl2yuRE56M3MzkAzXbI5U9BLWibpVUlDkm6r0N4m6QlJL0p6TtLCkrYLJD0maaekQUkfq+UKmJnVQ+ne/Dsada++atBLagHuJ7lFYCewQlJnWbd1wEBEXE5yc/B7S9ruBb4XEZcBV+B7xjaVZjiiwKwetr+5/d29+XccO3GMgTcH6lPQGLJcpngxMBQRuwAkPQpcC7xS0qcT+DpAROyUNE/SRcAvgU8CX0rbjgJHa1a9TbjSIwp8uzizkx77/GP1LiGzLEM3s4E9JdPD6bxS24HrACQtBi4B5gAfAkaAb0raJulBSTPHXbVNimY5osDMxpYl6CvdGqj8Kv13AW2SBoBuYBtwnOQ/ho8AD0TEh4FDwClj/ACSVkvql9Q/MjKSsXybSM1yRIGZjS1L0A8Dc0um5wB7SztExMGIWBkRi0jG6NuB3emywxHxo7TrYyTBf4qI2BgRhYgotLe3n95aWM010xEFZja2LEH/PLBA0nxJZwHLgSdLO6RH1pyVTt4APJuG/18BeyRdmrZdzXvH9q1BNdMRBWY2tqpfxkbEcUlrgKeAFuChiNghqSttLwIdwCOS3iYJ8lUlD9ENfCv9Q7ALWFnjdbAJ0ExHFJjZ2NQIN8UtVygUor+/v641nHLj4ybVKOvRCHU0Qg2NVMd4eT0aqw5JWyOiUKnNZ8aameWcg94sI588Zs3KQW+WUTNcjtasEge9WQY+ecyamYPeLAOfPGbNzEFvVoVPHrNm56A3q8Inj1mzc9CbVeGTx6zZZblMsdmU1kyXozWrJJ9nxt55fu2KGa8736rz83tbnHx+b4vTIVW6cO3pacR8KVeL9ayFtrY29u/ff8bLj3VmbC6DPi+nNDcKr0c+67DaaYTX1JdAMDObwhz0ZmY556A3M8s5B72ZWc456M3Mcs5Bb2aWc5mCXtIySa9KGpJ0W4X2NklPSHpR0nOSFpa1t0jaJunPa1W4mZllUzXoJbUA9wPXAJ3ACkmdZd3WAQMRcTnwReDesvabgMHxl2tmZqcryx79YmAoInZFxFHgUeDasj6dwA8AImInME/SRQCS5gCfBR6sWdVmZpZZlqCfDewpmR5O55XaDlwHIGkxcAkwJ237BvB7wAnGIGm1pH5J/SMjIxnKMjOzLLIEfaULQZSf63sX0CZpAOgGtgHHJX0OeDMitlZ7kojYGBGFiCi0t7dnKMvMzLLIEvTDwNyS6TnA3tIOEXEwIlZGxCKSMfp2YDfwCeDzkn5KMuTzDyX9SQ3qnlC+CbSZ5UmWoH8eWCBpvqSzgOXAk6UdJF2QtgHcADybhv+/jYg5ETEvXe5/RcT1Nax/Qvgm0GaWJ1WDPiKOA2uAp0iOnPl2ROyQ1CWpK+3WAeyQtJPk6JybJqrgieabQJtZ3mS68UhEbAY2l80rlvz+Q2BBlcd4BnjmtCucZJVuAn37lbfXuSozszPnM2NL+CbQZpZHDvoSvgm0meWRg76EbwJtZnnkm4OX8E2gzSyPvEdvZpZzDnozs5xz0JuZ5ZyD3sws5xz0ZmY556A3M8s5B72ZWc456M3Mcs5Bb2aWcw56Mzst3d3dtLa2IonW1la6u7vrXZJV4aA3s8y6u7spFots2LCBQ4cOsWHDBorFosO+wSmi/Pav9VcoFKK/v/+Ml5dEI6xXo9QxXl6PfNZxJlpbW9mwYQO33HLLu/Puuece1q1bx5EjR+pYWX01wmsqaWtEFCq21bu4SmoR9I2gra2N/fv317uMqmqxvRrxfVTO74vxk8ShQ4c4++yz3513+PBhZs6c2RTvgTPVDJ+RsYI+09CNpGWSXpU0JOm2Cu1tkp6Q9KKk5yQtTOfPlfS0pEFJOyRNyi0GI2LcP7V4nGb5MNdqezU6vy/Gb8aMGRSL770/Q7FYZMaMGXWqaHI0+2ek6mWKJbUA9wO/CQwDz0t6MiJeKem2DhiIiH8q6bK0/9XAceDWiHhB0rnAVkn/s2xZM2sSN954I2vXrgWgq6uLYrHI2rVr6erqqrKk1VOW69EvBoYiYheApEeBa4HSsO4Evg4QETslzZN0UUS8AbyRzv+5pEFgdtmyZtYkent7AVi3bh233norM2bMoKur69351piyDN3MBvaUTA+n80ptB64DkLQYuASYU9pB0jzgw8CPKj2JpNWS+iX1j4yMZCrezCZfb28vR44cISI4cuSIQ74JZAn6St9ClA843QW0SRoAuoFtJMM2yQNI5wD/Hfg3EXGw0pNExMaIKEREob29PUvtZmaWQZahm2Fgbsn0HGBvaYc0vFcCKPl6enf6g6TpJCH/rYh4vAY1m5nZaciyR/88sEDSfElnAcuBJ0s7SLogbQO4AXg2Ig6mob8JGIyIe2pZuJmZZVN1jz4ijktaAzwFtAAPRcQOSV1pexHoAB6R9DbJF62r0sU/AfwL4KV0WAdgXURsru1qmJnZaLIM3ZAG8+ayecWS338ILKiw3BYqj/Gbmdkk8bVuzMxyzkFvZpZzDnozs5xz0JuZ5ZyD3sws5xz0ZmY556A3M8s5B72ZWc456M3Mcs5Bb2aWcw56M7Occ9CbmeWcg97MLOcc9GZmOeegNzPLOQe9mVnOOejNzHIuU9BLWibpVUlDkm6r0N4m6QlJL0p6TtLCrMuamdnEqhr0klqA+4FrgE5ghaTOsm7rgIGIuBz4InDvaSxrZmYTKMse/WJgKCJ2RcRR4FHg2rI+ncAPACJiJzBP0kUZlzUzswmUJehnA3tKpofTeaW2A9cBSFoMXALMybgs6XKrJfVL6h8ZGclWvU2ovr4+Fi5cSEtLCwsXLqSvr6/eJZnZGcgS9KowL8qm7wLaJA0A3cA24HjGZZOZERsjohARhfb29gxl2UTq6+ujp6eH3t5ejhw5Qm9vLz09PQ57syaUJeiHgbkl03OAvaUdIuJgRKyMiEUkY/TtwO4sy1pjWr9+PZs2bWLp0qVMnz6dpUuXsmnTJtavX1/v0szsNGUJ+ueBBZLmSzoLWA48WdpB0gVpG8ANwLMRcTDLsvUiacyfrH3yanBwkCVLlrxn3pIlSxgcHKxTRWZ2pqoGfUQcB9YATwGDwLcjYoekLkldabcOYIeknSRH2Nw01rK1X43TFxHj/smzjo4OtmzZ8p55W7ZsoaOjo04VmdmZmpalU0RsBjaXzSuW/P5DYEHWZa3x9fT0sGrVKjZt2sSSJUvYsmULq1at8tCNWRPKFPQ29axYsQKA7u5uBgcH6ejoYP369e/ON7PmoUYcgigUCtHf31/vMsxOISn3w3bWnCRtjYhCpTZf68bMLOcc9GZmOeegNzPLOQe9mVnOOejNzHLOQW9mlnMOejOznHPQm5nlnIPezCznHPRmZjnnoDczyzkHvZlZzjnozcxyzkFvZpZzmYJe0jJJr0oaknRbhfbzJX1X0nZJOyStLGm7OZ33sqQ+Sa21XAEzMxtb1aCX1ALcT3KLwE5ghaTOsm5fBl6JiCuAq4C7JZ0laTbwFaAQEQuBFpL7xpqZ2STJske/GBiKiF0RcRR4FLi2rE8A5yq5Y/Y5wH7geNo2DXi/pGnA2cDemlRuZmaZZAn62cCekunhdF6p+0huEL4XeAm4KSJORMTrwO8DrwFvAG9FxPcrPYmk1ZL6JfWPjIyc5mqYmdlosgS9Kswrv5fap4EB4IPAIuA+SedJaiPZ+5+fts2UdH2lJ4mIjRFRiIhCe3t7xvLNzKyaLEE/DMwtmZ7DqcMvK4HHIzEE7AYuAz4F7I6IkYg4BjwOfHz8ZZuZWVZZgv55YIGk+ZLOIvky9cmyPq8BVwNIugi4FNiVzr9S0tnp+P3VwGCtijczs+qmVesQEcclrQGeIjlq5qGI2CGpK20vAl8DHpb0EslQz9qI2Afsk/QY8ALJl7PbgI0TsypmZlaJIsqH2+uvUChEf39/vcswO4UkGvEzYyZpa0QUKrX5zFgzs5xz0JuZ5ZyD3sws5xz0ZmY556A3M8u5qodXmk0lyeke4+vjo3Ks0TjozUo4pC2PPHRjZpZzDnozs5xz0JuZ5ZyD3sws5xz0ZmY556A3M8s5B72ZWc456M3Mcs5Bb2aWcw56M7OcyxT0kpZJelXSkKTbKrSfL+m7krZL2iFpZUnbBZIek7RT0qCkj9VyBczMbGxVg15SC3A/cA3QCayQ1FnW7cvAKxFxBXAVcHd6I3GAe4HvRcRlwBX45uBmZpMqyx79YmAoInZFxFHgUeDasj4BnKvksn7nAPuB45LOAz4JbAKIiKMR8bNaFW9mZtVlCfrZwJ6S6eF0Xqn7gA5gL/AScFNEnAA+BIwA35S0TdKDkmZWehJJqyX1S+ofGRk53fUwM7NRZAn6ShffLr+W66eBAeCDwCLgvnRvfhrwEeCBiPgwcAg4ZYwfICI2RkQhIgrt7e3Zqjczs6qyBP0wMLdkeg7JnnuplcDjkRgCdgOXpcsOR8SP0n6PkQS/mZlNkixB/zywQNL89AvW5cCTZX1eA64GkHQRcCmwKyL+Ctgj6dK039XAKzWp3MzMMql6h6mIOC5pDfAU0AI8FBE7JHWl7UXga8DDkl4iGepZGxH70ofoBr6V/pHYRbL3b2Zmk0SNeOu0QqEQ/f399S7DzKxpSNoaEYVKbT4z1sws5xz0ZmY556A3M8s5B72ZWc456M3Mcs5Bb2aWcw56M7Occ9CbmeWcg97MLOcc9GW6u7tpbW1FEq2trXR3d9e7JDOzcXHQl+ju7qZYLLJhwwYOHTrEhg0bKBaLDnsza2q+1k2J1tZWNmzYwC233PLuvHvuuYd169Zx5MiRSa/HzCyrsa5146AvIYlDhw5x9tlnvzvv8OHDzJw5k0bcTmZm7/BFzTKaMWMGxWLxPfOKxSIzZsyoU0VmZuNX9Xr0U8mNN97I2rVrAejq6qJYLLJ27Vq6urrqXJmZ2Zlz0Jfo7e0FYN26ddx6663MmDGDrq6ud+ebmTUjj9GbmeXAuMfoJS2T9KqkIUm3VWg/X9J3JW2XtEPSyrL2FknbJP35ma2CmZmdqapBL6kFuB+4BugEVkjqLOv2ZeCViLgCuAq4O71H7DtuAgZrUrGZmZ2WLHv0i4GhiNgVEUeBR4Fry/oEcK4kAecA+4HjAJLmAJ8FHqxZ1WZmllmWoJ8N7CmZHk7nlboP6AD2Ai8BN0XEibTtG8DvAScYg6TVkvol9Y+MjGQoy8zMssgS9Kowr/wb3E8DA8AHgUXAfZLOk/Q54M2I2FrtSSJiY0QUIqLQ3t6eoSwzM8siy+GVw8Dckuk5JHvupVYCd0VyCM+QpN3AZcAngM9L+gzQCpwn6U8i4vqxnnDr1q37JP1l1pWYIBcC++pcQ6PwtjjJ2+Ikb4uTGmFbXDJaQ9XDKyVNA34MXA28DjwP/E5E7Cjp8wDw1xFxp6SLgBeAKyJiX0mfq4DfjYjPnfl6TB5J/aMdqjTVeFuc5G1xkrfFSY2+Laru0UfEcUlrgKeAFuChiNghqSttLwJfAx6W9BLJUM/a0pA3M7P6yXRmbERsBjaXzSuW/L4X+EdVHuMZ4JnTrtDMzMbFFzUb3cZ6F9BAvC1O8rY4ydvipIbeFg15CQQzM6sd79GbmeWcg97MLOemZNBL6kkvvvaipAFJH5U0TdIGSX+RzhuQ1FOyzNvpvB3pxdtukdTU20/SL0p+/0y67hdLulPSYUkfGKVvSLq7ZPp3Jd05aYVPgtFeb0mfLnl//CK92N+ApEfqXXMtVfiM/A9JXy/rs0jSYPr7TyX9n7L2AUkvT2bdE22s9376uXk9Xe+dkh5olIxoiCImk6SPAZ8DPhIRlwOfIrnEw38gObP31yJiEfDrwPSSRX8ZEYsi4u8Bvwl8BrhjMmufKJKuBnqBZRHxWjp7H3DrKIv8LXCdpAsno746qfh6R8RT6fxFQD/whXT6i/UstpZG+YzcBfx2WdflwH8tmT5X0tz0MTomo9Y6qPbe/4P0vdEJ/BrwG5NV2FimXNADvwLsi4i/BUiP9/8ZcCPQHRFH0vk/j4g7Kz1ARLwJrAbWpBdya1qSfh34z8BnI+InJU0PAb8taVaFxY6THGVw8ySUWHd5er0zOuUzEhH/G/iZpI+W9PvnJBc5fMe3OfnHYAXQNxnFTrKs7/2zSK4GcGDCK8pgKgb994G5kn4s6Y8k/Qbwq8BrEfHzrA8SEbtItt8HqvVtYDOA7wD/JCJ2lrX9giTsbxpl2fuBL0g6fwLraxg5eb2zqvQZgSS4lwNIuhL4m4j4i5LlHgOuS3//x8B3J6vgSTbWe/9mSQPAG8CPI2JgMgsbzZQL+oj4BfD3SfbQRoA/JbmG/rskrUzH2fa886/oKJp97+4Y8H+BVaO0/yHwLyWdV94QEQeBR4CvTFx5DafZX+9MKn1GJH2JZO/9n6Xjzss5dY99P3BA0nKS+08cnrSiJ1GV9/47QzcfAGam26LuplzQA0TE2xHxTETcAawh2fu4WNK5afs30xfrLZLLPpxC0oeAt4E3J6fqCXGC5N/vfyBpXXljRPyMZAz2X4+y/DdI/kjMnKD6GkZOXu/MKnxGfisi9gA/JRl3/i2SoZpyf0qyx5vHYZtS32CM935EHAO+B3xyEmsa1ZQLekmXSlpQMmsR8CqwieTyyq1pvxaScbZKj9EOFIH7osnPOIuIwyRfvH1BUqU9+3uAf0WFy2VExH6SD/to/xHkQp5e7yxG+Yy8czXZPuAPgJ9ExHCFxZ8A/hPJtbFyq9p7P/0u5+PATyq1T7ZM17rJmXOAXkkXkHyxMkTyL+pbJBdne1nSz4FfAn/MyUsyvz8de5ueLvdfSEKw6UXEfknLgGcl7Str2yfpCUb/8ulukj2+vMnt653BaJ8RgP8G3At0V1ow/Z7rPwJMge+tK733b5Z0Pcn75kXgjya9qgp8CQQzs5ybckM3ZmZTjYPezCznHPRmZjnnoDczyzkHvZlZzjnozcxyzkFvZpZz/x98tg7w2pQ3fQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "# evaluate a given model using cross-validation\n",
    "def evaluate_model(model, X, y):\n",
    "    cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "    scores = cross_val_score(model, X, y, scoring='accuracy', cv=cv, error_score='raise')\n",
    "    return scores\n",
    " \n",
    "# get the models to evaluate\n",
    "models = get_models()\n",
    "# evaluate the models and store results\n",
    "results, names = list(), list()\n",
    "for name, model in models.items():\n",
    "    scores = evaluate_model(model, X, y)\n",
    "    results.append(scores)\n",
    "    names.append(name)\n",
    "    print('>%s %.3f (%.3f)' % (name, mean(scores), std(scores)))\n",
    "# plot model performance for comparison\n",
    "pyplot.boxplot(results, labels=names, showmeans=True)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "69bdb134",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">SGD 0.943 (0.067)\n",
      ">KNN 0.932 (0.076)\n",
      ">DT 0.948 (0.056)\n",
      ">SVM 0.974 (0.054)\n",
      ">NB 0.955 (0.066)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAYcElEQVR4nO3df3Bc13ne8e8jCBRjSJapEtZYImXJE44CirE1Nko7Nf1jx3FMqlHVyB6LqDuqNWhQTUXU46YeqYZnrFRd1a3jxgqkBMMWrKu2phI7ZkxrXEmZCI2KtJ4QSiiJP0QbphURYaYCQ1qyRVMEibd/7IW4Anexl8Audu/d5zOzA+w99yzePbv74uy5556riMDMzPLromYHYGZmjeVEb2aWc070ZmY550RvZpZzTvRmZjl3cbMDqGT16tVx7bXXNjsMM7PMePrpp49FRHelspZM9Ndeey0TExPNDsPMLDMk/VW1Mg/dmJnlnBO9mVnOOdGbmeWcE72ZWc450ZuZ5VzNRC9ph6SXJO2rUi5JvyNpUtKzkt5dVrZZ0qGk7J56Bm5mZumk6dF/Ddi8QPkWYF1yGwB+D0BSB/BQUr4e6JO0finBmpnZhauZ6CPiKeD4ArvcAjwcJd8D3iLpbcBGYDIiDkfEaeCRZF8zM1tG9Thh6mrgSNn9qWRbpe3vrfYgkgYofSPgmmuuWVJAV1xxBSdOnFjSY9TDqlWrOH58of+Rjee2OMdtUebey5v798vd+3KT/37+26IeiV4VtsUC2yuKiO3AdoDe3t4lXQ3lxIkTtMIFVaRKTbC83BbnuC3KNDu5thD95ist876Iexvz2PVI9FPA2rL7a4CjwIoq283MbBnVY3rlbuD2ZPbN+4CXI+JvgD3AOknXSVoBbE32NTOzZVSzRy9pJ/BhYLWkKeCLQCdARIwA3wVuAiaBk8AdSdkZSduAx4EOYEdE7G/AczAzswXUTPQR0VejPIC7qpR9l9I/AjMzaxKfGWtmlnNO9GZmOedEb2aWc070ZmY550RvZpZzTvRmZjnnRG9mlnNO9GZmOedEb2aWc070ZnbBpk9O8+nHPs2xnx1rdiiWghO9mV2wkWdH+Iv/9xeMPDPS7FAsBSd6M7sg0yen+fbktwmCP5r8I/fqM8CJ3swuyMizI8zGLACzMetefQY40ZtZanO9+ZnZGQBmZmfcq88AJ3ozS628Nz/HvfrW50RvZqk989Izr/fm58zMzrD3pb3NCchSqcc1Y82sTXzzH3yz2SHYIrhHb2aWc070ZmY550RvZpZzTvRmZjnnRG9mlnNO9GZmS5CFBd6c6M3MliALC7w50ZuZLVJWFnhLleglbZZ0SNKkpHsqlK+StEvSs5L+XNKGsrIXJD0naa+kiXoGb2bWTFlZ4K1mopfUATwEbAHWA32S1s/b7fPA3oh4J3A78MC88kJE3BgRvXWI2cys6bK0wFuaHv1GYDIiDkfEaeAR4JZ5+6wH/gQgIp4HrpV0ZV0jNWuyLBx0s+WTpQXe0iT6q4EjZfenkm3lngFuBZC0EXg7sCYpC+AJSU9LGqj2RyQNSJqQNDE9PZ02frNlk4WDbrZ8srTAW5pFzVRhW8y7/yXgAUl7geeAvwTOJGXvj4ijkt4K/LGk5yPiqfMeMGI7sB2gt7d3/uObNdX8g253vutOVv/c6maH1RQ7d+6kWCxy8OBBenp6GBoaoq+vr9lhLbssLfCWJtFPAWvL7q8BjpbvEBGvAHcASBLwo+RGRBxNfr4kaReloaDzEn0rmT45zeee+hy/9aHfatsPcx7FF98M916+qLojf2cVs5deCheJ2ZlTjPznXr7wtycWH0dG7dy5k6GhIUZHR9m0aRPj4+P09/cDtGWyzwpFLNx5lnQx8H3gI8BfA3uAfxQR+8v2eQtwMiJOS/p14AMRcbukLuCiiPhJ8vsfA/8mIh5b6G/29vbGxMTiJ+hIotbzWsh937uPbxz6Bp+8/pN84X1faFoc9dAKMbRKHIuNYfrkNFu+tYXXzr72+rZLOi7hsY8/tqiOQCu0xWJt2LCB4eFhCoXC69vGxsYYHBxk3759TYxs8Vrl9VhqHJKerjbhpWaiTx7gJuCrQAewIyKKku4EiIgRSb8EPAycBQ4A/RFxQtI7gF3Jw1wMfD0iirX+3lIT/WJ7bQDTHRexZc1VvHbRRVwyO8tjU0dZfXa2dsWqsby8+Lp1kJc3cTNjuO9797HrB7veMB7beVEnt667dVEdgVZoi8Xq6Ojg1KlTdHZ2vr5tZmaGlStXcvbs2SZGtnit8no0MtGnuvBIRHwX+O68bSNlv/9fYF2FeoeBd11QtHWg33xl0Q028r37mP3BLpidYfbiSxj56G8sulcvibh3UVXrZinDFXWPI6OydNCt0Xp6ehgfH39Dj358fJyenp4mRmW1+ApTZarNi83ygbel/NOraxwt8E9vsbJ00K3RhoaGuO222+jq6uLFF1/kmmuu4dVXX+WBB+afOmOtxEsglMnSvFizZmuFDoSl40Rfxl/RzRZWLBYZGBigq6sLSXR1dTEwMECxWPPQmzWRh27K+Cv6+fI21bQ0+7e5Vq1a1ewQFu3AgQOcPHnyvOmVL7zwQrNDswW4R28LytPZoBGx5Fs9Huf48eNNbonFW7FiBdu2baNQKNDZ2UmhUGDbtm2sWLGi2aHZApzoraqsLMFqy+f06dMMDw8zNjbGzMwMY2NjDA8Pc/r06WaHtiSSmn5r5Dc9D91YVZWWYF3KCWSWfevXr2fdunVs2bKF1157jUsuuYQtW7bQ1dXV7NAWrR4HlVtlLn417tFbRVlagtWWT6FQ4NFHH+X+++/n1Vdf5f777+fRRx99w7x6az1O9FaRp5paJWNjY9x9993s2LGDyy67jB07dnD33XczNjbW7NBsAamWQFhuzV7rpl5aIY7FxvCJ3Z/g0IlD522/ftX1i5qd1AptUQ95eR6LlcclEOqhFd4XS14CwdqPp5paJV4CIZs8dGNmqQ0NDdHf3/+GWTf9/f0MDQ01OzRbgHv0Zpba3Jrzg4ODr194pFgsei36Fucx+gZqhThaIYZWimOp8vI8rL5a4X2x0Bi9h27MzHLOid7MLOdyO0bvxavMzEpymejb4ZRmM7O0PHRjZpZzTvRmZjnnRG9mlnNO9GZmOedEb2aWc070ZmY550RvZpZzqRK9pM2SDkmalHRPhfJVknZJelbSn0vakLaumZk1Vs1EL6kDeAjYAqwH+iStn7fb54G9EfFO4HbggQuoa2ZmDZSmR78RmIyIwxFxGngEuGXePuuBPwGIiOeBayVdmbKumZk1UJpEfzVwpOz+VLKt3DPArQCSNgJvB9akrEtSb0DShKSJ6enpdNGbmVlNaRJ9pdXB5i8C8yVglaS9wCDwl8CZlHVLGyO2R0RvRPR2d3enCMvMzNJIs6jZFLC27P4a4Gj5DhHxCnAHgErLRv4oub2pVl0zM2usND36PcA6SddJWgFsBXaX7yDpLUkZwD8FnkqSf826ZmbWWDV79BFxRtI24HGgA9gREfsl3ZmUjwA9wMOSzgIHgP6F6jbmqZiZWSW5vGZsPeRlPfpWeR6tEsdS5eV5WH21wvvC14w1M2tjTvRmZjnnRG9mlnNO9GZmOedEb2aWc070ZmY5l+bMWDOztlY64X9p+zRz+qUTvZlZDc2eI79UHroxM8s59+jbQJqvnY22atWqZodg1rac6HOuHl85W+H0bjNbPCd6szJZP+hmVokTvVkZJ2nLIx+MNTPLOSd6M7sgO3fuZMOGDXR0dLBhwwZ27tzZ7JCsBg/dmFlqO3fuZGhoiNHRUTZt2sT4+Dj9/f0A9PX1NTk6q8Y9ejNLrVgsMjo6SqFQoLOzk0KhwOjoKMVisdmh2QJ8hakqPKXwHLeFzeno6ODUqVN0dna+vm1mZoaVK1dy9uzZJkZmvsKUmdVFT08P4+Pjb9g2Pj5OT09PkyKyNJzozSy1oaEh+vv7GRsbY2ZmhrGxMfr7+xkaGmp2aLYAH4w1s9TmDrgODg5y8OBBenp6KBaLPhDb4jxGX4XHpc9xW5i1Po/Rm5m1MSd6M7Occ6I3M8u5VIle0mZJhyRNSrqnQvnlkr4j6RlJ+yXdUVb2gqTnJO2V1NyBdzOzNlRz1o2kDuAh4KPAFLBH0u6IOFC2213AgYi4WVI3cEjS/4iI00l5ISKO1Tt4MzOrLU2PfiMwGRGHk8T9CHDLvH0CuEylhbovBY4DZ+oaqZmZLUqaRH81cKTs/lSyrdyDQA9wFHgO+ExEzCZlATwh6WlJA9X+iKQBSROSJqanp1M/ATMzW1iaRF/pcjrzJ1V/DNgLXAXcCDwo6c1J2fsj4t3AFuAuSR+s9EciYntE9EZEb3d3d5rYzcwshTSJfgpYW3Z/DaWee7k7gG9FySTwI+AXACLiaPLzJWAXpaEgs0zxGuyWZWkS/R5gnaTrJK0AtgK75+3zIvARAElXAtcDhyV1Sbos2d4F/Aqwr17Bmy2HuTXYh4eHOXXqFMPDwwwNDTnZW2akWgJB0k3AV4EOYEdEFCXdCRARI5KuAr4GvI3SUM+XIuK/S3oHpV48lGb4fD0iai5c7SUQWku7t8WGDRsYHh6mUCi8vm1sbIzBwUH27XO/xVrDQksgeK2bKto9uZVr97bwGuyWBV7rxmwJvAa7ZZ0TvVkNXoPdss6J3qyGvr4+isUig4ODrFy5ksHBwbZeg90zkLLHFx4xS6Gvr69tE3u5uRlIo6OjbNq0ifHxcfr7+wHcPi3MPXozS61YLDI6OkqhUKCzs5NCocDo6CjFYs3JdNZEnnVTRbvPNCnntrA5noHUujzrxszqwjOQssmJ3sxS8wykbPLBWDNLbe6A6+DgIAcPHqSnp6etZyBlhcfoq/C49DluC7PW5zF6M7M25kRvZpZzTvRmZjnnRG9mlnNO9GZmOedEb2aWc070ZmaLlJWVPH3ClJnZImRpJU/36M3MFiFLK3m27Zmxkpb8GK3Ydo3gM2PNztdqK3n6zNgKImLJNzNrX1laybNtE72Z2VJkaSVPH4w1M1uELK3k2bZj9Jaex+jNWp/H6M3M2liqRC9ps6RDkiYl3VOh/HJJ35H0jKT9ku5IW9fMzBqrZqKX1AE8BGwB1gN9ktbP2+0u4EBEvAv4MPAVSStS1jUzswZK06PfCExGxOGIOA08Atwyb58ALlNpcvqlwHHgTMq6ZmbWQGkS/dXAkbL7U8m2cg8CPcBR4DngMxExm7IuAJIGJE1Impienk4ZvpmZ1ZIm0Vc6hXT+FIyPAXuBq4AbgQclvTll3dLGiO0R0RsRvd3d3SnCMjOzNNIk+ilgbdn9NZR67uXuAL4VJZPAj4BfSFnXzMwaKE2i3wOsk3SdpBXAVmD3vH1eBD4CIOlK4HrgcMq6ZmbWQDXPjI2IM5K2AY8DHcCOiNgv6c6kfAS4D/iapOcoDdfcHRHHACrVbcxTMTOzSnxmrNXkM2PNWp/PjDWzusnKVZWWQ1bawouamVlqWbqqUqNlqi3qsS57vW/vec97wlpH6W1iFnHDDTfEk08++YZtTz75ZNxwww1Niqh5Wq0tgImoklM9Rm81eYze5rTaVZWaqdXawmP0ZlYXWbqqUqNlqS2c6M0stSxdVanRstQWPhhrZqll6apKjZaltvAYvdXkMXqz1ucxejOzNuZEb2aWc070ZmY550RvZpZzTvTzZGXtCjOztDy9skym1q4wM0vJPfoyxWKR0dFRCoUCnZ2dFAoFRkdHKRaLzQ7NzGzRPI++TKutXdEqPI/erPV5Hn1KWVq7wswsLSf6Mllau8LMmi8rkzd8MLZMltauMLPmytLkDY/RW00eozc734YNGxgeHqZQKLy+bWxsjMHBQfbt27fs8Sw0Ru9EbzU50Zudr9Umb/hgrJlZnWVp8oYTvZnZImRp8oYPxpqZLUKWJm94jN5q8hi9Wetb8hi9pM2SDkmalHRPhfLPSdqb3PZJOivpiqTsBUnPJWXO3mZmy6zm0I2kDuAh4KPAFLBH0u6IODC3T0R8Gfhysv/NwGcj4njZwxQi4lhdIzczs1TS9Og3ApMRcTgiTgOPALcssH8f0Jqnh5mZtaE0if5q4EjZ/alk23kkvQnYDPxh2eYAnpD0tKSBan9E0oCkCUkT09PTKcIyM7M00iR6VdhW7cjczcCfzRu2eX9EvBvYAtwl6YOVKkbE9ojojYje7u7uFGGZmVkaaRL9FLC27P4a4GiVfbcyb9gmIo4mP18CdlEaCjIzs2WSJtHvAdZJuk7SCkrJfPf8nSRdDnwI+HbZti5Jl839DvwKsPyLQJiZtbGas24i4oykbcDjQAewIyL2S7ozKR9Jdv014ImIeLWs+pXALklzf+vrEfFYPZ+AmZktzCdMWU0+Ycqs9XlRMzOzNuZEb2aWc070Zilk5ZJxZpV49UqzGrJ0yTizStyjN6uhWCwyOjpKoVCgs7OTQqHA6OgoxWKx2aGZpeJZN1ZTu8+6abVLxplV4lk3ZkuQpUvGmVXiRG9WQ5YuGWdWiQ/GmtWQpUvGmVXiMXqrqd3H6M2ywGP0ZmZtzInezCznnOjNzHLOid7MLOec6M3Mcs6J3sws55zozcxyzonezCznnOjNzHLOid7MLOec6M3Mcs6J3sws57x6pSFpyft40TOz1uVEb07SZjmXauhG0mZJhyRNSrqnQvnnJO1NbvsknZV0RZq6ZmbWWDUTvaQO4CFgC7Ae6JO0vnyfiPhyRNwYETcC/xr404g4nqaumZk1Vpoe/UZgMiIOR8Rp4BHglgX27wN2LrKumZnVWZpEfzVwpOz+VLLtPJLeBGwG/vBC65qZWWOkSfSVpltUO3p3M/BnEXH8QutKGpA0IWlieno6RVhmZpZGmkQ/Bawtu78GOFpl362cG7a5oLoRsT0ieiOit7u7O0VYZmaWRppEvwdYJ+k6SSsoJfPd83eSdDnwIeDbF1rXzMwap+Y8+og4I2kb8DjQAeyIiP2S7kzKR5Jdfw14IiJerVW33k/CzMyqUyueLCNpGvirJoexGjjW5BhahdviHLfFOW6Lc1qhLd4eERXHvVsy0bcCSRMR0dvsOFqB2+Ict8U5botzWr0tvKiZmVnOOdGbmeWcE31125sdQAtxW5zjtjjHbXFOS7eFx+jNzHLOPXozs5xzojczy7m2TPSShiTtl/Rssob+eyVdLOl+ST8oW1t/qKzO2WTbfknPSPqXkjLdfpJ+Wvb7Tclzv0bSvZJOSnprlX1D0lfK7v8rSfcuW+DLoNrrLeljZe+PnybXWtgr6eFmx1xPFT4j/1PSv5u3z42SDia/vyDpf88r3ytp33LG3WgLvfeTz81fJ8/7eUm/1yo5oiWCWE6Sfgn4VeDdEfFO4JcprbD5b4GrgF9M1tX/ANBZVvVnyZr7NwAfBW4CvricsTeKpI8Aw8DmiHgx2XwM+I0qVV4DbpW0ejnia5KKr3dEPF527YUJ4FPJ/dubGWw9VfmMfAm4bd6uW4Gvl92/TNLa5DF6liPWJqj13v/t5L2xHvhFSsvCNF3bJXrgbcCxiHgNICKOAT8Gfh0YjIhTyfafRMS9lR4gIl4CBoBtSnPB1RYm6QPAfwL+fkT8sKxoB3Db3JXC5jlDaZbBZ5chxKbL0+ud0nmfkYj4U+DHkt5btt8nKV1jYs4fcO6fQfl1KfIk7Xt/BbASONHwiFJox0T/BLBW0vcl/a6kDwE/D7wYET9J+yARcZhS+7211r4t7BJKi9D9w4h4fl7ZTykl+89UqfsQ8KlkMbvcy8nrnValzwiUEvdWAEnvA/42In5QVu+bwK3J7zcD31mugJfZQu/9z0raC/wN8P2I2LucgVXTdok+In4KvIdSD20a+H3gw+X7SLojGWc7MvdVtIqs9+5mgP8D9Fcp/x3gn0h68/yCiHgFeBj4F40Lr+Vk/fVOpdJnRNKnKfXeP5GMO89fkhzgOHBC0lbgIHBy2YJeRjXe+3NDN28FupK2aLq2S/QAEXE2Iv5XRHwR2Eap93GNpMuS8v+SvFgvU1p18zyS3gGcBV5anqgbYpbS1++/K+nz8wsj4seUxmD/eZX6X6X0T6KrQfG1jJy83qlV+Ix8PCKOAC9QGnf+OKWhmvl+n1KPN4/DNuW+ygLv/YiYAR4DPriMMVXVdole0vWS1pVtuhE4BIwCD0pamezXQWmcrdJjdAMjwIOR8TPOIuIkpQNvn5JUqWf/H4F/RoUlrZMrif0B1b8R5EKeXu80qnxG5laT3Qn8NvDDiJiqUH0X8B8oLU2eW7Xe+8mxnL8H/LBS+XKruR59Dl0KDEt6C6UDK5OUvqK+DNwH7JP0E+BnwH/l3BWxfi4Ze+tM6v03Skkw8yLiuKTNwFOSjs0rOyZpF9UPPn2FUo8vb3L7eqdQ7TMC8A3gAWCwUsXkONe/B2iD49aV3vuflfSPKb1vngV+d9mjqsBLIJiZ5VzbDd2YmbUbJ3ozs5xzojczyzknejOznHOiNzPLOSd6M7Occ6I3M8u5/w9Q2Tfq1x1uOwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.model_selection import RepeatedKFold\n",
    "# evaluate a given model using cross-validation\n",
    "def evaluate_model(model, X, y):\n",
    "    cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "    scores = cross_val_score(model, X, y, scoring='precision', cv=cv, error_score='raise')\n",
    "    return scores\n",
    "# get the models to evaluate\n",
    "models = get_models()\n",
    "# evaluate the models and store results\n",
    "results, names = list(), list()\n",
    "for name, model in models.items():\n",
    "    scores = evaluate_model(model, X, y)\n",
    "    results.append(scores)\n",
    "    names.append(name)\n",
    "    print('>%s %.3f (%.3f)' % (name, mean(scores), std(scores)))\n",
    "# plot model performance for comparison\n",
    "pyplot.boxplot(results, labels=names, showmeans=True)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "d9b79c41",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">SGD 0.904 (0.134)\n",
      ">KNN 0.984 (0.026)\n",
      ">DT 0.962 (0.036)\n",
      ">SVM 0.970 (0.036)\n",
      ">NB 0.969 (0.036)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAATjUlEQVR4nO3dfZBd913f8fenazvmIQ/2eAPED5E7NemKhXhgcULHaeKhaeQAdUkyRSKdNJltXTdYZdInXJY27lAXKJMSUBw0BhsaZrqmDUlsWmMzw4gEtXTqVSobKYqDYsASZuoVdpw4juOV9O0fe2Vdr+/uXkl399797fs1szP3nN/vnPvVufd+9Lvn4Z5UFZKkje+vDLsASdJgGOiS1AgDXZIaYaBLUiMMdElqxHnDeuJLLrmktmzZMqynl6QNad++fceqarxX29ACfcuWLczNzQ3r6SVpQ0ryZ8u1uctFkhphoEtSIwx0SWqEgS5JjTDQJakRqwZ6kruSPJHkwDLtSfJLSQ4neTjJdw++TEnSavoZof86sG2F9uuBqzp/NwK/fO5lSZLO1KqBXlWfAZ5cocsNwMdq0f8GXpXk2wZVoCSpP4O4sOhS4EjX9NHOvL9Y2jHJjSyO4rniiivO+YkvvvhinnrqqXNez7m46KKLePLJlf6/Wye3vnLYFSy69elhV+D7opvvi64a2t8W6ecGF0m2AP+9qiZ7tP0P4Geqam9n+veAf1VV+1Za59TUVJ3rlaJJGPYNOkahhlGpYxRqGJU6RqGGUaljFGoYlToGUUOSfVU11attEGe5HAUu75q+DHh8AOvVOpp/dp733v9ejn3t2LBLkXSWBhHo9wLv6Zzt8kbg6ap6ye4WjbbdD+/ms//vs+x+aPewS5F0lvo5bXEW+EPgdUmOJplOclOSmzpd7gMeBQ4DvwK8f82q1ZqYf3aeew7fQ1F86vCnHKVLG9SqB0Wrascq7QX82MAq0rrb/fBuTtZJAE7WSXY/tJufeuNPDbkqSWfKK0U3uVOj84WTCwAsnFxwlC5tUAb6Jtc9Oj/l1Chd8mD5aRthWxjom9xDTzz0wuj8lIWTC+x/Yv9wCtJI8WD5aRthW/R1Hvpa8Dz09uoYhRpGpY5RqOFc65h/dp7rP3E9Xz/xdV429jLuf+f9XPINl6xrDYPUyrZY6/PQpaZshK/W66HXwfLNaqNsCwNdWmIjfLVeax4sP20jbQsDvSFJhvp30UUXDXsTnDPPyV/kwfLTNtK2MNAbUVXn/Heu6xmJH6M6Rxvlq/Va82D5aRtpW3hQ9ByNQg2D0sq/5Wz/Hd0Hvk452wNgo7ItR6GOUahhVOrwoKi0TjbSV2upl0H8Hro0UuqDrzir375+6DXfysLLLnjRvIWTC+x/+Dfg/p8/8xqkdWagqzn5d18+q6+1Hx9kDQl16wBXKPVhQwf62Y7EBl6DpA0hyVCff63PBNvQgX62I7GB1uBITCOu9RDr1yCyYhQOrK5kQwe6tBxDbNFmCDGdZqCrOYaYNitPW5SkRjhC3yT63QWxWj9HrdLoMtA3CYNYap+7XCSpEQa6JDXCQJekRmzqQPfONJJa0legJ9mW5JEkh5Pc0qP9oiSfTPJwkv+TZHLwpQ6ed6aR1JJVAz3JGHA7cD2wFdiRZOuSbj8J7K+q7wLeA/zioAsdNO9MI6k1/YzQrwEOV9WjVfU8cDdww5I+W4HfA6iqzwNbknzLQCsdMO9MI6k1/QT6pcCRrumjnXndHgLeAZDkGuC1wGVLV5TkxiRzSebm5+fPruIB2Eg3fZWkfvUT6L0uHVx6lcrPAhcl2Q/sBP4vcPwlC1XdUVVTVTU1Pj5+prUOjHemkdSifq4UPQpc3jV9GfB4d4eq+jLwPoAsXjv+J52/kbSRbvoqSf3qJ9AfBK5KciXw58B24Ee7OyR5FfBsZx/7PwQ+0wn5kfTxvzPIe9NI0mhYNdCr6niSm4EHgDHgrqo6mOSmTvtuYAL4WJITwOeA6TWsWTon/lCZWtXXj3NV1X3AfUvm7e56/IfAVYMtTVobBrFateF/bdE700hnz28rbdnQge6daaRz43u/LRs60CVpUFr4tmKgSxJtfFvZ1L+2KEktMdAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDWir0BPsi3JI0kOJ7mlR/srk/x2koeSHEzyvsGXKklayaqBnmQMuB24HtgK7EiydUm3HwM+V1WvB94CfCjJBQOuVZK0gn5G6NcAh6vq0ap6HrgbuGFJnwJeniTANwNPAscHWqkkaUX9BPqlwJGu6aOded0+AkwAjwN/BPx4VZ1cuqIkNyaZSzI3Pz9/liVLknrpJ9DTY14tmX4bsB94DXA18JEkr3jJQlV3VNVUVU2Nj4+fYamSpJX0E+hHgcu7pi9jcSTe7X3AJ2rRYeBPgL8+mBIlSf3oJ9AfBK5KcmXnQOd24N4lfR4Dvh8gybcArwMeHWShkqSVnbdah6o6nuRm4AFgDLirqg4muanTvhv4aeDXk/wRi7tofqKqjq1h3ZKkJVYNdICqug+4b8m83V2PHwf+9mBLkySdCa8UlaRGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqRF9/R76RpX0uh3qmferWnoLVUkaPU0HukEsaTNxl4skNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqRF+BnmRbkkeSHE5yS4/2f5lkf+fvQJITSS4efLmSpOWsGuhJxoDbgeuBrcCOJFu7+1TVz1fV1VV1NfCvgU9X1ZNrUK8kaRn9jNCvAQ5X1aNV9TxwN3DDCv13ALODKE6S1L9+Av1S4EjX9NHOvJdI8o3ANuC3lmm/Mclckrn5+fkzrVWStIJ+Ar3XD50sd039DwH/c7ndLVV1R1VNVdXU+Ph4vzVKkvrQT6AfBS7vmr4MeHyZvttxd4skDUU/gf4gcFWSK5NcwGJo37u0U5JXAm8G7hlsiZKkfqz6a4tVdTzJzcADwBhwV1UdTHJTp313p+sPA79bVV9ds2olScvKsH5idmpqqubm5oby3JK0USXZV1VTvdq8UlSSGmGgS1IjNm2gz87OMjk5ydjYGJOTk8zOenKOpI2t6VvQLWd2dpaZmRnuvPNOrr32Wvbu3cv09DQAO3bsGHJ1knR2NuVB0cnJSXbt2sV11133wrw9e/awc+dODhw4MJSaJKkfKx0U3ZSBPjY2xnPPPcf555//wryFhQUuvPBCTpw4MZSaJKkfnuWyxMTEBHv37n3RvL179zIxMTGkiiTp3G3KQJ+ZmWF6epo9e/awsLDAnj17mJ6eZmZmZtilSdJZ25QHRU8d+Ny5cyeHDh1iYmKC2267zQOikja0TbkPXZI2KvehS9ImYKBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqRF9BXqSbUkeSXI4yS3L9HlLkv1JDib59GDLlCStZtXfQ08yBtwOvBU4CjyY5N6q+lxXn1cBHwW2VdVjSV69RvVKkpbRzwj9GuBwVT1aVc8DdwM3LOnzo8AnquoxgKp6YrBlSpJW00+gXwoc6Zo+2pnX7duBi5L8fpJ9Sd7Ta0VJbkwyl2Rufn7+7CqWJPXUT6Cnx7yltzk6D/ge4AeAtwH/Jsm3v2ShqjuqaqqqpsbHx8+4WEnS8vq5p+hR4PKu6cuAx3v0OVZVXwW+muQzwOuBLwykSknSqvoZoT8IXJXkyiQXANuBe5f0uQd4U5Lzknwj8Abg0GBLlSStZNURelUdT3Iz8AAwBtxVVQeT3NRp311Vh5LcDzwMnAR+taoOrGXhkqQXS9XS3eHrY2pqqubm5oby3JK0USXZV1VTvdq8UlSSGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRvQV6Em2JXkkyeEkt/Rof0uSp5Ps7/z928GXKklayXmrdUgyBtwOvBU4CjyY5N6q+tySrn9QVT+4BjVKkvrQzwj9GuBwVT1aVc8DdwM3rG1ZkqQz1U+gXwoc6Zo+2pm31PcleSjJ7yT5jl4rSnJjkrkkc/Pz82dRriRpOf0EenrMqyXTnwVeW1WvB3YBn+q1oqq6o6qmqmpqfHz8jAqVJK2sn0A/ClzeNX0Z8Hh3h6r6clU903l8H3B+kksGVqUkaVX9BPqDwFVJrkxyAbAduLe7Q5JvTZLO42s66/3LQRcrSVreqme5VNXxJDcDDwBjwF1VdTDJTZ323cC7gH+S5DjwNWB7VS3dLSNJWkMZVu5OTU3V3NzcUJ5bkjaqJPuqaqpXm1eKSlIjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktSIvgI9ybYkjyQ5nOSWFfp9b5ITSd41uBIlSf1YNdCTjAG3A9cDW4EdSbYu0+/ngAcGXaQkaXX9jNCvAQ5X1aNV9TxwN3BDj347gd8CnhhgfZKkPvUT6JcCR7qmj3bmvSDJpcAPA7tXWlGSG5PMJZmbn58/01olSSvoJ9DTY14tmf4w8BNVdWKlFVXVHVU1VVVT4+PjfZYoSerHeX30OQpc3jV9GfD4kj5TwN1JAC4B3p7keFV9ahBFSpJW188I/UHgqiRXJrkA2A7c292hqq6sqi1VtQX4OPD+UQ/z2dlZJicnGRsbY3JyktnZ2WGXJEnnZNURelUdT3Izi2evjAF3VdXBJDd12lfcbz6KZmdnmZmZ4c477+Taa69l7969TE9PA7Bjx44hVydJZydVS3eHr4+pqamam5sbynNPTk6ya9currvuuhfm7dmzh507d3LgwIGh1CRJ/Uiyr6qmerZtxkAfGxvjueee4/zzz39h3sLCAhdeeCEnTqx4XFeShmqlQN+Ul/5PTEywd+/eF83bu3cvExMTQ6pIks7dpgz0mZkZpqen2bNnDwsLC+zZs4fp6WlmZmaGXZoknbV+TltszqkDnzt37uTQoUNMTExw2223eUBU0oa2KfehS9JG5T50SdoEDHRJaoSBLkmNMNAlqREGuiQ1YmhnuSSZB/5sKE/+YpcAx4ZdxIhwW5zmtjjNbXHaKGyL11ZVz98fH1qgj4okc8udArTZuC1Oc1uc5rY4bdS3hbtcJKkRBrokNcJAhzuGXcAIcVuc5rY4zW1x2khvi02/D12SWuEIXZIaYaBLUiOaDfQkM0kOJnk4yf4kb0hyXpL/kOSPO/P2J5npWuZEZ97BJA8l+WdJNvw2SvJM1+O3d/79VyS5NcmzSV69TN9K8qGu6X+R5NZ1K3wdLPeaJ3lb13vkmSSPdB5/bNg1D0qPz8jvJPmZJX2uTnKo8/hPk/zBkvb9SZq6b+NK7/vOZ+bPO//uzyf55VHKiJEpZJCSfB/wg8B3V9V3AX8LOAL8e+A1wHdW1dXAm4Dzuxb9WlVdXVXfAbwVeDvwwfWsfS0l+X5gF7Ctqh7rzD4G/PNlFvk68I4kl6xHfUPS8zWvqgc6868G5oB3d6bfM8xiB2WZz8jPAj+ypOt24L90Tb88yeWddbR6i6/V3ve/0HlfbAW+E3jzehW2miYDHfg24FhVfR2gqo4BXwL+EbCzqp7rzP9KVd3aawVV9QRwI3BzkqxH0WspyZuAXwF+oKq+2NV0F/AjSS7usdhxFo/qf2AdShy61l7zVbzkM1JVnwa+lOQNXf3+HnB31/R/5XTo7wBm16PYddbv+/4C4ELgqTWvqE+tBvrvApcn+UKSjyZ5M/DXgMeq6iv9rqSqHmVxG716tb4j7mXAPcDfrarPL2l7hsVQ//Fllr0deHeSV65hfSOjodd8Nb0+I7AY0NsBkrwR+Muq+uOu5T4OvKPz+IeA316vgtfZSu/7DyTZD/wF8IWq2r+eha2kyUCvqmeA72FxtDUP/Cbwlu4+Sd7X2Q925NRXyGW0MFJbAP4XML1M+y8B/yDJK5Y2VNWXgY8B/3Ttyhs5LbzmK+r1GUnyXhZH4+/q7BfezktH4E8CTyXZDhwCnl23otfRKu/7U7tcXg18U2dbjIQmAx2gqk5U1e9X1QeBm1kcTVyR5OWd9l/rvChPA2O91pHkrwIngCfWp+o1c5LFr87fm+QnlzZW1ZdY3E/6/mWW/zCL/xl80xrVNzIaes1X1eMz8s6qOgL8KYv7hd/J4i6WpX6TxRFsi7tbun2YFd73VbUA3A/8zXWsaUVNBnqS1yW5qmvW1cAjwJ3AR5Jc2Ok3xuJ+sF7rGAd2Ax+pBq6+qqpnWTwI9u4kvUbq/wn4x/S4cXhVPcniB3u5EX4TWnvNV7LMZ+TUr5/OAr8AfLGqjvZY/JPAfwQeWNMih2y1933nOMvfAL7Yq30YXvLhbcQ3A7uSvIrFAxyHWfxq+TTw08CBJF8Bvgb8Z+DxznLf0Nk3dn5nud9gMeiaUFVPJtkGfCbJsSVtx5J8kuUPBH2IxVFca5p+zVew3GcE4L8Bvwjs7LVg5zjUzwG0f+y45/v+A0n+PovvmYeBj657Vcvw0n9JakSTu1wkaTMy0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1Ij/j8wxo8KamJSnQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.model_selection import RepeatedKFold\n",
    "# evaluate a given model using cross-validation\n",
    "def evaluate_model(model, X, y):\n",
    "    cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "    scores = cross_val_score(model, X, y, scoring='recall', cv=cv, error_score='raise')\n",
    "    return scores\n",
    "# get the models to evaluate\n",
    "models = get_models()\n",
    "# evaluate the models and store results\n",
    "results, names = list(), list()\n",
    "for name, model in models.items():\n",
    "    scores = evaluate_model(model, X, y)\n",
    "    results.append(scores)\n",
    "    names.append(name)\n",
    "    print('>%s %.3f (%.3f)' % (name, mean(scores), std(scores)))\n",
    "# plot model performance for comparison\n",
    "pyplot.boxplot(results, labels=names, showmeans=True)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "e7014912",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">SGD 0.937 (0.046)\n",
      ">KNN 0.956 (0.043)\n",
      ">DT 0.955 (0.035)\n",
      ">SVM 0.972 (0.038)\n",
      ">NB 0.961 (0.041)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD4CAYAAADlwTGnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAa50lEQVR4nO3df5Ac5Z3f8fcna2HOgGEVFkWWBMKOClbRXXTORvhi7myVwlmQnAU4zknlnEG1PpmKpWDHubKKvYpxOUoUAsa+g2MjZ3WGxF6MOXTIKQKmKCVEleSsRV5JSEJmET+0SCWtLNkyhTEr6Zs/updtRrM7vbuzOzPbn1fV1E738zwz3+7tmW/30z39KCIwM7Pi+Vu1DsDMzGrDCcDMrKCcAMzMCsoJwMysoJwAzMwK6j21DmAsLr300pg/f36twzAzayjPPffc8YhoKZ3fUAlg/vz59PT01DoMM7OGIunVcvPdBWRmVlBOAGZmBeUEYGZWUE4AZmYF5QRgZlZQFROApM2Sjkl6foRySfozSX2Sdkv6cKZsuaQDadn6zPyZkp6W9GL6t7k6i2NmZnnlOQL4DrB8lPLrgQXpYw3wAICkJuD+tHwhsErSwrTNeuCZiFgAPJNOm5nZFKqYACLiWeDEKFVWAA9F4v8Bl0iaDSwB+iLiYES8DTyc1h1q82D6/EHgxnHGb2Zm41SNH4LNAQ5lpvvTeeXmX5M+nxURRwAi4oiky0Z6cUlrSI4suPzyy6sQ7ugkVeV1GmGchZkzZ3Ly5MmaxtDc3MyJE6PtX9QHbxdTq162i+m+LqqRAMp9MmKU+WMSEZuATQBtbW2T/ump9AGV1BAf4jxOnjxZ82Wp1hfrZPN2MbXqZbuY7uuiGlcB9QPzMtNzgcOjzAc4mnYTkf49VoU4zMxsDKqRALYCn02vBvoI8Iu0e2cHsEDSlZLOA1amdYfa3JI+vwV4vApxmJnZGFTsApLUDXwcuFRSP/BVYAZARHQCTwA3AH3Am8DqtOy0pLXAU0ATsDki9qYvuxF4RFI78Brw6Souk5mZ5VAxAUTEqgrlAXxhhLInSBJE6fyfActyxmhmZpPAvwQ2MysoJwAzs4JyAjAzKygnADOzgnICMDMrKCcAM7OCcgIwMysoJwAzs4Kqxs3gzGwaia++H+68uPYx2KRzAjCzd9HXTtXFHTDjzpqGUAjuAjIzKygnADOzgnICMDMrKCcAM7OCcgIwMysoJwAzs4JyAjAzKygnADOzgnICMDMrKCcAM7OCypUAJC2XdEBSn6T1ZcqbJW2RtFvSjyUtSudfJak38zgl6Ytp2Z2SXs+U3VDVJTMzs1FVvBeQpCbgfuA6oB/YIWlrROzLVLsD6I2ImyRdndZfFhEHgMWZ13kd2JJpd29E3F2VJTEzszHJcwSwBOiLiIMR8TbwMLCipM5C4BmAiHgBmC9pVkmdZcBLEfHqBGM2M7MqyJMA5gCHMtP96bysXcDNAJKWAFcAc0vqrAS6S+atTbuNNktqLvfmktZI6pHUMzAwkCNcMzPLI08CUJl5pfeK3Qg0S+oF1gE/AU6/8wLSecAngR9k2jwAfIiki+gIcE+5N4+ITRHRFhFtLS0tOcI1G7uZM2ciadwPYELtJTFz5swar4WJG3hzgFufvJXjvzpe61AshzwJoB+Yl5meCxzOVoiIUxGxOiIWA58FWoCXM1WuB3ZGxNFMm6MRcSYizgLfJulqMquJkydPEhE1fZw8ebLWq2HCOnd3svPoTjp3ddY6FMshTwLYASyQdGW6J78S2JqtIOmStAzgc8CzEXEqU2UVJd0/kmZnJm8Cnh9r8GZWPwbeHODxvscJgr/u+2sfBTSAilcBRcRpSWuBp4AmYHNE7JV0W1reCbQCD0k6A+wD2ofaS3ofyRVEny956bskLSbpTnqlTLmZNZDO3Z2cjbMAnI2zdO7q5E8/8qc1jmpipvvwmKr10G9j0dbWFj09PRN6jZkzZ9b8ULu5uZkTJ07UNAZIh92rh6H/6mAbrIc46iGG8cYx8OYA1z92Pb8+8+t35r236b08+aknufQ3Lp2SGCZDPcRRjRgkPRcRbaXzC/dLYPf1mlVfdu9/yNBRgNUvDwpfYNP98Namzq5juxg8O/iueYNnB+k91lubgCyXwnUBTZdDuukSRz3EUC9x1EMM9RJHPcRQL3FMZheQjwDM7BxDv22olebmsr8LtSpzAjCzd6nC3mbN95otn8KdBDYzs4QTgJlZQbkLyAxfEWXF5ARgBuhrp2reby2JuLOmIVgVDbw5wJ88+yfc/bG7x/VjuKngLiAzs0nQCDfGcwIwM6uyRrkxnhOAmVmVlbsxXj1yAjAzq6Khvf+hW2MMnh2s26MAJwAzsypqpBvjOQHYuHjov2FeF5bVSDfG82WgNi7ZKxwafdCPifK6sKxHP/lorUPIzUcANmaNcoXDVPC6sEbmBGBj1ihXOEwFrwtrZE4ANiaNdIXDZPO6sEbnBGBj0khXOEw2rwtrdLkSgKTlkg5I6pO0vkx5s6QtknZL+rGkRZmyVyTtkdQrqSczf6akpyW9mP71CBANoJGucJhsXhfW6CoOCSmpCfgpcB3QD+wAVkXEvkyd/wS8ERFfk3Q1cH9ELEvLXgHaIuJ4yeveBZyIiI1pUmmOiK+MFouHhJx+cdRDDPUSRz3EUA3TZTmgPpZlMoeEzHMEsAToi4iDEfE28DCwoqTOQuAZgIh4AZgvaVaF110BPJg+fxC4MUcsZmZWJXl+BzAHOJSZ7geuKamzC7gZ2C5pCXAFMBc4CgTwI0kB/OeI2JS2mRURRwAi4oiky8q9uaQ1wBqAyy+/PNdCjcb3fTczS+RJAOVGhy49HtkIfEtSL7AH+AlwOi37aEQcTr/gn5b0QkQ8mzfANGFsgqQLKG+7kfi+72ZmiTwJoB+Yl5meCxzOVoiIU8BqAEkCXk4fRMTh9O8xSVtIupSeBY5Kmp3u/c8Gjk1wWczMbAzynAPYASyQdKWk84CVwNZsBUmXpGUAnwOejYhTki6QdFFa5wLg94Hn03pbgVvS57cAj09sUczMbCwqHgFExGlJa4GngCZgc0TslXRbWt4JtAIPSToD7APa0+azgC3JQQHvAb4XEU+mZRuBRyS1A68Bn67eYpmZWSUVLwOtJ74MdPrFUQ8x1Esc9RBDNUyX5YD6WJZaXwZqZmbTkBOAmVlBOQGYmRWUE4CZWUE5AZiZFZQTgJlZQTkBmJkVlBPAGAy8OcCtT97qEZ/MCkRSTR/NzZM3VIoTwBh07u5k59GdHvHJrCAiYkKParzGiRMnJm35nAByGhr/NQiP+2pm04ITQE7Z8V897quZTQdOADkM7f0Pjf86eHbQRwFm1vCcAHLI7v0P8VHA9DOdT/aZlZNnQJjC23Vs1zt7/0MGzw7Se6y3NgFZ1VXhbos1v2uk2Vg5AeTw6CcfrXUIZmZV5y4gM7OCcgIwMysoJwAzs4LyOYCCS8drrhlf+WJWO04ABeYrX2w88uw05Knjbaf2cnUBSVou6YCkPknry5Q3S9oiabekH0talM6fJ2mbpP2S9kq6PdPmTkmvS+pNHzdUb7HMbLJM9N422fvkWG1VPAKQ1ATcD1wH9AM7JG2NiH2ZancAvRFxk6Sr0/rLgNPAlyNip6SLgOckPZ1pe29E3F3NBTIzs3zyHAEsAfoi4mBEvA08DKwoqbMQeAYgIl4A5kuaFRFHImJnOv+XwH5gTtWiNzOzccuTAOYAhzLT/Zz7Jb4LuBlA0hLgCmButoKk+cBvA3+Tmb027TbaLKns2UBJayT1SOoZGBjIEa6ZmeWRJwGUO5tT2oG3EWiW1AusA35C0v2TvIB0IfBXwBcj4lQ6+wHgQ8Bi4AhwT7k3j4hNEdEWEW0tLS05wjUzszzyXAXUD8zLTM8FDmcrpF/qqwGUnP5/OX0gaQbJl/93I+KxTJujQ88lfRv47+NbBDMzG488RwA7gAWSrpR0HrAS2JqtIOmStAzgc8CzEXEqTQZdwP6I+EZJm9mZyZuA58e7EGZmNnYVjwAi4rSktcBTQBOwOSL2SrotLe8EWoGHJJ0B9gHtafOPAn8E7Em7hwDuiIgngLskLSbpTnoF+Hy1FsrMzCpTI12P29bWFj09PRN6jXr48VI9xFAN02U5qsHrwsqpl+1C0nMR0VY63/cCMjMrKCcAM7OCcgIwMysoJwAzs4JyAjAzKygnADOzgnICMDMrKCcAM7OCcgIwMysoJwAzs4JyAjAzKygnADOzgnICMDMrKCcAM7OCcgIwMysoJwAzs4JyAjAzKygnADOzgqo4JvB0lIxVXzvNzc01fX8zMyhgApjo+Jz1MsanmdlE5eoCkrRc0gFJfZLWlylvlrRF0m5JP5a0qFJbSTMlPS3pxfSvd4vNzKZQxQQgqQm4H7geWAiskrSwpNodQG9E/BbwWeBbOdquB56JiAXAM+m0mZlNkTxHAEuAvog4GBFvAw8DK0rqLCT5EiciXgDmS5pVoe0K4MH0+YPAjRNZEDMzG5s8CWAOcCgz3Z/Oy9oF3AwgaQlwBTC3QttZEXEEIP17Wbk3l7RGUo+knoGBgRzhmplZHnkSQLlLZkrPgm4EmiX1AuuAnwCnc7YdVURsioi2iGhraWkZS1Mzm0Ld3d0sWrSIpqYmFi1aRHd3d61DsgryXAXUD8zLTM8FDmcrRMQpYDWAkmssX04f7xul7VFJsyPiiKTZwLFxLYGZ1Vx3dzcdHR10dXVx7bXXsn37dtrb2wFYtWpVjaOzkeQ5AtgBLJB0paTzgJXA1mwFSZekZQCfA55Nk8JobbcCt6TPbwEen9iimFmtbNiwga6uLpYuXcqMGTNYunQpXV1dbNiwodah2SiU55p2STcA3wSagM0RsUHSbQAR0Snpd4CHgDPAPqA9Ik6O1Dad/7eBR4DLgdeAT0fEidHiaGtri56ennEsZvX4dwDDvC6GFX1dNDU18dZbbzFjxox35g0ODnL++edz5syZGkZWW/WyXUh6LiLaSufn+iFYRDwBPFEyrzPz/P8CC/K2Tef/DFiW5/3NrL61trayfft2li5d+s687du309raWsOorBLfC8jMJqyjo4P29na2bdvG4OAg27Zto729nY6OjlqHZqMo3K0gzKz6hk70rlu3jv3799Pa2sqGDRt8ArjO5ToHUC98DqC+eF0M87qwcupluxjpHIC7gMzMCsoJwMysoJwAzMwKygnAzKygnADMzArKCcDMrKCcAMzMCsoJwMysoJwAzMwKygnAzKygnADMzArKCcDMqsJDQjYe3w3UzCbMQ0I2Jh8BmNmEeUjIxuTbQY9RvdzetR54XQwr+rrwkJDl1ct24dtBm9mkGRoSMstDQtY/JwAzmzAPCdmYcp0ElrQc+BbQBPyXiNhYUn4x8N+Ay9PXvDsi/lLSVcD3M1U/CPzbiPimpDuBPwYG0rI70gHkzazBeEjIxlTxHICkJuCnwHVAP7ADWBUR+zJ17gAujoivSGoBDgB/JyLeLnmd14FrIuLVNAG8ERF35w3W5wDqi9fFMK8LK6detouJnANYAvRFxMH0C/1hYEVJnQAukiTgQuAEcLqkzjLgpYh4dczRm5lZ1eVJAHOAQ5np/nRe1n1AK3AY2APcHhFnS+qsBEp/GbJW0m5JmyU1l3tzSWsk9UjqGRgYKFfFbNJJGvWRp85QPbN6kScBlNtqS49pPgH0Ah8AFgP3SXr/Oy8gnQd8EvhBps0DwIfS+keAe8q9eURsioi2iGhraWnJEa5Z9UVEVR5m9SRPAugH5mWm55Ls6WetBh6LRB/wMnB1pvx6YGdEHB2aERFHI+JMeqTwbZKuJjMzmyJ5EsAOYIGkK9M9+ZXA1pI6r5H08SNpFnAVcDBTvoqS7h9JszOTNwHPjy10MzObiIqXgUbEaUlrgadILgPdHBF7Jd2WlncCXwe+I2kPSZfRVyLiOICk95FcQfT5kpe+S9Jiku6kV8qUm5nZJPKtIMaoXi7rqgdeF2ajq5fPiG8FYWZm7+IEYGZWUE4AZmYF5QRgZlZQHhHMzGyc8vy6O0+dWp0odgIwMxunerjCZyLcBWRmVlBOAGZmBeUEYGZWUE4AZmYF5QRgZlZQTgBmZgXlBGBmVdHd3c2iRYtoampi0aJFdHeXDgBo9ca/AzCzCevu7qajo4Ouri6uvfZatm/fTnt7OwCrVq2qcXQ2Eh8BmNmEbdiwga6uLpYuXcqMGTNYunQpXV1dbNiwodah2Sg8HsAY1cv9veuB14UNaWpq4q233mLGjBnvzBscHOT888/nzJkzNYzMwOMBmNkkam1tZfv27e+at337dlpbW2sUkeXhBGBmE9bR0UF7ezvbtm1jcHCQbdu20d7eTkdHR61Ds1H4JLCZTdjQid5169axf/9+Wltb2bBhg08A1zmfAxgj93sP87owawwTOgcgabmkA5L6JK0vU36xpB9K2iVpr6TVmbJXJO2R1CupJzN/pqSnJb2Y/m0e78KZmdnYVUwAkpqA+4HrgYXAKkkLS6p9AdgXEX8f+Dhwj6TzMuVLI2JxSQZaDzwTEQuAZ9JpMzObInmOAJYAfRFxMCLeBh4GVpTUCeAiJUPfXAicAE5XeN0VwIPp8weBG/MGbWZmE5cnAcwBDmWm+9N5WfcBrcBhYA9we0ScTcsC+JGk5yStybSZFRFHANK/l5V7c0lrJPVI6hkYGMgRrpmZ5ZEnAZQb0LL0zN8ngF7gA8Bi4D5J70/LPhoRHybpQvqCpN8bS4ARsSki2iKiraWlZSxNzcxsFHkSQD8wLzM9l2RPP2s18Fgk+oCXgasBIuJw+vcYsIWkSwngqKTZAOnfY+NdiGqSNOojT508g0CbmdVangSwA1gg6cr0xO5KYGtJndeAZQCSZgFXAQclXSDponT+BcDvA8+nbbYCt6TPbwEen8iCVEtEVOVhZlbvKv4QLCJOS1oLPAU0AZsjYq+k29LyTuDrwHck7SHpMvpKRByX9EFgS7pH/B7gexHxZPrSG4FHJLWTJJBPV3nZzMxsFP4hmI2bfwhm1hh8MzgzsynSKIPj+F5AZmZV1EiD4/gIwMysihppcByfA7Bx8zkAs3PV4+A4PgdgZjYFGmlwHCcAM7MqaqTBcXwS2MysihppcByfA7Bx8zkAs8bgcwBmZvYuTgBmZgXlBGBmVlBOAGZmBeUEYGZWUE4AZmYF5QRgZlZQTgBmZgXlBGBmVlBOAGZmBeV7AdmI0rGcJ1zHt4swq09OADYif3GbTW+5uoAkLZd0QFKfpPVlyi+W9ENJuyTtlbQ6nT9P0jZJ+9P5t2fa3CnpdUm96eOG6i2WmZlVUvEIQFITcD9wHdAP7JC0NSL2Zap9AdgXEX8gqQU4IOm7wGngyxGxU9JFwHOSns60vTci7q7qEpmZWS55jgCWAH0RcTAi3gYeBlaU1AngIiUdwhcCJ4DTEXEkInYCRMQvgf3AnKpFb2Zm45YnAcwBDmWm+zn3S/w+oBU4DOwBbo+Is9kKkuYDvw38TWb2Wkm7JW2W1FzuzSWtkdQjqWdgYCBHuGZmlkeeBFDuMo/Ss4OfAHqBDwCLgfskvf+dF5AuBP4K+GJEnEpnPwB8KK1/BLin3JtHxKaIaIuItpaWlhzhmplZHnkSQD8wLzM9l2RPP2s18Fgk+oCXgasBJM0g+fL/bkQ8NtQgIo5GxJn0SOHbJF1NZmY2RfIkgB3AAklXSjoPWAlsLanzGrAMQNIs4CrgYHpOoAvYHxHfyDaQNDszeRPw/PgWwczMxqPiVUARcVrSWuApoAnYHBF7Jd2WlncCXwe+I2kPSZfRVyLiuKRrgT8C9kjqTV/yjoh4ArhL0mKS7qRXgM9XdcnMzGxUDTUovKQB4NUah3EpcLzGMdQLr4thXhfDvC6G1cu6uCIizjmJ2lAJoB5I6omItlrHUQ+8LoZ5XQzzuhhW7+vCN4MzMysoJwAzs4JyAhi7TbUOoI54XQzzuhjmdTGsrteFzwGYmRWUjwDMzArKCcDMrKCcADIkdaTjFuxOxyi4RtJ7JP17SS9mxi7oyLQ5k87bm46H8K8lNfx6lfRG5vkN6fJfno7j8Kaky0aoG5LuyUz/G0l3Tlngk2yk/7ekT2S2jzfS8TN6JT1U65irqcxn5H9I+g8ldRZL2p8+f0XS/y4p75U0rX75P9p2XzL2yQuSHqiX74i6CKIeSPod4J8CH46I3wL+McldUP8dyU3ufjMiFgO/C8zINP1VRCyOiL9HMmbCDcBXpzL2ySRpGfDnwPKIeC2dfRz48ghNfg3cLOnSqYivBsr+vyPiqXT+YqAH+Ew6/dlaBltNI3xGNgJ/WFJ1JfC9zPRFkualr9E6FbHWQKXt/t5021gI/CbwsakKbDROAMNmA8cj4tcAEXEc+Dnwx8C6iHgrnf/LiLiz3AtExDFgDcltrisPllvnJP0uyY36/klEvJQp2gz8oaSZZZqdJrny4UtTEGJNTbf/dw7nfEYi4n8BP5d0TabePycZN2TIIwwniVVA91QEO8XybvfnAecDJyc9ohycAIb9CJgn6aeS/kLSx4C/C7yWDmaTS0QcJFmvl1WqW+feCzwO3BgRL5SUvUGSBG4/p1XifuAzki6exPjqwjT6f+dR7jMCyRf6SgBJHwF+FhEvZto9CtycPv8D4IdTFfAUG227/1J6P7QjwE8joncqAxuJE0AqIt4A/gHJHt0A8H3g49k6klan/XiHhg5pRzAd9gYHgf8DtI9Q/mfALdlxH4akYz48BPyryQuvrkyH/3dF5T4jkm4l2dv/Z2m/9krO3cM/AZyUtJJkVMA3pyzoKVRhux/qAroMuCBdFzXnBJCRjk/wPyPiq8Bakr2Vy5WMZ0xE/GX6T/wFyZ1RzyHpg8AZ4NjURD1pzpIcyv9DSXeUFkbEz0n6ef/lCO2/SZI8Lpik+OrCNPp/51LmM/KpiDhEckffjwGfIunyKfV9kj3k6dj9k/VNRtnuI2IQeBL4vSmMaUROAClJV0lakJm1GDhAMp7BfZLOT+s1kfTjlXuNFqATuC+mwS/sIuJNkpN+n5FU7kjgGyS38T7ntuIRcYLki2CkI4iGN93+35WM8BkZujtvN3Av8FJE9JdpvgW4i+S28tNWpe0+PVf0j4CXypVPtYrjARTIhcCfS7qE5IROH8mh7i9Ixjt4XtIvgV8BDzI8KtpvpH17M9J2/5Xki3FaiIgTkpYDz0o6XlJ2XNIWRj7xdQ/JXuJ0Mq3/3xWM9BkB+AHwLWBduYbpebT/CFCA8+XltvsvSfoXJNvNbuAvpjyqMnwrCDOzgnIXkJlZQTkBmJkVlBOAmVlBOQGYmRWUE4CZWUE5AZiZFZQTgJlZQf1/Jc+ZUJSel/0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.model_selection import RepeatedKFold\n",
    "# evaluate a given model using cross-validation\n",
    "def evaluate_model(model, X, y):\n",
    "    cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "    scores = cross_val_score(model, X, y, scoring='f1', cv=cv, error_score='raise')\n",
    "    return scores\n",
    "# get the models to evaluate\n",
    "models = get_models()\n",
    "# evaluate the models and store results\n",
    "results, names = list(), list()\n",
    "for name, model in models.items():\n",
    "    scores = evaluate_model(model, X, y)\n",
    "    results.append(scores)\n",
    "    names.append(name)\n",
    "    print('>%s %.3f (%.3f)' % (name, mean(scores), std(scores)))\n",
    "# plot model performance for comparison\n",
    "pyplot.boxplot(results, labels=names, showmeans=True)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "d5cdfce8",
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "X has 467 features, but StandardScaler is expecting 20 features as input.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-26-7b7aa7522aaa>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     15\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     16\u001b[0m \u001b[1;31m# apply the whole pipeline to data\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 17\u001b[1;33m \u001b[0mpred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mSeries\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpipe\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpr\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mpred_cols\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     18\u001b[0m \u001b[0mprint\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mpred\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\sklearn\\utils\\metaestimators.py\u001b[0m in \u001b[0;36m<lambda>\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    118\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    119\u001b[0m         \u001b[1;31m# lambda, but not partial, allows help() to work with update_wrapper\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 120\u001b[1;33m         \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mlambda\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    121\u001b[0m         \u001b[1;31m# update the docstring of the returned function\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    122\u001b[0m         \u001b[0mupdate_wrapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mout\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\sklearn\\pipeline.py\u001b[0m in \u001b[0;36mpredict\u001b[1;34m(self, X, **predict_params)\u001b[0m\n\u001b[0;32m    416\u001b[0m         \u001b[0mXt\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    417\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0m_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtransform\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_iter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mwith_final\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 418\u001b[1;33m             \u001b[0mXt\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtransform\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mXt\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    419\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msteps\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mXt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mpredict_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    420\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\sklearn\\preprocessing\\_data.py\u001b[0m in \u001b[0;36mtransform\u001b[1;34m(self, X, copy)\u001b[0m\n\u001b[0;32m    881\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    882\u001b[0m         \u001b[0mcopy\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcopy\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mcopy\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 883\u001b[1;33m         X = self._validate_data(X, reset=False,\n\u001b[0m\u001b[0;32m    884\u001b[0m                                 \u001b[0maccept_sparse\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'csr'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    885\u001b[0m                                 \u001b[0mestimator\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mFLOAT_DTYPES\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\sklearn\\base.py\u001b[0m in \u001b[0;36m_validate_data\u001b[1;34m(self, X, y, reset, validate_separately, **check_params)\u001b[0m\n\u001b[0;32m    435\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    436\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcheck_params\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'ensure_2d'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 437\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_check_n_features\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreset\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mreset\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    438\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    439\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\sklearn\\base.py\u001b[0m in \u001b[0;36m_check_n_features\u001b[1;34m(self, X, reset)\u001b[0m\n\u001b[0;32m    363\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    364\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mn_features\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mn_features_in_\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 365\u001b[1;33m             raise ValueError(\n\u001b[0m\u001b[0;32m    366\u001b[0m                 \u001b[1;34mf\"X has {n_features} features, but {self.__class__.__name__} \"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    367\u001b[0m                 f\"is expecting {self.n_features_in_} features as input.\")\n",
      "\u001b[1;31mValueError\u001b[0m: X has 467 features, but StandardScaler is expecting 20 features as input."
     ]
    }
   ],
   "source": [
    "# prepare the pipeline\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import joblib\n",
    "pipe = make_pipeline(StandardScaler(), GaussianNB())\n",
    "pipe.fit(X, y)\n",
    "joblib.dump(pipe, 'model.pkl')\n",
    "#Prediction\n",
    "#Loading the saved model with joblib\n",
    "pipe = joblib.load('model.pkl')\n",
    "\n",
    "# New data to predict\n",
    "pr = pd.read_excel(r'ligands2screen.xlsx')\n",
    "pred_cols = list(pr.columns.values)[:-1]\n",
    "\n",
    "# apply the whole pipeline to data\n",
    "pred = pd.Series(pipe.predict(pr[pred_cols]))\n",
    "print (pred)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d236ad9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}