1 classification_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 44,
   "source": [
    "# Import library\r\n",
    "\r\n",
    "import pandas as pd \r\n",
    "import numpy as np\r\n",
    "\r\n",
    "from xgboost import XGBClassifier\r\n",
    "from sklearn.linear_model import LogisticRegression\r\n",
    "from sklearn.ensemble import RandomForestClassifier\r\n",
    "from sklearn.model_selection import train_test_split, cross_val_score\r\n",
    "from sklearn.metrics import confusion_matrix, classification_report\r\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\r\n",
    "from sklearn.model_selection import GridSearchCV\r\n",
    "from sklearn.preprocessing import   MinMaxScaler"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "source": [
    "df = pd.read_csv('processed_data.csv')\r\n",
    "df.head()"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "   duration     goal_usd  status  usd_pledged  US_based  comics  crafts  \\\n",
       "0      16.0   2000.00000       1   6061.00000         1     0.0     0.0   \n",
       "1      30.0   3870.99771       1   3914.50512         0     1.0     0.0   \n",
       "2      30.0   1100.00000       1   1110.00000         1     0.0     0.0   \n",
       "3      45.0   3500.00000       1   4807.00000         1     0.0     0.0   \n",
       "4      60.0  30000.00000       1  40368.00000         1     0.0     0.0   \n",
       "\n",
       "   dance  design  fashion  ...    3    4    5    6    7    8    9   10   11  \\\n",
       "0    0.0     0.0      0.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0  1.0  0.0   \n",
       "1    0.0     0.0      0.0  ...  0.0  0.0  0.0  0.0  0.0  1.0  0.0  0.0  0.0   \n",
       "2    0.0     0.0      1.0  ...  0.0  0.0  0.0  1.0  0.0  0.0  0.0  0.0  0.0   \n",
       "3    0.0     0.0      0.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  1.0  0.0  0.0   \n",
       "4    0.0     0.0      0.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  1.0   \n",
       "\n",
       "    12  \n",
       "0  0.0  \n",
       "1  0.0  \n",
       "2  0.0  \n",
       "3  0.0  \n",
       "4  0.0  \n",
       "\n",
       "[5 rows x 30 columns]"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>goal_usd</th>\n",
       "      <th>status</th>\n",
       "      <th>usd_pledged</th>\n",
       "      <th>US_based</th>\n",
       "      <th>comics</th>\n",
       "      <th>crafts</th>\n",
       "      <th>dance</th>\n",
       "      <th>design</th>\n",
       "      <th>fashion</th>\n",
       "      <th>...</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>16.0</td>\n",
       "      <td>2000.00000</td>\n",
       "      <td>1</td>\n",
       "      <td>6061.00000</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>30.0</td>\n",
       "      <td>3870.99771</td>\n",
       "      <td>1</td>\n",
       "      <td>3914.50512</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>30.0</td>\n",
       "      <td>1100.00000</td>\n",
       "      <td>1</td>\n",
       "      <td>1110.00000</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>45.0</td>\n",
       "      <td>3500.00000</td>\n",
       "      <td>1</td>\n",
       "      <td>4807.00000</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>60.0</td>\n",
       "      <td>30000.00000</td>\n",
       "      <td>1</td>\n",
       "      <td>40368.00000</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ]
     },
     "metadata": {},
     "execution_count": 32
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "source": [
    "# drop 'usd_pledged' column \r\n",
    "df.drop ('usd_pledged', axis=1, inplace = True)\r\n",
    "\r\n",
    "# train-test-split\r\n",
    "y = df['status']\r\n",
    "X = df.drop ('status', axis=1)\r\n",
    "x_train, x_test, y_train, y_test = train_test_split(X,y,train_size = 0.8, random_state = 42) #shuffle = False that means no random "
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "source": [
    "# Min-max normalization scaling for independent variable as model like logistic regression \r\n",
    "# use a weighted sum of input variables will be affected\r\n",
    "\r\n",
    "class_scl = MinMaxScaler()\r\n",
    "x_train = class_scl.fit_transform(x_train)\r\n",
    "x_test = class_scl.transform(x_test)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "source": [
    "## Define function for performance result\r\n",
    "\r\n",
    "# Function to print KFold Cross validation performance on train set \r\n",
    "def KFoldresult_5fold(model, x_train, y_train, is_logreg):\r\n",
    "    accuracy = cross_val_score (model, x_train,y_train, cv=5)\r\n",
    "    print (model)\r\n",
    "    print (f'KFolds cross validation: \\n {accuracy} \\n')\r\n",
    "    print (f'Mean accuracy: \\n {accuracy.mean()}\\n')\r\n",
    "    print ('Coefficient of feature: \\n' )\r\n",
    "    if is_logreg:\r\n",
    "        for index, co in enumerate(model.coef_[0]):\r\n",
    "            print (f'Feature {index}: {co:.5f}')\r\n",
    "    else:\r\n",
    "        for index, co in enumerate(model.feature_importances_):\r\n",
    "            print (f'Feature {index}: {co:.5f}')\r\n",
    "    return accuracy\r\n",
    "\r\n",
    "# Function to return prediction and print prediction result on test set \r\n",
    "def predictionresult(model, x_test, y_test):\r\n",
    "    y_pred = model.predict(x_test)\r\n",
    "    print (f'Confusion_matrix: \\n {confusion_matrix(y_test, y_pred)} \\n')\r\n",
    "    print (f'Classification report: \\n {classification_report(y_test,y_pred)} \\n')\r\n",
    "    return y_pred\r\n",
    "\r\n",
    "# Function to print out Grid Search parameters: \r\n",
    "def gridsearch(model, parameters, X_train, y_train):\r\n",
    "    search = GridSearchCV(model, parameters, n_jobs=-1)\r\n",
    "    search.fit(x_train,y_train)\r\n",
    "    print(f'Parameter tested: {parameters}')\r\n",
    "    print(f'Best Score : {search.best_score_}')\r\n",
    "    print(f'Best parameters: {search.best_params_}')\r\n",
    "    return search\r\n",
    "\r\n",
    "def performace(y_ture, y_pred):\r\n",
    "    return [accuracy_score(y_ture, y_pred), recall_score(y_ture, y_pred), precision_score(y_ture, y_pred), f1_score(y_ture, y_pred)]"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Logistic Regression "
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "source": [
    "# Create model -log\r\n",
    "log = LogisticRegression()\r\n",
    "log.fit (x_train,y_train)\r\n",
    "\r\n",
    "KFoldresult_5fold(log, x_train, y_train, True)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "LogisticRegression()\n",
      "KFolds cross validation: \n",
      " [0.64930263 0.65278032 0.65709997 0.65585533 0.65284083] \n",
      "\n",
      "Mean accuracy: \n",
      " 0.6535758160912685\n",
      "\n",
      "Coefficient of feature: \n",
      "\n",
      "Feature 0: -2.35682\n",
      "Feature 1: -7.41670\n",
      "Feature 2: 0.12108\n",
      "Feature 3: 1.12745\n",
      "Feature 4: -0.58436\n",
      "Feature 5: 1.13096\n",
      "Feature 6: 0.28714\n",
      "Feature 7: -0.03949\n",
      "Feature 8: 0.11382\n",
      "Feature 9: -1.09049\n",
      "Feature 10: 0.13950\n",
      "Feature 11: -1.07439\n",
      "Feature 12: 0.48318\n",
      "Feature 13: -0.53232\n",
      "Feature 14: 0.58867\n",
      "Feature 15: -0.77669\n",
      "Feature 16: 0.30255\n",
      "Feature 17: 0.03061\n",
      "Feature 18: 0.15187\n",
      "Feature 19: 0.09966\n",
      "Feature 20: 0.04089\n",
      "Feature 21: 0.02668\n",
      "Feature 22: -0.12479\n",
      "Feature 23: -0.08045\n",
      "Feature 24: 0.06201\n",
      "Feature 25: 0.12457\n",
      "Feature 26: 0.09394\n",
      "Feature 27: -0.03656\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "array([0.64930263, 0.65278032, 0.65709997, 0.65585533, 0.65284083])"
      ]
     },
     "metadata": {},
     "execution_count": 36
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "source": [
    "log_y_predict = predictionresult(log, x_test, y_test)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Confusion_matrix: \n",
      " [[ 7045  7857]\n",
      " [ 3897 15347]] \n",
      "\n",
      "Classification report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.64      0.47      0.55     14902\n",
      "           1       0.66      0.80      0.72     19244\n",
      "\n",
      "    accuracy                           0.66     34146\n",
      "   macro avg       0.65      0.64      0.63     34146\n",
      "weighted avg       0.65      0.66      0.65     34146\n",
      " \n",
      "\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "source": [
    "# Optimizing hyperparameters\r\n",
    "param = {'C':np.linspace(0.1,1,10), 'penalty': ['l1', 'l2']} \r\n",
    "gridsearch (log, param, x_train, y_train)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "C:\\Users\\user\\anaconda3\\envs\\FTDS\\lib\\site-packages\\sklearn\\model_selection\\_search.py:918: UserWarning: One or more of the test scores are non-finite: [       nan 0.65342938        nan 0.65339278        nan 0.65342206\n",
      "        nan 0.65345867        nan 0.65351724        nan 0.65350992\n",
      "        nan 0.65352457        nan 0.65355385        nan 0.65356117\n",
      "        nan 0.65357582]\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Parameter tested: {'C': array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1. ]), 'penalty': ['l1', 'l2']}\n",
      "Best Score : 0.6535758160912685\n",
      "Best parameters: {'C': 1.0, 'penalty': 'l2'}\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "GridSearchCV(estimator=LogisticRegression(), n_jobs=-1,\n",
       "             param_grid={'C': array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1. ]),\n",
       "                         'penalty': ['l1', 'l2']})"
      ]
     },
     "metadata": {},
     "execution_count": 38
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Apply the best parameters {'C': 1.0, 'penalty': 'l2'} \r\n",
    " - best parameters is same as default parameters for model 'log'"
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Random Forest Classifier"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "source": [
    "ranforest = RandomForestClassifier(random_state = 42, n_jobs=-1)\r\n",
    "ranforest.fit (x_train,y_train)\r\n",
    "\r\n",
    "KFoldresult_5fold(ranforest, x_train, y_train, False)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "RandomForestClassifier(n_jobs=-1, random_state=42)\n",
      "KFolds cross validation: \n",
      " [0.63810082 0.64059011 0.64187136 0.64454369 0.64072339] \n",
      "\n",
      "Mean accuracy: \n",
      " 0.6411658727048033\n",
      "\n",
      "Coefficient of feature: \n",
      "\n",
      "Feature 0: 0.25277\n",
      "Feature 1: 0.53021\n",
      "Feature 2: 0.01631\n",
      "Feature 3: 0.00955\n",
      "Feature 4: 0.00642\n",
      "Feature 5: 0.00506\n",
      "Feature 6: 0.00517\n",
      "Feature 7: 0.00474\n",
      "Feature 8: 0.00745\n",
      "Feature 9: 0.02483\n",
      "Feature 10: 0.00573\n",
      "Feature 11: 0.00902\n",
      "Feature 12: 0.00889\n",
      "Feature 13: 0.00667\n",
      "Feature 14: 0.00868\n",
      "Feature 15: 0.01842\n",
      "Feature 16: 0.00371\n",
      "Feature 17: 0.00698\n",
      "Feature 18: 0.00647\n",
      "Feature 19: 0.00693\n",
      "Feature 20: 0.00754\n",
      "Feature 21: 0.00739\n",
      "Feature 22: 0.00662\n",
      "Feature 23: 0.00646\n",
      "Feature 24: 0.00723\n",
      "Feature 25: 0.00712\n",
      "Feature 26: 0.00730\n",
      "Feature 27: 0.00634\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "array([0.63810082, 0.64059011, 0.64187136, 0.64454369, 0.64072339])"
      ]
     },
     "metadata": {},
     "execution_count": 39
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "source": [
    "ranforest_y_predict = predictionresult(ranforest, x_test, y_test)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Confusion_matrix: \n",
      " [[ 8264  6638]\n",
      " [ 5480 13764]] \n",
      "\n",
      "Classification report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.60      0.55      0.58     14902\n",
      "           1       0.67      0.72      0.69     19244\n",
      "\n",
      "    accuracy                           0.65     34146\n",
      "   macro avg       0.64      0.63      0.64     34146\n",
      "weighted avg       0.64      0.65      0.64     34146\n",
      " \n",
      "\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "source": [
    "param = {'max_depth': list(range(1,10))}\r\n",
    "gridsearch (ranforest, param, x_train, y_train)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Parameter tested: {'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9]}\n",
      "Best Score : 0.6731535057613895\n",
      "Best parameters: {'max_depth': 9}\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "GridSearchCV(estimator=RandomForestClassifier(n_jobs=-1, random_state=42),\n",
       "             n_jobs=-1, param_grid={'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9]})"
      ]
     },
     "metadata": {},
     "execution_count": 41
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "source": [
    "ranforest = RandomForestClassifier(max_depth=9, random_state = 42, n_jobs=-1)\r\n",
    "ranforest.fit (x_train,y_train)\r\n",
    "\r\n",
    "KFoldresult_5fold(ranforest, x_train, y_train, False)\r\n",
    "ranforest_y_predict = predictionresult(ranforest, x_test, y_test) "
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "RandomForestClassifier(max_depth=9, n_jobs=-1, random_state=42)\n",
      "KFolds cross validation: \n",
      " [0.67126698 0.67196251 0.67467145 0.67628217 0.67158442] \n",
      "\n",
      "Mean accuracy: \n",
      " 0.6731535057613895\n",
      "\n",
      "Coefficient of feature: \n",
      "\n",
      "Feature 0: 0.17911\n",
      "Feature 1: 0.36724\n",
      "Feature 2: 0.01338\n",
      "Feature 3: 0.04094\n",
      "Feature 4: 0.01569\n",
      "Feature 5: 0.01333\n",
      "Feature 6: 0.00413\n",
      "Feature 7: 0.00092\n",
      "Feature 8: 0.00425\n",
      "Feature 9: 0.13688\n",
      "Feature 10: 0.00529\n",
      "Feature 11: 0.03308\n",
      "Feature 12: 0.02948\n",
      "Feature 13: 0.01475\n",
      "Feature 14: 0.03413\n",
      "Feature 15: 0.09151\n",
      "Feature 16: 0.00128\n",
      "Feature 17: 0.00102\n",
      "Feature 18: 0.00144\n",
      "Feature 19: 0.00097\n",
      "Feature 20: 0.00103\n",
      "Feature 21: 0.00078\n",
      "Feature 22: 0.00310\n",
      "Feature 23: 0.00125\n",
      "Feature 24: 0.00093\n",
      "Feature 25: 0.00129\n",
      "Feature 26: 0.00109\n",
      "Feature 27: 0.00172\n",
      "Confusion_matrix: \n",
      " [[ 6944  7958]\n",
      " [ 3192 16052]] \n",
      "\n",
      "Classification report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.69      0.47      0.55     14902\n",
      "           1       0.67      0.83      0.74     19244\n",
      "\n",
      "    accuracy                           0.67     34146\n",
      "   macro avg       0.68      0.65      0.65     34146\n",
      "weighted avg       0.68      0.67      0.66     34146\n",
      " \n",
      "\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## XGBoost "
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "source": [
    "xgmodel = XGBClassifier(use_label_encoder = False, eval_metric='mlogloss',n_jobs = -1 )\r\n",
    "xgmodel.fit (x_train,y_train)\r\n",
    "\r\n",
    "KFoldresult_5fold(xgmodel, x_train, y_train, False)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
      "              colsample_bynode=1, colsample_bytree=1, eval_metric='mlogloss',\n",
      "              gamma=0, gpu_id=-1, importance_type='gain',\n",
      "              interaction_constraints='', learning_rate=0.300000012,\n",
      "              max_delta_step=0, max_depth=6, min_child_weight=1, missing=nan,\n",
      "              monotone_constraints='()', n_estimators=100, n_jobs=-1,\n",
      "              num_parallel_tree=1, random_state=0, reg_alpha=0, reg_lambda=1,\n",
      "              scale_pos_weight=1, subsample=1, tree_method='exact',\n",
      "              use_label_encoder=False, validate_parameters=1, verbosity=None)\n",
      "KFolds cross validation: \n",
      " [0.68572684 0.68429915 0.68810631 0.68953399 0.68520281] \n",
      "\n",
      "Mean accuracy: \n",
      " 0.6865738199861737\n",
      "\n",
      "Coefficient of feature: \n",
      "\n",
      "Feature 0: 0.02391\n",
      "Feature 1: 0.02619\n",
      "Feature 2: 0.01533\n",
      "Feature 3: 0.10807\n",
      "Feature 4: 0.08634\n",
      "Feature 5: 0.07641\n",
      "Feature 6: 0.02733\n",
      "Feature 7: 0.01246\n",
      "Feature 8: 0.01849\n",
      "Feature 9: 0.11794\n",
      "Feature 10: 0.02280\n",
      "Feature 11: 0.12847\n",
      "Feature 12: 0.04813\n",
      "Feature 13: 0.06783\n",
      "Feature 14: 0.05140\n",
      "Feature 15: 0.07917\n",
      "Feature 16: 0.01188\n",
      "Feature 17: 0.00676\n",
      "Feature 18: 0.00753\n",
      "Feature 19: 0.00546\n",
      "Feature 20: 0.00558\n",
      "Feature 21: 0.00517\n",
      "Feature 22: 0.01098\n",
      "Feature 23: 0.00864\n",
      "Feature 24: 0.00642\n",
      "Feature 25: 0.00605\n",
      "Feature 26: 0.00536\n",
      "Feature 27: 0.00990\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "array([0.68572684, 0.68429915, 0.68810631, 0.68953399, 0.68520281])"
      ]
     },
     "metadata": {},
     "execution_count": 45
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "source": [
    "xgmodel_y_predict = predictionresult(xgmodel, x_test, y_test)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Confusion_matrix: \n",
      " [[ 8001  6901]\n",
      " [ 3662 15582]] \n",
      "\n",
      "Classification report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.69      0.54      0.60     14902\n",
      "           1       0.69      0.81      0.75     19244\n",
      "\n",
      "    accuracy                           0.69     34146\n",
      "   macro avg       0.69      0.67      0.67     34146\n",
      "weighted avg       0.69      0.69      0.68     34146\n",
      " \n",
      "\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "C:\\Users\\user\\anaconda3\\envs\\FTDS\\lib\\site-packages\\xgboost\\data.py:112: UserWarning: Use subset (sliced data) of np.ndarray is not recommended because it will generate extra copies and increase memory consumption\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "source": [
    "# optimizing hyperparameter \r\n",
    "param = {'max_depth': list(range(1,10))}\r\n",
    "gridsearch (xgmodel, param, x_train, y_train)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Parameter tested: {'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9]}\n",
      "Best Score : 0.6878477581345602\n",
      "Best parameters: {'max_depth': 4}\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "GridSearchCV(estimator=XGBClassifier(base_score=0.5, booster='gbtree',\n",
       "                                     colsample_bylevel=1, colsample_bynode=1,\n",
       "                                     colsample_bytree=1, eval_metric='mlogloss',\n",
       "                                     gamma=0, gpu_id=-1, importance_type='gain',\n",
       "                                     interaction_constraints='',\n",
       "                                     learning_rate=0.300000012,\n",
       "                                     max_delta_step=0, max_depth=6,\n",
       "                                     min_child_weight=1, missing=nan,\n",
       "                                     monotone_constraints='()',\n",
       "                                     n_estimators=100, n_jobs=-1,\n",
       "                                     num_parallel_tree=1, random_state=0,\n",
       "                                     reg_alpha=0, reg_lambda=1,\n",
       "                                     scale_pos_weight=1, subsample=1,\n",
       "                                     tree_method='exact',\n",
       "                                     use_label_encoder=False,\n",
       "                                     validate_parameters=1, verbosity=None),\n",
       "             n_jobs=-1, param_grid={'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9]})"
      ]
     },
     "metadata": {},
     "execution_count": 48
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "source": [
    "xgmodel = XGBClassifier(use_label_encoder = False, eval_metric='mlogloss',n_jobs = -1, max_depth = 4)\r\n",
    "xgmodel.fit (x_train,y_train)\r\n",
    "\r\n",
    "KFoldresult_5fold(xgmodel, x_train, y_train, False)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
      "              colsample_bynode=1, colsample_bytree=1, eval_metric='mlogloss',\n",
      "              gamma=0, gpu_id=-1, importance_type='gain',\n",
      "              interaction_constraints='', learning_rate=0.300000012,\n",
      "              max_delta_step=0, max_depth=4, min_child_weight=1, missing=nan,\n",
      "              monotone_constraints='()', n_estimators=100, n_jobs=-1,\n",
      "              num_parallel_tree=1, random_state=0, reg_alpha=0, reg_lambda=1,\n",
      "              scale_pos_weight=1, subsample=1, tree_method='exact',\n",
      "              use_label_encoder=False, validate_parameters=1, verbosity=None)\n",
      "KFolds cross validation: \n",
      " [0.68664202 0.68730095 0.68828934 0.69096167 0.68604481] \n",
      "\n",
      "Mean accuracy: \n",
      " 0.6878477581345602\n",
      "\n",
      "Coefficient of feature: \n",
      "\n",
      "Feature 0: 0.03512\n",
      "Feature 1: 0.03663\n",
      "Feature 2: 0.01710\n",
      "Feature 3: 0.08387\n",
      "Feature 4: 0.12428\n",
      "Feature 5: 0.04718\n",
      "Feature 6: 0.02220\n",
      "Feature 7: 0.01111\n",
      "Feature 8: 0.01455\n",
      "Feature 9: 0.14623\n",
      "Feature 10: 0.01443\n",
      "Feature 11: 0.13647\n",
      "Feature 12: 0.03581\n",
      "Feature 13: 0.06240\n",
      "Feature 14: 0.04572\n",
      "Feature 15: 0.09566\n",
      "Feature 16: 0.00699\n",
      "Feature 17: 0.00591\n",
      "Feature 18: 0.00496\n",
      "Feature 19: 0.00523\n",
      "Feature 20: 0.00348\n",
      "Feature 21: 0.00308\n",
      "Feature 22: 0.01348\n",
      "Feature 23: 0.00979\n",
      "Feature 24: 0.00459\n",
      "Feature 25: 0.00404\n",
      "Feature 26: 0.00304\n",
      "Feature 27: 0.00665\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "array([0.68664202, 0.68730095, 0.68828934, 0.69096167, 0.68604481])"
      ]
     },
     "metadata": {},
     "execution_count": 49
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "source": [
    "xgmodel_y_predict = predictionresult(xgmodel, x_test, y_test)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Confusion_matrix: \n",
      " [[ 7811  7091]\n",
      " [ 3515 15729]] \n",
      "\n",
      "Classification report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.69      0.52      0.60     14902\n",
      "           1       0.69      0.82      0.75     19244\n",
      "\n",
      "    accuracy                           0.69     34146\n",
      "   macro avg       0.69      0.67      0.67     34146\n",
      "weighted avg       0.69      0.69      0.68     34146\n",
      " \n",
      "\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "C:\\Users\\user\\anaconda3\\envs\\FTDS\\lib\\site-packages\\xgboost\\data.py:112: UserWarning: Use subset (sliced data) of np.ndarray is not recommended because it will generate extra copies and increase memory consumption\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "source": [
    "log_score = performace(y_test, log_y_predict)\r\n",
    "rf_score = performace(y_test, ranforest_y_predict)\r\n",
    "xg_score = performace(y_test, xgmodel_y_predict)\r\n",
    "\r\n",
    "models_scores_table = pd.DataFrame({'Logistic Regression': log_score, 'Random Forest Classifier': rf_score, 'XGBoost':xg_score},\r\n",
    "                                    index=['Accuracy', 'Recall', 'Precision', 'F1 Score'])\r\n",
    "\r\n",
    "models_scores_table['Best Score'] = models_scores_table.idxmax(axis=1)\r\n",
    "\r\n",
    "models_scores_table"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "           Logistic Regression  Random Forest Classifier   XGBoost  \\\n",
       "Accuracy              0.655772                  0.673461  0.689393   \n",
       "Recall                0.797495                  0.834130  0.817346   \n",
       "Precision             0.661395                  0.668555  0.689264   \n",
       "F1 Score              0.723096                  0.742220  0.747860   \n",
       "\n",
       "                         Best Score  \n",
       "Accuracy                    XGBoost  \n",
       "Recall     Random Forest Classifier  \n",
       "Precision                   XGBoost  \n",
       "F1 Score                    XGBoost  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Logistic Regression</th>\n",
       "      <th>Random Forest Classifier</th>\n",
       "      <th>XGBoost</th>\n",
       "      <th>Best Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Accuracy</th>\n",
       "      <td>0.655772</td>\n",
       "      <td>0.673461</td>\n",
       "      <td>0.689393</td>\n",
       "      <td>XGBoost</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recall</th>\n",
       "      <td>0.797495</td>\n",
       "      <td>0.834130</td>\n",
       "      <td>0.817346</td>\n",
       "      <td>Random Forest Classifier</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Precision</th>\n",
       "      <td>0.661395</td>\n",
       "      <td>0.668555</td>\n",
       "      <td>0.689264</td>\n",
       "      <td>XGBoost</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1 Score</th>\n",
       "      <td>0.723096</td>\n",
       "      <td>0.742220</td>\n",
       "      <td>0.747860</td>\n",
       "      <td>XGBoost</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "metadata": {},
     "execution_count": 51
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [],
   "outputs": [],
   "metadata": {}
  }
 ],
 "metadata": {
  "orig_nbformat": 4,
  "language_info": {
   "name": "python",
   "version": "3.8.8",
   "mimetype": "text/x-python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "pygments_lexer": "ipython3",
   "nbconvert_exporter": "python",
   "file_extension": ".py"
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.8.8 64-bit ('FTDS': conda)"
  },
  "interpreter": {
   "hash": "7e7ae8b1718174e9f443afd7248b60321c4690586460e9e9bc3b215bfdbba77d"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}