onehot.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9983b70a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.inspection import permutation_importance\n",
    "import time \n",
    "\n",
    "from Script import *\n",
    "\n",
    "df_train = pd.read_csv(\"Data/train.csv\")\n",
    "df_test  = pd.read_csv(\"Data/test.csv\")\n",
    "\n",
    "df_train = df_train.drop(\"i\", axis=1)\n",
    "y = df_train.pop(\"y\")\n",
    "df_train.insert(54,\"y\", y)\n",
    "df_test = df_test.drop([\"y\", \"i\"], axis = 1)\n",
    "\n",
    "\n",
    "df_train, df_test = preprocess_data(df_train, df_test, verbose=False)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "col_c = list(df_train.filter(like='c', axis=1).columns)\n",
    "col_o = list(df_train.filter(like='o', axis=1).columns)\n",
    "\n",
    "df_train = df_train.drop(col_c, axis=1)\n",
    "df_test = df_test.drop(col_c, axis=1)\n",
    "\n",
    "\n",
    "\n",
    "X = df_train.iloc[:, 0:-1]\n",
    "y = df_train.iloc[:, -1]\n",
    "\n",
    "\n",
    "# y = y.cat.rename_categories(['no', 'yes'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2099e213",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_onehot= to_one_hot(X, X)\n",
    "X_onehot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f9e64a8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import FunctionTransformer\n",
    "def self_to_one_hot(X):\n",
    "    transformed_X = to_one_hot(X,X)\n",
    "    return transformed_X\n",
    "onehot_pipe  = FunctionTransformer(self_to_one_hot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "302dd061",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier, VotingClassifier\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "import joblib \n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "562af510",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.ensemble import HistGradientBoostingClassifier\n",
    "from sklearn.compose import make_column_transformer, make_column_selector\n",
    "\n",
    "one_hot_encoder = make_column_transformer(\n",
    "    (\n",
    "        OneHotEncoder(sparse=False, handle_unknown=\"ignore\"),\n",
    "        make_column_selector(dtype_include=\"category\"),\n",
    "    ),\n",
    "    remainder=\"passthrough\",\n",
    ")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a247ee3d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6810671099607414 \n",
      " 0.6774672563536124\n",
      "0.6498655860638691 \n",
      " 0.606157613384659\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier, VotingClassifier\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "import joblib \n",
    "# rf_grid_search = joblib.load(\"Temp/rf_grid_search.joblib\")\n",
    "\n",
    "\n",
    "best_params = {'n_estimators': 1600, 'min_samples_split': 2, 'min_samples_leaf': 4, 'max_features': 'sqrt', 'max_depth': 70, 'bootstrap': True}\n",
    "rf_best = RandomForestClassifier()\n",
    "rf_best.set_params(**best_params)\n",
    "\n",
    "\n",
    "hist = HistGradientBoostingClassifier(random_state=0)\n",
    "\n",
    "\n",
    "\n",
    "rf_one_hot = hist_one_hot = make_pipeline(\n",
    "    one_hot_encoder, rf_best\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "hist_one_hot = make_pipeline(\n",
    "    one_hot_encoder, hist\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "rf_best_one_hot_scores = cross_val_score(rf_one_hot, X, y, cv=5)\n",
    "xgb_one_hot_scores = cross_val_score(hist_one_hot, X, y, cv=5)\n",
    "\n",
    "\n",
    "rf_best_scores = cross_val_score(rf_best, X, y, cv=5)\n",
    "xgb_scores = cross_val_score(hist, X, y, cv=5)\n",
    "\n",
    "\n",
    "print(rf_best_one_hot_scores.mean(), \"\\n\", rf_best_scores.mean())\n",
    "print(xgb_one_hot_scores.mean(), \"\\n\", xgb_scores.mean())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c7616da8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting Classifiers\n",
      "Hard Voting \n",
      "aux stop\n",
      "hard voting 0.6815764585634061\n",
      "aux stop\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "\n",
    "print(\"Fitting Classifiers\")\n",
    "eclf1 = VotingClassifier(estimators=[\n",
    "        ('rf_onehot', rf_one_hot), ('xgbhits_onehot', hist_one_hot)], voting='hard')\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "print(\"Hard Voting \")\n",
    "hard_scores = cross_val_score(eclf1, X, y, cv=10)\n",
    "\n",
    "print(\"aux stop\")\n",
    "\n",
    "name_of_model = \"ONEHOTALL_rf_best+xgboostHIST\"\n",
    "\n",
    "joblib.dump(hard_scores, \"Temp/\"+name_of_model+\"_hard_scores\"+\".joblib\")\n",
    "\n",
    "\n",
    "print(\"hard voting\", hard_scores.mean())\n",
    "\n",
    "\n",
    "print(\"aux stop\")\n",
    "\n",
    "# joblib.dump(eclf1, \"Temp/\"+name_of_model+\".joblib\")\n",
    "\n",
    "eclf1.fit(X, y)\n",
    "\n",
    "preds_hard = eclf1.predict(df_test)\n",
    "save_predictions(preds_hard, \"Predictions/\"+name_of_model+\"_hard\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5820fbba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6808929956060996"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.load(\"Temp/rf_best+xgboost_hard_scores.joblib\").mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6500bbcd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6815764585634061"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.load(\"Temp/ONEHOTALL_rf_best+xgboostHIST_hard_scores.joblib\").mean()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "07c785fc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting Classifiers\n",
      "soft Voting \n",
      "aux stop\n",
      "soft voting 0.6558741981719496\n",
      "aux stop\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "print(\"Fitting Classifiers\")\n",
    "eclf2 = VotingClassifier(estimators=[\n",
    "        ('rf_onehot', rf_one_hot), ('xgbhits_onehot', hist_one_hot)], voting='soft')\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "print(\"soft Voting \")\n",
    "soft_scores = cross_val_score(eclf2, X, y, cv=10)\n",
    "\n",
    "print(\"aux stop\")\n",
    "\n",
    "name_of_model = \"ONEHOTALL_rf_best+xgboostHIST\"\n",
    "\n",
    "joblib.dump(soft_scores, \"Temp/\"+name_of_model+\"_soft_scores\"+\".joblib\")\n",
    "\n",
    "\n",
    "print(\"soft voting\", soft_scores.mean())\n",
    "\n",
    "\n",
    "print(\"aux stop\")\n",
    "\n",
    "# joblib.dump(eclf1, \"Temp/\"+name_of_model+\".joblib\")\n",
    "\n",
    "eclf2.fit(X, y)\n",
    "\n",
    "preds_soft = eclf2.predict(df_test)\n",
    "save_predictions(preds_soft, \"Predictions/\"+name_of_model+\"_soft\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:TFgpu]",
   "language": "python",
   "name": "conda-env-TFgpu-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}