tryin_nn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.inspection import permutation_importance\n",
    "import time \n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "\n",
    "from Script import *\n",
    "df_train = pd.read_csv(\"Data/train.csv\")\n",
    "df_test  = pd.read_csv(\"Data/test.csv\")\n",
    "\n",
    "df_train = df_train.drop(\"i\", axis=1)\n",
    "y = df_train.pop(\"y\")\n",
    "df_train.insert(54,\"y\", y)\n",
    "df_test = df_test.drop([\"y\", \"i\"], axis = 1)\n",
    "\n",
    "\n",
    "df_train, df_test = preprocess_data(df_train, df_test, verbose=False)\n",
    "\n",
    "col_c = list(df_train.filter(like='c', axis=1).columns)\n",
    "col_o = list(df_train.filter(like='o', axis=1).columns)\n",
    "\n",
    "df_train = df_train.drop(col_c+col_o, axis=1)\n",
    "df_test = df_test.drop(col_c+col_o, axis=1)\n",
    "\n",
    "X = df_train.iloc[:, 0:-1]\n",
    "y = df_train.iloc[:, -1]\n",
    "\n",
    "Xn = keras.utils.normalize(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder, LabelEncoder\n",
    "from sklearn.model_selection import train_test_split\n",
    "from tensorflow import keras\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Dropout\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(Xn,\n",
    "                                                   y, test_size=0.2,\n",
    "                                                   random_state = 1)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/antonio/miniconda3/envs/TFgpu/lib/python3.9/site-packages/pandas/io/formats/format.py:1429: FutureWarning: Index.ravel returning ndarray is deprecated; in a future version this will return a view on self.\n",
      "  for val, m in zip(values.ravel(), mask.ravel())\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "4427    0.0\n",
       "3419    0.0\n",
       "1576    0.0\n",
       "3483    1.0\n",
       "1077    1.0\n",
       "       ... \n",
       "905     0.0\n",
       "5192    1.0\n",
       "3980    1.0\n",
       "235     1.0\n",
       "5157    1.0\n",
       "Name: y, Length: 4665, dtype: category\n",
       "Categories (2, float64): [0.0, 1.0]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-24 10:02:19.458814: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2021-11-24 10:02:19.459133: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcublas.so.11'; dlerror: libcublas.so.11: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-11.5/lib64\n",
      "2021-11-24 10:02:19.459200: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcublasLt.so.11'; dlerror: libcublasLt.so.11: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-11.5/lib64\n",
      "2021-11-24 10:02:19.459283: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcufft.so.10'; dlerror: libcufft.so.10: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-11.5/lib64\n",
      "2021-11-24 10:02:19.459448: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcurand.so.10'; dlerror: libcurand.so.10: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-11.5/lib64\n",
      "2021-11-24 10:02:19.459504: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcusolver.so.11'; dlerror: libcusolver.so.11: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-11.5/lib64\n",
      "2021-11-24 10:02:19.459558: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcusparse.so.11'; dlerror: libcusparse.so.11: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-11.5/lib64\n",
      "2021-11-24 10:02:19.460878: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1835] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.\n",
      "Skipping registering GPU devices...\n",
      "2021-11-24 10:02:19.461290: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "units_per_layer = [50, 50, 50, 50, 50, 50]\n",
    "input_s = X_train.shape[1]\n",
    "output_s = 1\n",
    "activation_ = 'relu'\n",
    "\n",
    "\n",
    "\n",
    "model = Sequential()\n",
    "depth = len(units_per_layer)\n",
    "model.add(Dense(units_per_layer[0], activation=activation_, input_shape=(input_s,)))\n",
    "for i in range(1, depth):\n",
    "    model.add(Dense(units_per_layer[i], activation=activation_))\n",
    "    model.add(Dropout(rate = 0.1))\n",
    "model.add(Dense(output_s, activation = 'sigmoid'))   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense (Dense)                (None, 50)                2050      \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 50)                2550      \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 50)                2550      \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 50)                2550      \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 50)                2550      \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 50)                2550      \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 1)                 51        \n",
      "=================================================================\n",
      "Total params: 14,851\n",
      "Trainable params: 14,851\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/antonio/miniconda3/envs/TFgpu/lib/python3.9/site-packages/pandas/io/formats/format.py:1429: FutureWarning: Index.ravel returning ndarray is deprecated; in a future version this will return a view on self.\n",
      "  for val, m in zip(values.ravel(), mask.ravel())\n",
      "2021-11-24 10:02:22.993561: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/150\n",
      "8/8 [==============================] - 1s 26ms/step - loss: 0.6806 - accuracy: 0.6334 - val_loss: 0.6624 - val_accuracy: 0.6549\n",
      "Epoch 2/150\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 0.6532 - accuracy: 0.6498 - val_loss: 0.6319 - val_accuracy: 0.6549\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/antonio/miniconda3/envs/TFgpu/lib/python3.9/site-packages/pandas/io/formats/format.py:1429: FutureWarning: Index.ravel returning ndarray is deprecated; in a future version this will return a view on self.\n",
      "  for val, m in zip(values.ravel(), mask.ravel())\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3/150\n",
      "8/8 [==============================] - 0s 6ms/step - loss: 0.6403 - accuracy: 0.6498 - val_loss: 0.6265 - val_accuracy: 0.6549\n",
      "Epoch 4/150\n",
      "8/8 [==============================] - 0s 6ms/step - loss: 0.6381 - accuracy: 0.6525 - val_loss: 0.6267 - val_accuracy: 0.6967\n",
      "Epoch 5/150\n",
      "8/8 [==============================] - 0s 6ms/step - loss: 0.6336 - accuracy: 0.6750 - val_loss: 0.6206 - val_accuracy: 0.7031\n",
      "Epoch 6/150\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 0.6279 - accuracy: 0.6870 - val_loss: 0.6124 - val_accuracy: 0.6999\n",
      "Epoch 7/150\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 0.6231 - accuracy: 0.6886 - val_loss: 0.6040 - val_accuracy: 0.6988\n",
      "Epoch 8/150\n",
      "8/8 [==============================] - 0s 6ms/step - loss: 0.6167 - accuracy: 0.6881 - val_loss: 0.5999 - val_accuracy: 0.6977\n",
      "Epoch 9/150\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 0.6088 - accuracy: 0.6881 - val_loss: 0.5999 - val_accuracy: 0.6988\n",
      "Epoch 10/150\n",
      "8/8 [==============================] - 0s 9ms/step - loss: 0.6082 - accuracy: 0.6884 - val_loss: 0.6005 - val_accuracy: 0.6988\n",
      "Epoch 11/150\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 0.6084 - accuracy: 0.6894 - val_loss: 0.6006 - val_accuracy: 0.6999\n",
      "Epoch 12/150\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 0.6079 - accuracy: 0.6894 - val_loss: 0.5986 - val_accuracy: 0.7010\n",
      "Epoch 13/150\n",
      "8/8 [==============================] - 0s 6ms/step - loss: 0.6056 - accuracy: 0.6884 - val_loss: 0.6028 - val_accuracy: 0.6988\n",
      "Epoch 14/150\n",
      "8/8 [==============================] - 0s 6ms/step - loss: 0.6069 - accuracy: 0.6892 - val_loss: 0.6021 - val_accuracy: 0.6988\n",
      "Epoch 15/150\n",
      "8/8 [==============================] - 0s 6ms/step - loss: 0.6085 - accuracy: 0.6892 - val_loss: 0.6017 - val_accuracy: 0.6988\n",
      "Epoch 16/150\n",
      "8/8 [==============================] - 0s 6ms/step - loss: 0.6053 - accuracy: 0.6900 - val_loss: 0.6050 - val_accuracy: 0.6988\n",
      "Epoch 17/150\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 0.6064 - accuracy: 0.6905 - val_loss: 0.6026 - val_accuracy: 0.6977\n",
      "Epoch 18/150\n",
      "8/8 [==============================] - 0s 6ms/step - loss: 0.6064 - accuracy: 0.6902 - val_loss: 0.6032 - val_accuracy: 0.6988\n",
      "Epoch 19/150\n",
      "8/8 [==============================] - 0s 6ms/step - loss: 0.6056 - accuracy: 0.6902 - val_loss: 0.6009 - val_accuracy: 0.6988\n",
      "Epoch 20/150\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 0.6054 - accuracy: 0.6894 - val_loss: 0.6046 - val_accuracy: 0.6967\n",
      "Epoch 21/150\n",
      "8/8 [==============================] - 0s 6ms/step - loss: 0.6058 - accuracy: 0.6908 - val_loss: 0.6030 - val_accuracy: 0.6967\n",
      "Epoch 22/150\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 0.6030 - accuracy: 0.6905 - val_loss: 0.6024 - val_accuracy: 0.6977\n",
      "Epoch 23/150\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 0.6041 - accuracy: 0.6916 - val_loss: 0.6069 - val_accuracy: 0.6967\n",
      "Epoch 24/150\n",
      "8/8 [==============================] - 0s 6ms/step - loss: 0.6029 - accuracy: 0.6913 - val_loss: 0.6038 - val_accuracy: 0.6999\n",
      "Epoch 25/150\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 0.6018 - accuracy: 0.6921 - val_loss: 0.6058 - val_accuracy: 0.6945\n",
      "Epoch 26/150\n",
      "8/8 [==============================] - 0s 6ms/step - loss: 0.6040 - accuracy: 0.6913 - val_loss: 0.6103 - val_accuracy: 0.6935\n",
      "Epoch 27/150\n",
      "8/8 [==============================] - 0s 7ms/step - loss: 0.6025 - accuracy: 0.6932 - val_loss: 0.6060 - val_accuracy: 0.6967\n",
      "Epoch 28/150\n",
      "8/8 [==============================] - 0s 6ms/step - loss: 0.6025 - accuracy: 0.6927 - val_loss: 0.6021 - val_accuracy: 0.6988\n",
      "Epoch 29/150\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 0.5999 - accuracy: 0.6916 - val_loss: 0.6110 - val_accuracy: 0.6924\n",
      "Epoch 30/150\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 0.6019 - accuracy: 0.6919 - val_loss: 0.6081 - val_accuracy: 0.6956\n",
      "Epoch 31/150\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 0.6005 - accuracy: 0.6932 - val_loss: 0.6040 - val_accuracy: 0.6999\n",
      "Epoch 32/150\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 0.5998 - accuracy: 0.6929 - val_loss: 0.6129 - val_accuracy: 0.6945\n",
      "Epoch 33/150\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 0.6035 - accuracy: 0.6921 - val_loss: 0.6045 - val_accuracy: 0.7010\n",
      "Epoch 34/150\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 0.5985 - accuracy: 0.6937 - val_loss: 0.6081 - val_accuracy: 0.6967\n",
      "Epoch 35/150\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 0.6002 - accuracy: 0.6940 - val_loss: 0.6107 - val_accuracy: 0.6967\n",
      "Epoch 36/150\n",
      "8/8 [==============================] - 0s 4ms/step - loss: 0.6005 - accuracy: 0.6940 - val_loss: 0.6133 - val_accuracy: 0.6967\n",
      "Epoch 37/150\n",
      "8/8 [==============================] - 0s 4ms/step - loss: 0.5978 - accuracy: 0.6951 - val_loss: 0.6096 - val_accuracy: 0.6999\n",
      "Epoch 38/150\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 0.5985 - accuracy: 0.6953 - val_loss: 0.6159 - val_accuracy: 0.6977\n",
      "Epoch 39/150\n",
      "8/8 [==============================] - 0s 4ms/step - loss: 0.5993 - accuracy: 0.6940 - val_loss: 0.6166 - val_accuracy: 0.6967\n",
      "Epoch 40/150\n",
      "8/8 [==============================] - 0s 4ms/step - loss: 0.5987 - accuracy: 0.6953 - val_loss: 0.6180 - val_accuracy: 0.6956\n",
      "Epoch 41/150\n",
      "8/8 [==============================] - 0s 4ms/step - loss: 0.5969 - accuracy: 0.6943 - val_loss: 0.6181 - val_accuracy: 0.6967\n",
      "Epoch 42/150\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 0.5980 - accuracy: 0.6961 - val_loss: 0.6175 - val_accuracy: 0.6967\n",
      "Epoch 43/150\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 0.5960 - accuracy: 0.6959 - val_loss: 0.6273 - val_accuracy: 0.6956\n",
      "Epoch 44/150\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 0.5955 - accuracy: 0.6972 - val_loss: 0.6194 - val_accuracy: 0.6967\n",
      "Epoch 45/150\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 0.5969 - accuracy: 0.6953 - val_loss: 0.6273 - val_accuracy: 0.6967\n",
      "Epoch 46/150\n",
      "8/8 [==============================] - 0s 4ms/step - loss: 0.5955 - accuracy: 0.6961 - val_loss: 0.6317 - val_accuracy: 0.6945\n",
      "Epoch 47/150\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 0.5959 - accuracy: 0.6953 - val_loss: 0.6249 - val_accuracy: 0.6956\n",
      "Epoch 48/150\n",
      "8/8 [==============================] - 0s 4ms/step - loss: 0.5951 - accuracy: 0.6972 - val_loss: 0.6268 - val_accuracy: 0.6967\n",
      "Epoch 49/150\n",
      "8/8 [==============================] - 0s 4ms/step - loss: 0.5954 - accuracy: 0.6969 - val_loss: 0.6456 - val_accuracy: 0.6924\n",
      "Epoch 50/150\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 0.5960 - accuracy: 0.6964 - val_loss: 0.6203 - val_accuracy: 0.6967\n",
      "Epoch 51/150\n",
      "8/8 [==============================] - 0s 4ms/step - loss: 0.5944 - accuracy: 0.6961 - val_loss: 0.6319 - val_accuracy: 0.6956\n",
      "Epoch 52/150\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 0.5948 - accuracy: 0.6956 - val_loss: 0.6429 - val_accuracy: 0.6956\n",
      "Epoch 53/150\n",
      "8/8 [==============================] - 0s 4ms/step - loss: 0.5937 - accuracy: 0.6969 - val_loss: 0.6267 - val_accuracy: 0.6967\n",
      "Epoch 54/150\n",
      "8/8 [==============================] - 0s 4ms/step - loss: 0.5936 - accuracy: 0.6980 - val_loss: 0.6480 - val_accuracy: 0.6999\n",
      "Epoch 55/150\n",
      "8/8 [==============================] - 0s 4ms/step - loss: 0.5939 - accuracy: 0.6967 - val_loss: 0.6349 - val_accuracy: 0.6988\n",
      "Epoch 00055: early stopping\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fcce81a9f70>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch = 512\n",
    "verbose = True\n",
    "epochs = 150\n",
    "checkpoint_filepath = \"Temp/\"\n",
    "\n",
    "model.compile(loss='binary_crossentropy',\n",
    "             optimizer='adam',\n",
    "             metrics=['accuracy'])\n",
    "early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=50, verbose=True)\n",
    "\n",
    "model_checkpoint = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_filepath,\n",
    "                                                save_weights_only=True,\n",
    "                                                monitor='val_loss',\n",
    "                                                mode='min',\n",
    "                                                save_best_only=True,\n",
    "                                                verbose=False)\n",
    "\n",
    "model.fit(X_train, y_train, epochs=epochs, batch_size=batch, verbose=verbose, validation_split=0.2, callbacks = [early_stopping, model_checkpoint])\n",
    "\n",
    "#predictions = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.6529563])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(np.round(model.predict(X_test)) == np.array(y_test).reshape(-1, 1))/len(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1167,)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.round(model.predict(X_test)).shape\n",
    "np.array(y_test).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_nn = np.round(model.predict(keras.utils.normalize(df_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Script.save_predictions(pred_test, \"try_keras\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def maj_vote(arr):\n",
    "    counts = np.bincount(arr)\n",
    "    re = np.argmax(counts)\n",
    "    return  re\n",
    "\n",
    "def flatten(l):\n",
    "    for i in l:\n",
    "        if isinstance(i,list):\n",
    "            yield from flatten(i)\n",
    "        else:\n",
    "            yield i\n",
    "\n",
    "def fix_shape(preds):\n",
    "    pred_list_aux = []\n",
    "    for  l in preds:\n",
    "        pred_list_aux.append(np.array([int(x) for x in flatten(l)]).reshape(1, -1))\n",
    "    arrpred = np.array(pred_list_aux).transpose()\n",
    "    return arrpred\n",
    "\n",
    "def vote_given_preds(preds):\n",
    "    arrpred = fix_shape(preds)\n",
    "    return np.apply_along_axis(maj_vote, axis=1, arr=arrpred)  \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 1, 0],\n",
       "       [0, 0, 0],\n",
       "       [1, 1, 1],\n",
       "       [0, 1, 0]])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = [[1], [0], [1], [0]]\n",
    "b = [1, 0, 1, 1]\n",
    "c = [0, 0, 1, 0]\n",
    "preds = [a, b, c]\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "vote_given_preds(preds)\n",
    "\n",
    "# vote_given_preds(preds)\n",
    "\n",
    "# for i, col in enumerate(preds):\n",
    "#     print(col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.inspection import permutation_importance\n",
    "import time \n",
    "\n",
    "from Script import *\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.ensemble import RandomForestClassifier, VotingClassifier\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "\n",
    "\n",
    "import joblib \n",
    "# rf_grid_search = joblib.load(\"Temp/rf_grid_search.joblib\")\n",
    "\n",
    "# clf1 = rf_grid_search.best_estimator_\n",
    "\n",
    "clf1 = joblib.load(\"Temp/rf_best_light.joblib\")\n",
    "\n",
    "\n",
    "\n",
    "clf2 = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0,\n",
    "    max_depth=1, random_state=0)\n",
    "\n",
    "# print(\"Fitting Classifiers\")\n",
    "# eclf1 = VotingClassifier(estimators=[\n",
    "#         ('rf', clf1), ('xgb', clf2)], voting='hard')\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# print(\"Hard Voting \")\n",
    "# hard_scores = cross_val_score(eclf1, X, y, cv=10)\n",
    "\n",
    "# print(\"aux stop\")\n",
    "\n",
    "\n",
    "# import joblib\n",
    "# name_of_model = \"rf_best+xgboost_correct_types\"\n",
    "\n",
    "# joblib.dump(hard_scores, \"Temp/\"+name_of_model+\"_hard_scores\"+\".joblib\")\n",
    "\n",
    "\n",
    "# print(\"hard voting\", hard_scores.mean())\n",
    "\n",
    "\n",
    "# print(\"aux stop\")\n",
    "\n",
    "# # joblib.dump(eclf1, \"Temp/\"+name_of_model+\".joblib\")\n",
    "\n",
    "# eclf1.fit(X, y)\n",
    "\n",
    "# preds_hard = eclf1.predict(df_test)\n",
    "# save_predictions(preds_hard, \"Predictions/\"+name_of_model+\"_hard\")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf1.fit(X, y)\n",
    "clf2.fit(X, y)\n",
    "pred_rf = clf1.predict(df_test)\n",
    "pred_xg = clf2.predict(df_test)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_list = [pred_rf, pred_xg, pred_nn]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([1., 0., 0., ..., 1., 0., 1.]),\n",
       " array([1., 0., 0., ..., 1., 0., 0.]),\n",
       " array([[1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        ...,\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.]], dtype=float32)]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "vote_given_preds(pred_list)\n",
    "vote_preds = np.apply_along_axis(maj_vote, axis=1, arr=vote_given_preds(pred_list))  \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_predictions(vote_preds, \"Temp/bestrf+bestxgb+keras\")"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "8e11f771aa81ca7c54d36d3a0f5c5121193ab3df27c10335e034f223aac86bc0"
  },
  "kernelspec": {
   "display_name": "Python 3.9.7 64-bit ('TFgpu': conda)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}