AG_Predicting_price.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "private_outputs": true,
      "provenance": [],
      "mount_file_id": "1AnDJakZJHStw0ne-xUBwFcDwXtg7LNHV",
      "authorship_tag": "ABX9TyMwDBetzOA+2uxl7mlVMm/2",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/AndreGulyi/ML_projects/blob/main/AG_Predicting_price.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install xgboost\n",
        "!pip install lightgbm\n",
        "!pip install google.colab\n",
        "!pip install tensorflow"
      ],
      "metadata": {
        "id": "XDP6bb2pf_Ij"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Z4Fm09xo7y5k"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import xgboost\n",
        "from xgboost import XGBRegressor\n",
        "from lightgbm import LGBMRegressor\n",
        "import sklearn\n",
        "from sklearn import tree\n",
        "from sklearn import svm\n",
        "from sklearn import neighbors\n",
        "from sklearn import linear_model\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import r2_score\n",
        "from sklearn.metrics import mean_squared_error\n",
        "from sklearn.neural_network import MLPRegressor\n",
        "from sklearn.ensemble import RandomForestRegressor"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "url='https://drive.google.com/file/d/1CF5wtIXc3Pi-BW0F6_XjblPfZ-iy28xP/view?usp=share_link\"'\n",
        "url='https://drive.google.com/uc?id=' + url.split('/')[-2]\n",
        "df = pd.read_csv(url)"
      ],
      "metadata": {
        "id": "BbAkFEjG8FoM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.info()"
      ],
      "metadata": {
        "id": "Pvgl3QxG8Mzw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.describe()"
      ],
      "metadata": {
        "id": "1f8HNI148kUO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.head(10)"
      ],
      "metadata": {
        "id": "wST6rD5I8oUt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Data Manipulation"
      ],
      "metadata": {
        "id": "YhFFTIza9__r"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df[\"loc1\"].value_counts()"
      ],
      "metadata": {
        "id": "yvz99tQ89we7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df[\"loc2\"].value_counts()"
      ],
      "metadata": {
        "id": "67sawL7P-OK2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df = df[(df[\"loc1\"].str.contains(\"S\") == False)&(df[\"loc1\"].str.contains(\"T\") == False)]\n",
        "df.shape"
      ],
      "metadata": {
        "id": "Qd0-_D6--cIQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df[\"loc1\"] = pd.to_numeric(df[\"loc1\"], errors=\"coerce\")\n",
        "df[\"loc2\"] = pd.to_numeric(df[\"loc2\"], errors=\"coerce\")\n",
        "df.dropna(inplace=True)\n",
        "df.shape"
      ],
      "metadata": {
        "id": "e4IB7fOATPuf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Data Type Changing"
      ],
      "metadata": {
        "id": "djGNy-iNUC1Y"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "days_dummies = pd.get_dummies(df.dow)\n",
        "days_dummies.head()"
      ],
      "metadata": {
        "id": "ixBaAX1WUDYq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df2 = df.copy(deep=True)\n",
        "df2.drop(columns = 'dow', inplace =True)"
      ],
      "metadata": {
        "id": "vvxPswhPUOQR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "result = df2.join(days_dummies)\n",
        "result.head()"
      ],
      "metadata": {
        "id": "OUu3DKPyUbmo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Checking Outliers and Correlations"
      ],
      "metadata": {
        "id": "8P-peP66Uy4k"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from pandas.plotting import scatter_matrix\n",
        "_ = scatter_matrix(result.iloc[:,0:7], figsize=(12, 8))"
      ],
      "metadata": {
        "id": "R-lGOeYsUjfY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pd.DataFrame((result.corr()['price'])).sort_values(by='price', ascending = False).round(2)"
      ],
      "metadata": {
        "id": "os1BEyEbVAud"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "result.iloc[:,0:6]"
      ],
      "metadata": {
        "id": "FooUCjkT8uTQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "result.iloc[:,0:6].hist(bins=50, figsize=(20,15))\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "uZbzct0D80PM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "result.iloc[:,0:6].describe()"
      ],
      "metadata": {
        "id": "X6cvKqQz9b9n"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "result['para1'].value_counts()"
      ],
      "metadata": {
        "id": "gh3WRgQN9wNk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#just delete outliers\n",
        "result = result[result['para1']<10]"
      ],
      "metadata": {
        "id": "f-XeyV3q96bb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Feature Selection"
      ],
      "metadata": {
        "id": "fnugts4Q-KEB"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "5 best params"
      ],
      "metadata": {
        "id": "pLYi-mtt_LfY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "five_best = []\n",
        "df_5 = pd.DataFrame(result.corr()['price']).sort_values(by='price', ascending=False)\n",
        "df_5 = df_5.drop(df_5.index[0]).head(5)\n",
        "df_5"
      ],
      "metadata": {
        "id": "S1FCx7fL-Y9F"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for i in range(len(df_5)):\n",
        "  five_best.append(df_5.index[i])"
      ],
      "metadata": {
        "id": "l24OA8KC8l5I"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "five_best"
      ],
      "metadata": {
        "id": "LrjzzqfD_Hge"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "3 best params"
      ],
      "metadata": {
        "id": "jvzTYMZq_V47"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "three_best = []\n",
        "df_3 = pd.DataFrame(result.corr()['price']).sort_values(by='price', ascending=False)\n",
        "df_3 = df_3.drop(df_3.index[0]).head(3)\n",
        "for i in range(len(df_3)):\n",
        "  three_best.append(df_3.index[i])\n",
        "\n",
        "three_best"
      ],
      "metadata": {
        "id": "tgnTxNV6_ZOp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Machine Learning\n",
        "Regression Models"
      ],
      "metadata": {
        "id": "pDrCJlWDWj6P"
      }
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "mbQc8sNvWcUK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "feature_sets = {\n",
        "    \"full_dataset\": result.drop(columns=['price']),\n",
        "    \"three_best\": result[three_best],\n",
        "    \"five_best\": result[five_best],  \n",
        "}"
      ],
      "metadata": {
        "id": "9GPJo8adVtKA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "regression_models = {\n",
        "    'Linear':linear_model.LinearRegression(),\n",
        "    'Lasso':linear_model.Lasso(random_state=8),\n",
        "    'LassoCV':linear_model.LassoCV(random_state=8),\n",
        "    'ElasticNet':linear_model.ElasticNet(random_state=8),\n",
        "    'LassoLars':linear_model.LassoLars(random_state=8),\n",
        "    'BayesianRidge':linear_model.BayesianRidge(),\n",
        "    'Ridge':linear_model.Ridge(random_state=8),\n",
        "    'DecisionTree':tree.DecisionTreeRegressor(random_state=8, max_depth=5),\n",
        "    'RandomForest':RandomForestRegressor(random_state=8),\n",
        "    'XGBoost': XGBRegressor(random_state=8),\n",
        "    'LGMB': LGBMRegressor(random_state=8),\n",
        "    'MLP':MLPRegressor(random_state=8),\n",
        "}"
      ],
      "metadata": {
        "id": "2q7_WcZ4XImx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def make_regression(x_train, y_train, x_test, y_test, model, model_name, verbose=True):\n",
        "\n",
        "    model.fit(x_train,y_train)\n",
        "    \n",
        "    y_predict = model.predict(x_train)\n",
        "    train_error = mean_squared_error(y_train, y_predict, squared=False)\n",
        "    \n",
        "    y_predict = model.predict(x_test)\n",
        "    test_error = mean_squared_error(y_test, y_predict, squared=False)\n",
        "    \n",
        "    y_predict = model.predict(x_train)\n",
        "    r2 = r2_score(y_train, y_predict)\n",
        "    \n",
        "    if verbose:\n",
        "        print(\"----Model name = {}-----\".format(model_name))\n",
        "        print(\"Train error = \"'{}'.format(train_error.round(1)))\n",
        "        print(\"Test error = \"'{}'.format(test_error.round(1)))\n",
        "        print(\"r2_score = \"'{}'.format(r2.round(2)))\n",
        "        print(\"--------------------------------\")\n",
        "    \n",
        "    trained_model = model\n",
        "    \n",
        "    return trained_model, y_predict, train_error, test_error, r2"
      ],
      "metadata": {
        "id": "SXZP_mAkZFpH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pred_dict = {\n",
        "    \"regression_model\": [],\n",
        "    \"feature_set\": [],\n",
        "    \"Train Error\": [],\n",
        "    \"Test Error\": [],\n",
        "    \"R2\" : []\n",
        "}"
      ],
      "metadata": {
        "id": "FKXUjcSwVq_x"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for feature_set_name in feature_sets.keys():\n",
        "    \n",
        "    feature_set = feature_sets[feature_set_name]\n",
        "    print(\"Included columns are {}\".format(feature_set_name))\n",
        "    for model_name in regression_models.keys():        \n",
        "        \n",
        "        y = result[\"price\"]\n",
        "        x = feature_set\n",
        "        x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=8)\n",
        "    \n",
        "\n",
        "        trained_model, y_predict, train_error, test_error, r2 = make_regression(x_train, y_train, x_test, y_test, regression_models[model_name], model_name, verbose=True)\n",
        "\n",
        "\n",
        "        pred_dict[\"regression_model\"].append(model_name)\n",
        "        pred_dict[\"feature_set\"].append(feature_set_name)\n",
        "        pred_dict[\"Train Error\"].append(train_error)\n",
        "        pred_dict[\"Test Error\"].append(test_error)\n",
        "        pred_dict[\"R2\"].append(r2)"
      ],
      "metadata": {
        "id": "2RfJkOrzVmTZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pred_df = pd.DataFrame(pred_dict)\n",
        "pred_df.head(30)"
      ],
      "metadata": {
        "id": "tbrHI-mQVjt5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pred_df['feature_set_2'] = pred_df['feature_set'].apply(lambda x: x.split('_')[0])\n",
        "pred_df.head(5)"
      ],
      "metadata": {
        "id": "YJLPB8HXU-gd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pred_df[\"Model_with_Dataset\"] = pred_df['regression_model']+\"_\"+pred_df['feature_set_2']\n",
        "pred_df.head(5)"
      ],
      "metadata": {
        "id": "hJvxilQvT_s7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_show = pred_df[['Train Error', 'Test Error', 'R2', \"Model_with_Dataset\"]]\n",
        "df_train_error = df_show[['Model_with_Dataset','Train Error']]\n",
        "df_test_error = df_show[['Model_with_Dataset','Test Error']]"
      ],
      "metadata": {
        "id": "oNhkTrPoT6zr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "M4dlH_yzCRS1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a figure and subplots\n",
        "fig, (ax2, ax3, ax4) = plt.subplots(1, 3, figsize=(14, 6))\n",
        "\n",
        "\n",
        "# Create the first graph\n",
        "df_show.plot(kind='barh', x='Model_with_Dataset', y='R2', color='red', ax=ax2, legend=False)\n",
        "ax2.set_xlabel('R Squared')\n",
        "ax2.set_ylabel('Model')\n",
        "ax2.set_title('R-squared')\n",
        "\n",
        "# Create the second graph\n",
        "df_train_error.plot(kind='barh', x='Model_with_Dataset', y='Train Error', color='blue', ax=ax3, legend=False)\n",
        "ax3.set_xlabel('Train Error')\n",
        "ax3.set_ylabel('Model')\n",
        "ax3.set_title('Train Error')\n",
        "\n",
        "\n",
        "# Create the second graph\n",
        "df_test_error.plot(kind='barh', x='Model_with_Dataset', y='Test Error', color='green', ax=ax4, legend=False)\n",
        "ax4.set_xlabel('Test Error')\n",
        "ax4.set_ylabel('Model')\n",
        "ax4.set_title('Test Error')\n",
        "\n",
        "# Fit the figure\n",
        "plt.tight_layout()\n",
        "\n",
        "# Show the figure\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "eDX7_7SdTOD3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pred_df.drop(columns=['feature_set_2','Model_with_Dataset'], inplace=True)"
      ],
      "metadata": {
        "id": "CVnqKuNMSJnR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Model Evaluation"
      ],
      "metadata": {
        "id": "-MxotC7pDn1n"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Highest R Squared"
      ],
      "metadata": {
        "id": "z29sCROPDrUF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pred_df.sort_values(by='R2', ascending=False).head(10)"
      ],
      "metadata": {
        "id": "Mogf-_Ug-MB4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Min Test Error"
      ],
      "metadata": {
        "id": "qWtMKDGTD7Xh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pred_df.sort_values(by='Test Error', ascending=True).head(5)"
      ],
      "metadata": {
        "id": "OOk_yfEVD8HT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Min Train Error"
      ],
      "metadata": {
        "id": "qOGMpl1AERTz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pred_df.sort_values(by='Train Error', ascending=True).head(5)"
      ],
      "metadata": {
        "id": "R_B4onq8ETar"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Deep Learning"
      ],
      "metadata": {
        "id": "A4PLE5dpFv5i"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "y = result['price']\n",
        "x = result.drop(columns=['price'])"
      ],
      "metadata": {
        "id": "qZkAKr3YFzWr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x_train, x_test, y_train, y_test = train_test_split(x,y, test_size=0.2, random_state=8)"
      ],
      "metadata": {
        "id": "IE8R5cp5F7cG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow.keras import regularizers\n",
        "import time"
      ],
      "metadata": {
        "id": "_XcgnQIoGHQk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "start = time.time()\n",
        "\n",
        "tf.random.set_seed(42)\n",
        "\n",
        "# Define a new model with more layers\n",
        "model = tf.keras.models.Sequential([\n",
        "    tf.keras.layers.Dense(units=64, activation='relu', input_shape=[x_train.shape[1]], kernel_regularizer=regularizers.l2(0.01)),\n",
        "    tf.keras.layers.Dense(units=32, activation='relu', kernel_regularizer=regularizers.l2(0.01)),\n",
        "    tf.keras.layers.Dense(units=16, activation='relu', kernel_regularizer=regularizers.l2(0.01)),\n",
        "    tf.keras.layers.Dense(units=8, activation='relu', kernel_regularizer=regularizers.l2(0.01)),\n",
        "    tf.keras.layers.Dense(units=1)\n",
        "])\n",
        "\n",
        "# Compile the model with a lower learning rate\n",
        "model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.01),\n",
        "              loss=tf.losses.MeanSquaredError(),\n",
        "              metrics=[tf.metrics.MeanAbsoluteError()])\n",
        "\n",
        "\n",
        "# Fit the model to the training data\n",
        "history = model.fit(x_train, y_train, epochs=1000, batch_size=32, verbose = 0,\n",
        "                    validation_data=(x_test, y_test))\n",
        "\n",
        "y_train_pred = model.predict(x_train)\n",
        "r2_second = r2_score(y_train, y_train_pred)\n",
        "\n",
        "# Select the MAE and val_MAE for the four desired epochs\n",
        "epochs_to_plot = [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000]\n",
        "mae_second = [history.history['mean_absolute_error'][epoch - 1] for epoch in epochs_to_plot]\n",
        "val_mae_second = [history.history['val_mean_absolute_error'][epoch - 1] for epoch in epochs_to_plot]\n",
        "\n",
        "# Plot the MAE\n",
        "plt.plot(epochs_to_plot, mae_second, 'b', label=f'Training MAE: {mae_second[-1]:.3f}')\n",
        "\n",
        "# Plot the val_MAE\n",
        "plt.plot(epochs_to_plot, val_mae_second, 'r', label=f'Test MAE: {val_mae_second[-1]:.3f}')\n",
        "plt.legend()\n",
        "plt.show()\n",
        "\n",
        "end = time.time()\n",
        "elapsed_time_seconds = end - start\n",
        "elapsed_time = (elapsed_time_seconds) / 60\n",
        "elapsed_time_seconds = round(elapsed_time_seconds,2)\n",
        "elapsed_time = round(elapsed_time,2)\n",
        "print('Execution time:', elapsed_time_seconds, 'seconds which is:', elapsed_time, 'minutes.' )"
      ],
      "metadata": {
        "id": "thS1_f2WI82Y"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}