TS_Seq2Seq.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Importing libraries\nimport tensorflow as tf\nimport numpy as np\nimport os\nimport random\nimport pandas as pd\nimport seaborn as sns\nfrom datetime import datetime\nimport matplotlib.pyplot as plt\nplt.rc('font', size=16)\nfrom sklearn.preprocessing import MinMaxScaler\nimport warnings\nwarnings.filterwarnings('ignore')\ntf.get_logger().setLevel('ERROR')\n\ntfk = tf.keras\ntfkl = tf.keras.layers\n\n# Setting random seed for reproducibility\nseed = 42\n\nrandom.seed(seed)\nos.environ['PYTHONHASHSEED'] = str(seed)\nnp.random.seed(seed)\ntf.random.set_seed(seed)\ntf.compat.v1.set_random_seed(seed)\n\ndataset = pd.read_csv('../input/an2dl-homework-2/Training.csv')\nprint(dataset.shape)\ndataset.head()\n\n# Printing dataset info\ndataset.info()","metadata":{"execution":{"iopub.status.busy":"2022-01-04T11:45:31.582429Z","iopub.execute_input":"2022-01-04T11:45:31.582810Z","iopub.status.idle":"2022-01-04T11:45:37.559278Z","shell.execute_reply.started":"2022-01-04T11:45:31.582718Z","shell.execute_reply":"2022-01-04T11:45:37.558482Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"code","source":"def inspect_dataframe(df, columns):\n    figs, axs = plt.subplots(len(columns), 1, sharex=True, figsize=(17,17))\n    for i, col in enumerate(columns):\n        axs[i].plot(df[col])\n        axs[i].set_title(col)\n    plt.show()\n    \n# Plotting time series\ninspect_dataframe(dataset, dataset.columns)","metadata":{"execution":{"iopub.status.busy":"2022-01-04T11:45:37.561291Z","iopub.execute_input":"2022-01-04T11:45:37.561793Z","iopub.status.idle":"2022-01-04T11:45:38.495953Z","shell.execute_reply.started":"2022-01-04T11:45:37.561752Z","shell.execute_reply":"2022-01-04T11:45:38.495323Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"code","source":"window = 400\nstride = 4\nvalidation_size = 13700 #(20%)\ntarget_labels = dataset.columns\ntelescope = 864 # Direct method","metadata":{"execution":{"iopub.status.busy":"2022-01-04T11:45:38.500100Z","iopub.execute_input":"2022-01-04T11:45:38.500553Z","iopub.status.idle":"2022-01-04T11:45:38.506887Z","shell.execute_reply.started":"2022-01-04T11:45:38.500516Z","shell.execute_reply":"2022-01-04T11:45:38.506055Z"},"trusted":true},"execution_count":3,"outputs":[]},{"cell_type":"code","source":"X_train_raw = dataset.iloc[:-validation_size]\nX_validation_raw = dataset.iloc[-validation_size:]\n\nprint(X_train_raw.shape, X_validation_raw.shape)\n\n# Normalize both features and labels\nX_min = X_train_raw.min()\nX_max = X_train_raw.max()\n\nX_train_raw = (X_train_raw-X_min)/(X_max-X_min)\nX_validation_raw = (X_validation_raw-X_min)/(X_max-X_min)","metadata":{"execution":{"iopub.status.busy":"2022-01-04T11:45:38.508491Z","iopub.execute_input":"2022-01-04T11:45:38.509043Z","iopub.status.idle":"2022-01-04T11:45:38.528663Z","shell.execute_reply.started":"2022-01-04T11:45:38.509006Z","shell.execute_reply":"2022-01-04T11:45:38.528018Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"code","source":"def printplots():\n    for i in range(X_train_raw.shape[1]):\n        plt.figure(figsize=(17,5))\n        plt.plot(X_train_raw[(target_labels[i])], label='Train')\n        plt.plot(X_validation_raw[(target_labels[i])], label='Test')\n        plt.title((target_labels[i]))\n        plt.legend()\n        plt.show()\n        \nprintplots()","metadata":{"execution":{"iopub.status.busy":"2022-01-04T11:45:38.529936Z","iopub.execute_input":"2022-01-04T11:45:38.530432Z","iopub.status.idle":"2022-01-04T11:45:40.626464Z","shell.execute_reply.started":"2022-01-04T11:45:38.530395Z","shell.execute_reply":"2022-01-04T11:45:40.625818Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"code","source":"future = dataset[-window:]\nfuture = (future-X_min)/(X_max-X_min)\nfuture = np.expand_dims(future, axis=0)\nfuture.shape","metadata":{"execution":{"iopub.status.busy":"2022-01-04T11:45:40.627805Z","iopub.execute_input":"2022-01-04T11:45:40.628254Z","iopub.status.idle":"2022-01-04T11:45:40.637712Z","shell.execute_reply.started":"2022-01-04T11:45:40.628209Z","shell.execute_reply":"2022-01-04T11:45:40.636880Z"},"trusted":true},"execution_count":6,"outputs":[]},{"cell_type":"code","source":"def build_sequences(df, target_labels, window=window, stride=stride, telescope=telescope):\n    \n    assert window % stride == 0\n    dataset = []\n    temp_df = df.copy().values\n    temp_label = df[target_labels].copy().values\n    padding_len = len(df)%window\n    labels_in = []\n    labels_out = []\n\n    if(padding_len != 0):\n        # Compute padding length\n        padding_len = window - len(df)%window\n        padding = np.zeros((padding_len,temp_df.shape[1]), dtype='float32')\n        temp_df = np.concatenate((padding,df))\n        padding = np.zeros((padding_len,temp_label.shape[1]), dtype='float32')\n        temp_label = np.concatenate((padding,temp_label))\n        assert len(temp_df) % window == 0\n\n    for idx in np.arange(0,len(temp_df)-window-telescope,stride):\n        dataset.append(temp_df[idx:idx+window]) # Encoder input for training [Ex: 0 1 2 ... 199]\n        \n        y_dec_in = temp_label[idx+window-1:idx+window+telescope-1] # Decoder input for training [Ex: 199 200 201 ...]\n        labels_in.append(y_dec_in)\n        y_dec_out = temp_label[idx+window:idx+window+telescope] # Decoder output for training [Ex: 200 201 202 ...]\n        labels_out.append(y_dec_out)\n\n    dataset = np.array(dataset)\n    labels_in = np.array(labels_in)\n    labels_out = np.array(labels_out)\n    \n    return dataset, labels_out, labels_in","metadata":{"execution":{"iopub.status.busy":"2022-01-04T11:45:40.639427Z","iopub.execute_input":"2022-01-04T11:45:40.640045Z","iopub.status.idle":"2022-01-04T11:45:40.651692Z","shell.execute_reply.started":"2022-01-04T11:45:40.640004Z","shell.execute_reply":"2022-01-04T11:45:40.650966Z"},"trusted":true},"execution_count":7,"outputs":[]},{"cell_type":"code","source":"X_train, y_train_dec_out, y_train_dec_in = build_sequences(X_train_raw, target_labels, window, stride, telescope)\nX_val, y_val_dec_out, y_val_dec_in = build_sequences(X_validation_raw, target_labels, window, stride, telescope)","metadata":{"execution":{"iopub.status.busy":"2022-01-04T11:45:40.653284Z","iopub.execute_input":"2022-01-04T11:45:40.653849Z","iopub.status.idle":"2022-01-04T11:45:41.249433Z","shell.execute_reply.started":"2022-01-04T11:45:40.653810Z","shell.execute_reply":"2022-01-04T11:45:41.248678Z"},"trusted":true},"execution_count":8,"outputs":[]},{"cell_type":"code","source":"print(X_train.shape)\nprint(y_train_dec_out.shape)\nprint(y_train_dec_in.shape)\nprint(X_val.shape)\nprint(y_val_dec_out.shape)\nprint(y_val_dec_in.shape)","metadata":{"execution":{"iopub.status.busy":"2022-01-04T11:45:41.250750Z","iopub.execute_input":"2022-01-04T11:45:41.251012Z","iopub.status.idle":"2022-01-04T11:45:41.258197Z","shell.execute_reply.started":"2022-01-04T11:45:41.250962Z","shell.execute_reply":"2022-01-04T11:45:41.257352Z"},"trusted":true},"execution_count":9,"outputs":[]},{"cell_type":"code","source":"input_shape = X_train.shape[1:]\noutput_shape = y_train_dec_out.shape[1:]\nbatch_size = 32\nepochs = 200","metadata":{"execution":{"iopub.status.busy":"2022-01-04T11:45:41.262599Z","iopub.execute_input":"2022-01-04T11:45:41.262849Z","iopub.status.idle":"2022-01-04T11:45:41.268288Z","shell.execute_reply.started":"2022-01-04T11:45:41.262816Z","shell.execute_reply":"2022-01-04T11:45:41.266507Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"code","source":"def SEQ2SEQ_3_Models(input_shape, output_shape, memelem):\n   \n    # Encoder \n    encoder_input = tfkl.Input(shape=input_shape, name='encoder_in')\n    encoder_output, enc_h, enc_c = tfkl.LSTM(memelem, return_state=True, name='enc_lstm')(encoder_input)\n    encoder_state = [enc_h, enc_c]\n     \n    # Decoder - Layers written in a different way because we need to call them later\n    decoder_input = tfkl.Input(shape=output_shape, name='decoder_in')\n    decoder_lstm = tfkl.LSTM(memelem, return_sequences=True, return_state=True, name='decoder_lstm') \n    decoder_output, _, _ = decoder_lstm(decoder_input, initial_state=encoder_state)\n    \n    # Dense Output - Layers written in a different way because we need to call them later\n    dense_out = tfkl.Dense(7, activation='relu', name='dense_out')\n    out = dense_out(decoder_output)\n    \n    # Model (Not for forecasting)\n    general_model = tfk.Model([encoder_input, decoder_input], out)\n    \n    # ----------------------------------------------------------------------------------\n    # One step-at-a-time prediction (Like real time translation - last exercise session)\n    # ----------------------------------------------------------------------------------\n    \n    # Encoder\n    encoder_only = tfk.Model(encoder_input, encoder_state)\n    \n    # Decoder state\n    dec_h = tfkl.Input(shape=memelem)\n    dec_c = tfkl.Input(shape=memelem)\n    decoder_state_in = [dec_h, dec_c]\n    \n    # New version of the decoder - Takes in input previous prediction and previous state\n    decoder_input_single = tf.keras.Input(shape=[1, 7]) \n    decoder_output, h, c = decoder_lstm(decoder_input_single, initial_state=decoder_state_in)\n    \n    decoder_state = [h, c]\n    decoder_output = dense_out(decoder_output)\n    decoder_only = tfk.Model([decoder_input_single] + decoder_state_in, [decoder_output] + decoder_state) # Takes in input previous prediction and previous state\n    \n    return general_model, encoder_only, decoder_only","metadata":{"execution":{"iopub.status.busy":"2022-01-04T11:45:41.269507Z","iopub.execute_input":"2022-01-04T11:45:41.269756Z","iopub.status.idle":"2022-01-04T11:45:41.282641Z","shell.execute_reply.started":"2022-01-04T11:45:41.269724Z","shell.execute_reply":"2022-01-04T11:45:41.281772Z"},"trusted":true},"execution_count":11,"outputs":[]},{"cell_type":"code","source":"general_model, encoder_for_forecasting, decoder_for_forecasting = SEQ2SEQ_3_Models(input_shape, output_shape, 256)\ngeneral_model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-01-04T11:45:41.283784Z","iopub.execute_input":"2022-01-04T11:45:41.284317Z","iopub.status.idle":"2022-01-04T11:45:44.361144Z","shell.execute_reply.started":"2022-01-04T11:45:41.284281Z","shell.execute_reply":"2022-01-04T11:45:44.360455Z"},"trusted":true},"execution_count":12,"outputs":[]},{"cell_type":"code","source":"encoder_for_forecasting.summary()","metadata":{"execution":{"iopub.status.busy":"2022-01-04T11:45:44.362515Z","iopub.execute_input":"2022-01-04T11:45:44.362746Z","iopub.status.idle":"2022-01-04T11:45:44.370131Z","shell.execute_reply.started":"2022-01-04T11:45:44.362714Z","shell.execute_reply":"2022-01-04T11:45:44.369348Z"},"trusted":true},"execution_count":13,"outputs":[]},{"cell_type":"code","source":"decoder_for_forecasting.summary()","metadata":{"execution":{"iopub.status.busy":"2022-01-04T11:45:44.372028Z","iopub.execute_input":"2022-01-04T11:45:44.372266Z","iopub.status.idle":"2022-01-04T11:45:44.386553Z","shell.execute_reply.started":"2022-01-04T11:45:44.372225Z","shell.execute_reply":"2022-01-04T11:45:44.385655Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"code","source":"general_model.compile(loss=tfk.losses.MeanSquaredError(), optimizer=tfk.optimizers.Adam(5e-4), metrics=[tfk.metrics.RootMeanSquaredError()])","metadata":{"execution":{"iopub.status.busy":"2022-01-04T11:45:44.387741Z","iopub.execute_input":"2022-01-04T11:45:44.388079Z","iopub.status.idle":"2022-01-04T11:45:44.406339Z","shell.execute_reply.started":"2022-01-04T11:45:44.388039Z","shell.execute_reply":"2022-01-04T11:45:44.405710Z"},"trusted":true},"execution_count":15,"outputs":[]},{"cell_type":"code","source":"# Training\nhistory = general_model.fit(\n    x = [X_train, y_train_dec_in],\n    y = y_train_dec_out,\n    batch_size = batch_size,\n    epochs = epochs,\n    validation_data = ([X_val,y_val_dec_in], y_val_dec_out),\n    callbacks = [tfk.callbacks.EarlyStopping(monitor='val_loss', mode='min', patience=10, restore_best_weights=True),\n                tfk.callbacks.ReduceLROnPlateau(monitor='val_loss', mode='min', patience=5, factor=0.5, min_lr=1e-5)]\n).history","metadata":{"execution":{"iopub.status.busy":"2022-01-04T11:45:44.407552Z","iopub.execute_input":"2022-01-04T11:45:44.408247Z","iopub.status.idle":"2022-01-04T12:20:43.669699Z","shell.execute_reply.started":"2022-01-04T11:45:44.408213Z","shell.execute_reply":"2022-01-04T12:20:43.669031Z"},"trusted":true},"execution_count":16,"outputs":[]},{"cell_type":"code","source":"best_epoch = np.argmin(history['val_loss'])\nplt.figure(figsize=(17,4))\nplt.plot(history['loss'], label='Training loss', alpha=.8, color='#ff7f0e')\nplt.plot(history['val_loss'], label='Validation loss', alpha=.9, color='#5a9aa5')\nplt.axvline(x=best_epoch, label='Best epoch', alpha=.3, ls='--', color='#5a9aa5')\nplt.title('Root Mean Squared Error (Loss)')\nplt.legend()\nplt.grid(alpha=.3)\nplt.show()\n\nplt.figure(figsize=(17,4))\nplt.plot(history['root_mean_squared_error'], label='Training RMSE', alpha=.8, color='#ff7f0e')\nplt.plot(history['val_root_mean_squared_error'], label='Validation RMSE', alpha=.9, color='#5a9aa5')\nplt.axvline(x=best_epoch, label='Best epoch', alpha=.3, ls='--', color='#5a9aa5')\nplt.title('Root Mean Squared Error')\nplt.legend()\nplt.grid(alpha=.3)\nplt.show()\n\nplt.figure(figsize=(18,3))\nplt.plot(history['lr'], label='Learning Rate', alpha=.8, color='#ff7f0e')\nplt.axvline(x=best_epoch, label='Best epoch', alpha=.3, ls='--', color='#5a9aa5')\nplt.legend()\nplt.grid(alpha=.3)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-04T12:20:43.671262Z","iopub.execute_input":"2022-01-04T12:20:43.671527Z","iopub.status.idle":"2022-01-04T12:21:11.321535Z","shell.execute_reply.started":"2022-01-04T12:20:43.671491Z","shell.execute_reply":"2022-01-04T12:21:11.320840Z"},"trusted":true},"execution_count":17,"outputs":[]},{"cell_type":"code","source":"general_model.save('./general')\nencoder_for_forecasting.save('./encoder')\ndecoder_for_forecasting.save('./decoder')","metadata":{"execution":{"iopub.status.busy":"2022-01-04T12:21:11.322604Z","iopub.execute_input":"2022-01-04T12:21:11.322853Z","iopub.status.idle":"2022-01-04T12:21:25.381270Z","shell.execute_reply.started":"2022-01-04T12:21:11.322819Z","shell.execute_reply":"2022-01-04T12:21:25.380482Z"},"trusted":true},"execution_count":18,"outputs":[]},{"cell_type":"code","source":"# Predict the test set \npredictions = general_model.predict([X_val, y_val_dec_out])\nprint(predictions.shape)\n\nmean_squared_error = tfk.metrics.mse(y_val_dec_out.flatten(),predictions.flatten())\nmean_absolute_error = tfk.metrics.mae(y_val_dec_out.flatten(),predictions.flatten())\nmean_squared_error, mean_absolute_error","metadata":{"execution":{"iopub.status.busy":"2022-01-04T12:21:25.384510Z","iopub.execute_input":"2022-01-04T12:21:25.384909Z","iopub.status.idle":"2022-01-04T12:21:32.647546Z","shell.execute_reply.started":"2022-01-04T12:21:25.384874Z","shell.execute_reply":"2022-01-04T12:21:32.646857Z"},"trusted":true},"execution_count":19,"outputs":[]},{"cell_type":"code","source":"def inspect_multivariate_prediction(X, y, pred, columns, telescope, idx=None):\n    if(idx==None):\n        idx=np.random.randint(0,len(X))\n\n    figs, axs = plt.subplots(len(columns), 1, sharex=True, figsize=(17,17))\n    for i, col in enumerate(columns):\n        axs[i].plot(np.arange(len(X[0,:,i])), X[idx,:,i])\n        axs[i].plot(np.arange(len(X[0,:,i]), len(X_train[0,:,i])+telescope), y[idx,:,i], color='orange')\n        axs[i].plot(np.arange(len(X[0,:,i]), len(X_train[0,:,i])+telescope), pred[idx,:,i], color='green')\n        axs[i].set_title(col)\n        axs[i].set_ylim(0,1)\n    plt.show()\n    \n    \ninspect_multivariate_prediction(X_val, y_val_dec_out[:,:864,:], predictions, target_labels, telescope)","metadata":{"execution":{"iopub.status.busy":"2022-01-04T12:21:32.648926Z","iopub.execute_input":"2022-01-04T12:21:32.649356Z","iopub.status.idle":"2022-01-04T12:21:33.381470Z","shell.execute_reply.started":"2022-01-04T12:21:32.649316Z","shell.execute_reply":"2022-01-04T12:21:33.380828Z"},"trusted":true},"execution_count":20,"outputs":[]},{"cell_type":"code","source":"temp_telescope = 864 # To not work always with 864 samples","metadata":{"execution":{"iopub.status.busy":"2022-01-04T12:21:33.382837Z","iopub.execute_input":"2022-01-04T12:21:33.383287Z","iopub.status.idle":"2022-01-04T12:21:33.386755Z","shell.execute_reply.started":"2022-01-04T12:21:33.383245Z","shell.execute_reply":"2022-01-04T12:21:33.385838Z"},"trusted":true},"execution_count":21,"outputs":[]},{"cell_type":"code","source":"def inspect_multivariate_prediction2(X, y, pred, columns, telescope, idx=None):\n    if(idx==None):\n        idx=np.random.randint(0,len(X))\n\n    figs, axs = plt.subplots(len(columns), 1, sharex=True, figsize=(17,17))\n    for i, col in enumerate(columns):\n        axs[i].plot(np.arange(len(X[0,:,i])), X[idx,:,i])\n        axs[i].plot(np.arange(len(X[0,:,i]), len(X_train[0,:,i])+telescope), y[idx,:,i], color='orange')\n        axs[i].plot(np.arange(len(X[0,:,i]), len(X_train[0,:,i])+telescope), pred[:,i], color='green')\n        axs[i].set_title(col)\n        axs[i].set_ylim(0,1)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-04T12:21:33.388083Z","iopub.execute_input":"2022-01-04T12:21:33.388575Z","iopub.status.idle":"2022-01-04T12:21:33.399113Z","shell.execute_reply.started":"2022-01-04T12:21:33.388541Z","shell.execute_reply":"2022-01-04T12:21:33.398456Z"},"trusted":true},"execution_count":22,"outputs":[]},{"cell_type":"code","source":"for seq in range (100, 1100, 100): # To plot more than one prediction\n\n    states_value = encoder_for_forecasting.predict(np.expand_dims(X_val[seq,:,:], axis=0))\n\n    curr_value = (X_val[seq,window-1,:])*np.ones((1, X_val.shape[2]), dtype='float32')\n    curr_value = np.expand_dims(curr_value, axis=0)\n    predictions = []\n    predictions = tf.convert_to_tensor(predictions)\n\n    for _ in range(temp_telescope):\n        preds, h_, c_ = decoder_for_forecasting.predict([curr_value] + states_value)\n        curr_value = preds\n        states_value = [h_, c_]\n\n        if(len(predictions)==0):\n            predictions = preds[0]\n        else:\n            predictions = np.concatenate((predictions,preds[0]),axis=0)\n\n    inspect_multivariate_prediction2(X_val, y_val_dec_out[:,:temp_telescope,:], predictions, target_labels, temp_telescope, seq)","metadata":{"execution":{"iopub.status.busy":"2022-01-04T12:21:33.400599Z","iopub.execute_input":"2022-01-04T12:21:33.401109Z","iopub.status.idle":"2022-01-04T12:27:38.529348Z","shell.execute_reply.started":"2022-01-04T12:21:33.401053Z","shell.execute_reply":"2022-01-04T12:27:38.528717Z"},"trusted":true},"execution_count":23,"outputs":[]}]}