SentEval_Probing_notebook.ipynb

{
 "nbformat": 4,
 "nbformat_minor": 0,
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "SentEval_Probing_notebook.ipynb",
   "provenance": [],
   "collapsed_sections": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  },
  "language_info": {
   "name": "python"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "32edc4660d3e47a4b9cf602de562181b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_3ca673b9477541acb30e59c8cc4454a0",
       "IPY_MODEL_3e7361769c124a82a01c99b84fe0ac81",
       "IPY_MODEL_776fdec286e74dbda6ad4fbeea1a7a44"
      ],
      "layout": "IPY_MODEL_e62492d1dddf4d45a5ef2f0c1f9c2e7f"
     }
    },
    "3ca673b9477541acb30e59c8cc4454a0": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_c6317605d11e43b285d5288e559db75d",
      "placeholder": "​",
      "style": "IPY_MODEL_b711a40bc71e4436bcaf04bd5c811da4",
      "value": "Downloading: 100%"
     }
    },
    "3e7361769c124a82a01c99b84fe0ac81": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d2908ac7943646519f3d23924da990f5",
      "max": 570,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_34500810f6b54a409cf9b5d5439c8e81",
      "value": 570
     }
    },
    "776fdec286e74dbda6ad4fbeea1a7a44": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_67ea02e38991455ebc4055dc0426a5b5",
      "placeholder": "​",
      "style": "IPY_MODEL_3c473dc1f3bc45b2a67d0abc29ce0f66",
      "value": " 570/570 [00:00&lt;00:00, 15.1kB/s]"
     }
    },
    "e62492d1dddf4d45a5ef2f0c1f9c2e7f": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c6317605d11e43b285d5288e559db75d": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "b711a40bc71e4436bcaf04bd5c811da4": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "d2908ac7943646519f3d23924da990f5": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "34500810f6b54a409cf9b5d5439c8e81": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "67ea02e38991455ebc4055dc0426a5b5": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3c473dc1f3bc45b2a67d0abc29ce0f66": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "1f0bf991b66a4836a31513413a1e08c8": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_534a01e589fb4c5e8f4f7c00a2109d50",
       "IPY_MODEL_5f95f6176b1f4d7480099b075ced9302",
       "IPY_MODEL_b5ec13ba3bff4493bf18dcb14487e6d7"
      ],
      "layout": "IPY_MODEL_d26c1195e3694b20bc9c28f1991b6ebe"
     }
    },
    "534a01e589fb4c5e8f4f7c00a2109d50": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_513d2a2c4e4a4a1c81b56314586828c4",
      "placeholder": "​",
      "style": "IPY_MODEL_529582d8bc05466f8e435467cb1c2a25",
      "value": "Downloading: 100%"
     }
    },
    "5f95f6176b1f4d7480099b075ced9302": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_f1055d8e21cf44b3aedafad287b78f4f",
      "max": 231508,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_940fcfb5bd594e19af33ee3c0e080502",
      "value": 231508
     }
    },
    "b5ec13ba3bff4493bf18dcb14487e6d7": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_eff46d97986046a588ee76949b3cc6a9",
      "placeholder": "​",
      "style": "IPY_MODEL_33808e1025b8438e83af15c6382c0379",
      "value": " 232k/232k [00:00&lt;00:00, 628kB/s]"
     }
    },
    "d26c1195e3694b20bc9c28f1991b6ebe": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "513d2a2c4e4a4a1c81b56314586828c4": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "529582d8bc05466f8e435467cb1c2a25": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "f1055d8e21cf44b3aedafad287b78f4f": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "940fcfb5bd594e19af33ee3c0e080502": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "eff46d97986046a588ee76949b3cc6a9": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "33808e1025b8438e83af15c6382c0379": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    }
   }
  }
 },
 "cells": [
  {
   "cell_type": "code",
   "metadata": {
    "id": "221EpIl82ACD"
   },
   "source": [
    "# @title Installing Transformers\n",
    "from IPython.display import clear_output\n",
    "!pip install transformers==4.3.3\n",
    "!git clone https://github.com/facebookresearch/SentEval\n",
    "from google.colab import drive\n",
    "drive.mount('/content/drive')\n",
    "clear_output()"
   ],
   "execution_count": 1,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "WqxC1_f66w3h"
   },
   "source": [
    "# @title Importing Requirements\n",
    "\n",
    "from transformers import (\n",
    "    BertConfig,\n",
    "    BertTokenizer,\n",
    "    TFBertModel,\n",
    "    BertModel,\n",
    "    glue_processors,\n",
    "    glue_convert_examples_to_features,\n",
    "    set_seed\n",
    ")\n",
    "from transformers.optimization_tf import create_optimizer\n",
    "\n",
    "import tensorflow as tf\n",
    "import tensorflow_datasets\n",
    "import numpy as np\n",
    "import copy \n",
    "import os\n",
    "import pandas as pd\n"
   ],
   "execution_count": 2,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "7QPzzfc3W_KN"
   },
   "source": [
    "# @title Hyperparameters\n",
    "BATCH_SIZE =  32# @param {type:\"integer\"}\n",
    "EPOCHS =  10#@param {type:\"integer\"}\n",
    "MAX_LENGTH =   64#@param {type:\"integer\"}\n",
    "\n",
    "TASK = \"cola\" #@param [\"cola\", \"sst2\", \"mrpc\", \"sts\", \"qqp\", \"mnli\", \"qnli\", \"rte\"]\n",
    "\n",
    "MODEL_SIZE = \"full\"#@param [\"full\", \"7k\", \"2.5k\", \"1k\"]\n",
    "PROBE = \"ObjNumber\" #@param ['Length','BigramShift', 'TopConst', 'Tense','SubjNumber', 'ObjNumber', 'OddManOut', 'CoordinationInversion']\n",
    "MODEL_SEED = \"42\" #@param [42, 123, 1234]\n",
    "PROBE_SEED = \"60\" #@param [40, 50, 60]\n",
    "num_labels = 2\n",
    "\n",
    "if PROBE == \"TopConst\":\n",
    "  num_labels = 20\n",
    "elif PROBE == 'Length':\n",
    "    num_labels = 6\n",
    "\n",
    "LEARNING_RATE =  3e-4 #@param {type:\"number\"}\n",
    "WARMUP_RATIO =   0.1 #@param {type:\"number\"}\n",
    "LAYER = \"12\" #@param [1,2,3,4, 5,6, 7,8, 9,10, 11, 12] \n",
    "LAYER = int(LAYER)\n",
    "\n",
    "#####YOU CAN REPLACE THIS LINE WITH YOUR SAVED MODEL'S PATH####\n",
    "SAVED_MODELS_DIR = \"/content/drive/Shareddrives/Unlimited/Datasize/Fine-Tuned-Models/\" + TASK + '-' + MODEL_SIZE + '-' + str(MODEL_SEED)\n",
    "\n",
    "\n",
    "DATA_NAME = \"\"\n",
    "if PROBE == \"Length\":\n",
    "  DATA_NAME = \"sentence_length.txt\"\n",
    "elif PROBE == \"BigramShift\":\n",
    "  DATA_NAME = \"bigram_shift.txt\"\n",
    "elif PROBE == \"TopConst\":\n",
    "  DATA_NAME = \"top_constituents.txt\"\n",
    "elif PROBE == \"Tense\":\n",
    "  DATA_NAME = \"past_present.txt\"\n",
    "elif PROBE == \"SubjNumber\":\n",
    "  DATA_NAME = \"subj_number.txt\"\n",
    "elif PROBE == \"ObjNumber\":\n",
    "  DATA_NAME = \"obj_number.txt\"\n",
    "elif PROBE == \"OddManOut\":\n",
    "  DATA_NAME = \"odd_man_out.txt\"\n",
    "elif PROBE == \"CoordinationInversion\":\n",
    "  DATA_NAME = \"coordination_inversion.txt\"\n",
    "\n",
    "set_seed(int(PROBE_SEED))"
   ],
   "execution_count": 3,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "H5ql0uu6kPoH"
   },
   "source": [
    "# @title Loading Probing Dataset\n",
    "File_object = open(\"/content/SentEval/data/probing/\" + DATA_NAME,\"r\")\n",
    "Lines = File_object.readlines()\n",
    "\n",
    "split =[]\n",
    "text = []\n",
    "label = []\n",
    "for line in Lines:\n",
    "    tmp = line.split('\\t')\n",
    "    split.append(tmp[0])\n",
    "    label.append(tmp[1])\n",
    "    text.append(tmp[2][:-1])\n",
    "\n",
    "df = pd.DataFrame(list(zip(split, label, text)),\n",
    "              )\n",
    "\n",
    "if PROBE != 'Length':\n",
    "  df[1] = df[1].factorize()[0]\n",
    "\n",
    "df_train = df[df[0]==\"tr\"]\n",
    "df_val = df[df[0]==\"va\"]\n",
    "df_test = df[df[0]==\"te\"]"
   ],
   "execution_count": 10,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "UTAmWbYJ580Y"
   },
   "source": [
    "# @title Probing Model\n",
    "\n",
    "class ProbeModel(tf.keras.Model):\n",
    "  def __init__(self, bert_model, num_labels, layer, *inputs, **kwargs):\n",
    "    super(ProbeModel, self).__init__(name=\"ProbeModel\")\n",
    "    self.bert = bert_model\n",
    "    self.bert.trainable = False\n",
    "    self.dropout = tf.keras.layers.Dropout(0.1)\n",
    "    self.layer = layer\n",
    "    self.classifier = tf.keras.layers.Dense(\n",
    "                      num_labels,\n",
    "                      kernel_initializer=tf.keras.initializers.TruncatedNormal(stddev=0.02),\n",
    "                      name=\"classifier\")\n",
    "    \n",
    "  \n",
    "  def call(self, inputs, **kwargs):\n",
    "\n",
    "    outputs = self.bert(inputs, **kwargs)\n",
    "    pooled_out = outputs[2][self.layer]\n",
    "    pooled_out = pooled_out[:,0,:]\n",
    "    \n",
    "    droped_out = self.dropout(pooled_out, training=kwargs.get(\"training\", False))\n",
    "    output = self.classifier(droped_out)\n",
    "    return output"
   ],
   "execution_count": 11,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "mSRtIriB9SIP"
   },
   "source": [
    "# @title Tokenizer\n",
    "def tokenize(df, tokenizer, MAX_LENGTH):\n",
    "    input_ids, input_masks, input_segments = [],[],[]\n",
    "    for i in range(len(df)):\n",
    "        inputs = tokenizer.encode_plus(df[2].iloc[i], add_special_tokens=True, \n",
    "                                       return_attention_mask=True, return_token_type_ids=True, max_length = MAX_LENGTH,\n",
    "                                       pad_to_max_length=True, truncation =True )\n",
    "\n",
    "        input_ids.append(inputs['input_ids'])\n",
    "        input_masks.append(inputs['attention_mask'])\n",
    "        input_segments.append(inputs['token_type_ids'])  \n",
    "\n",
    "    return [tf.cast(input_ids,tf.int32),tf.cast(input_masks,tf.int32), tf.cast(input_segments,tf.int32)]"
   ],
   "execution_count": 12,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 192,
     "referenced_widgets": [
      "32edc4660d3e47a4b9cf602de562181b",
      "3ca673b9477541acb30e59c8cc4454a0",
      "3e7361769c124a82a01c99b84fe0ac81",
      "776fdec286e74dbda6ad4fbeea1a7a44",
      "e62492d1dddf4d45a5ef2f0c1f9c2e7f",
      "c6317605d11e43b285d5288e559db75d",
      "b711a40bc71e4436bcaf04bd5c811da4",
      "d2908ac7943646519f3d23924da990f5",
      "34500810f6b54a409cf9b5d5439c8e81",
      "67ea02e38991455ebc4055dc0426a5b5",
      "3c473dc1f3bc45b2a67d0abc29ce0f66",
      "1f0bf991b66a4836a31513413a1e08c8",
      "534a01e589fb4c5e8f4f7c00a2109d50",
      "5f95f6176b1f4d7480099b075ced9302",
      "b5ec13ba3bff4493bf18dcb14487e6d7",
      "d26c1195e3694b20bc9c28f1991b6ebe",
      "513d2a2c4e4a4a1c81b56314586828c4",
      "529582d8bc05466f8e435467cb1c2a25",
      "f1055d8e21cf44b3aedafad287b78f4f",
      "940fcfb5bd594e19af33ee3c0e080502",
      "eff46d97986046a588ee76949b3cc6a9",
      "33808e1025b8438e83af15c6382c0379"
     ]
    },
    "id": "0Pd-Mz7IMYEy",
    "outputId": "3a024047-a217-4e2a-c898-cf12fdad7a22"
   },
   "source": [
    "# @title Loading the Model\n",
    "\n",
    "casing = \"bert-base-uncased\" \n",
    "config = BertConfig.from_pretrained(casing, num_labels=num_labels)\n",
    "config.output_hidden_states = True\n",
    "tokenizer = BertTokenizer.from_pretrained(casing)\n",
    "bert_model = TFBertModel.from_pretrained(SAVED_MODELS_DIR, from_pt=True, config = config)\n",
    "bert_model.output_hidden_states=True\n",
    "model = ProbeModel(bert_model, num_labels, LAYER)\n"
   ],
   "execution_count": 13,
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": [
       "Downloading:   0%|          | 0.00/570 [00:00<?, ?B/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "32edc4660d3e47a4b9cf602de562181b"
      }
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": [
       "Downloading:   0%|          | 0.00/232k [00:00<?, ?B/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "1f0bf991b66a4836a31513413a1e08c8"
      }
     },
     "metadata": {}
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFBertModel: ['bert.embeddings.position_ids', 'classifier.weight', 'classifier.bias']\n",
      "- This IS expected if you are initializing TFBertModel from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFBertModel from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the weights of TFBertModel were initialized from the PyTorch model.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertModel for predictions without further training.\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "JV3izjIm4DNn",
    "outputId": "314d7680-26e5-40d9-8c0b-3e8a6d2ef2cc"
   },
   "source": [
    "# @title Making dataset ready \n",
    "\n",
    "train_dataset = tokenize(df_train, tokenizer, MAX_LENGTH)\n",
    "valid_dataset = tokenize(df_val, tokenizer, MAX_LENGTH)\n",
    "test_dataset  = tokenize(df_test, tokenizer, MAX_LENGTH)\n",
    "  \n",
    "\n",
    "train_steps = int(np.ceil(len(df_train) / BATCH_SIZE))\n",
    "valid_steps = int(np.ceil((len(df_val)) / BATCH_SIZE))\n",
    "test_steps = int(np.ceil(len(df_test) / BATCH_SIZE))\n",
    "\n",
    "clear_output()\n",
    "print(\"completed\")"
   ],
   "execution_count": 14,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "completed\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "yebPoFtH_GKT"
   },
   "source": [
    "# @title Freezing BERT\n",
    "model.bert.bert.embeddings.trainable = False\n",
    "model.bert.bert.pooler.trainable = False\n",
    "for i in range(config.num_hidden_layers):\n",
    "  model.bert.bert.encoder.layer[i].attention.self_attention.trainable = False\n",
    "  model.bert.bert.encoder.layer[i].attention.dense_output.dense.trainable = False\n",
    "  model.bert.bert.encoder.layer[i].attention.dense_output.LayerNorm.trainable = False\n",
    "  model.bert.bert.encoder.layer[i].intermediate.trainable = False\n",
    "  model.bert.bert.encoder.layer[i].bert_output.dense.trainable = False\n",
    "  model.bert.bert.encoder.layer[i].bert_output.LayerNorm.trainable = False"
   ],
   "execution_count": 15,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "h_tNzePP-i28"
   },
   "source": [
    "# @title Metrics, Loss & Optimizer\n",
    "\n",
    "loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
    "opt, scheduler = create_optimizer(init_lr=LEARNING_RATE,\n",
    "                       num_train_steps=train_steps * EPOCHS,\n",
    "                       num_warmup_steps=int(train_steps * EPOCHS * WARMUP_RATIO),\n",
    "                       adam_epsilon=1e-6,\n",
    "                       weight_decay_rate = 0)\n",
    "model.compile(optimizer=opt, loss=loss, metrics=['accuracy'])\n"
   ],
   "execution_count": 16,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "D49nsl1JL6Rz"
   },
   "source": [
    "# @title Callback\n",
    "class ModelCheckpoint(tf.keras.callbacks.Callback):\n",
    "  def __init__(self, monitor, save_path):\n",
    "    super(ModelCheckpoint, self).__init__()\n",
    "    self.monitor = monitor\n",
    "    self.save_path = save_path\n",
    "    self.bestScore = -np.Inf\n",
    "    self.bestLoss = np.Inf\n",
    "\n",
    "  def on_epoch_end(self, epoch, logs):\n",
    "    score = logs.get(self.monitor)\n",
    "    loss = logs.get(\"val_loss\")\n",
    "    if score > self.bestScore or (score == self.bestScore and loss < self.bestLoss):\n",
    "      path = os.path.join(TASK, str(epoch+1))\n",
    "      os.makedirs(path)\n",
    "      self.model.save_weights(path+'/best_weights.h5')\n",
    "      self.bestScore = score\n",
    "      self.bestLoss = loss\n",
    "      print(\"\\nModel saved as the best model\")\n",
    "\n",
    "monitor = \"val_accuracy\"\n",
    "checkpoint = ModelCheckpoint(monitor, SAVED_MODELS_DIR)\n"
   ],
   "execution_count": 17,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "2ahIUftw-zwV",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "9bd6b613-27e1-4de6-e4a7-a4c118e248e2"
   },
   "source": [
    "# @title Training\n",
    "history = model.fit(\n",
    "    train_dataset,\n",
    "    df_train[1],\n",
    "    epochs=EPOCHS,\n",
    "    batch_size = BATCH_SIZE,\n",
    "    validation_data=(valid_dataset, df_val[1]),\n",
    "    callbacks=[checkpoint]\n",
    "\n",
    ")\n",
    "print(model.summary())"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch 1/10\n",
      "3125/3125 [==============================] - ETA: 0s - loss: 0.5960 - accuracy: 0.6811\n",
      "Model saved as the best model\n",
      "3125/3125 [==============================] - 297s 93ms/step - loss: 0.5960 - accuracy: 0.6811 - val_loss: 0.5254 - val_accuracy: 0.7496\n",
      "Epoch 2/10\n",
      "3125/3125 [==============================] - ETA: 0s - loss: 0.5127 - accuracy: 0.7574\n",
      "Model saved as the best model\n",
      "3125/3125 [==============================] - 288s 92ms/step - loss: 0.5127 - accuracy: 0.7574 - val_loss: 0.4970 - val_accuracy: 0.7675\n",
      "Epoch 3/10\n",
      "3125/3125 [==============================] - ETA: 0s - loss: 0.5051 - accuracy: 0.7613\n",
      "Model saved as the best model\n",
      "3125/3125 [==============================] - 288s 92ms/step - loss: 0.5051 - accuracy: 0.7613 - val_loss: 0.4906 - val_accuracy: 0.7727\n",
      "Epoch 4/10\n",
      "3125/3125 [==============================] - ETA: 0s - loss: 0.5051 - accuracy: 0.7624\n",
      "Model saved as the best model\n",
      "3125/3125 [==============================] - 288s 92ms/step - loss: 0.5051 - accuracy: 0.7624 - val_loss: 0.4894 - val_accuracy: 0.7736\n",
      "Epoch 5/10\n",
      "3125/3125 [==============================] - ETA: 0s - loss: 0.5024 - accuracy: 0.7638\n",
      "Model saved as the best model\n",
      "3125/3125 [==============================] - 288s 92ms/step - loss: 0.5024 - accuracy: 0.7638 - val_loss: 0.4873 - val_accuracy: 0.7761\n",
      "Epoch 6/10\n",
      "3125/3125 [==============================] - 287s 92ms/step - loss: 0.5016 - accuracy: 0.7635 - val_loss: 0.4874 - val_accuracy: 0.7737\n",
      "Epoch 7/10\n",
      "3125/3125 [==============================] - 287s 92ms/step - loss: 0.5006 - accuracy: 0.7642 - val_loss: 0.4860 - val_accuracy: 0.7745\n",
      "Epoch 8/10\n",
      "3125/3125 [==============================] - 287s 92ms/step - loss: 0.5028 - accuracy: 0.7629 - val_loss: 0.4865 - val_accuracy: 0.7756\n",
      "Epoch 9/10\n",
      "3125/3125 [==============================] - 287s 92ms/step - loss: 0.4986 - accuracy: 0.7677 - val_loss: 0.4863 - val_accuracy: 0.7740\n",
      "Epoch 10/10\n",
      "3125/3125 [==============================] - 287s 92ms/step - loss: 0.4996 - accuracy: 0.7648 - val_loss: 0.4860 - val_accuracy: 0.7759\n",
      "Model: \"ProbeModel\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " tf_bert_model_2 (TFBertMode  multiple                 109482240 \n",
      " l)                                                              \n",
      "                                                                 \n",
      " dropout_113 (Dropout)       multiple                  0         \n",
      "                                                                 \n",
      " classifier (Dense)          multiple                  1538      \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 109,483,778\n",
      "Trainable params: 1,538\n",
      "Non-trainable params: 109,482,240\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "AhLSjbtFjzGb"
   },
   "source": [
    "# @title Retrieving the best model\n",
    "import os \n",
    "list_of_dirs = os.listdir('/content/' + TASK)\n",
    " \n",
    "\n",
    "final_list = list(map(int, list_of_dirs))\n",
    "best_model = max(final_list)\n",
    "\n",
    "model_path =\"/content/\"+ TASK + '/' + str(best_model) + '/best_weights.h5'\n",
    "model.load_weights(model_path)"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "BOeujsE5ci4q",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "a012bef3-7fb6-4afc-c592-c45c309bc33f"
   },
   "source": [
    "# @title Evaluation\n",
    "model.evaluate(test_dataset, df_test[1])"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "313/313 [==============================] - 29s 93ms/step - loss: 0.4732 - accuracy: 0.7943\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "[0.47322216629981995, 0.7943000197410583]"
      ]
     },
     "metadata": {},
     "execution_count": 39
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "5tfuApSveAuo"
   },
   "source": [
    "# @title Cleaning drive\n",
    "import shutil\n",
    "\n",
    "shutil.rmtree('/content/' + TASK + '/')\n",
    "\n",
    "print(PROBE + ':')\n",
    "print(TASK + ' layer: ' + str(LAYER) + ' with size:' + MODEL_SIZE + '  is done')"
   ],
   "execution_count": null,
   "outputs": []
  }
 ]
}