QueryProfiler Main Page - Added check for table retention issues when…

… using Transaction Ids and Statement Ids to create query plan
vertica · Nov 25, 2024 · 367d93c · 367d93c
1 parent 23c4b02
commit 367d93c
Showing 1 changed file with 62 additions and 0 deletions.
diff --git a/project/ui/qprof_main.ipynb b/project/ui/qprof_main.ipynb
@@ -65,6 +65,7 @@
     "logging.info('[Query Profile Main Page] Importing Libraries')\n",
     "from ipywidgets import interact, interactive, fixed, interact_manual\n",
     "import ipywidgets as widgets\n",
+    "from datetime import datetime\n",
     "import verticapy as vp\n",
     "from IPython.display import display,clear_output, IFrame, HTML\n",
     "import pickle\n",
@@ -611,6 +612,66 @@
     "target_schema_2_combo = widgets.HBox([target_schema__2, create_tooltip(\"Enter the schema in which you would like to store the profile data. Note that you will need the combination of schema and the key to load your saved data. If you do not provide this, you may not be able to export your profile tables to a tar file.\")])"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fd710f93-285e-4662-a722-c79cbd9559ed",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def check_rentention(transactions):\n",
+    "    # Initialize an empty condition string\n",
+    "    condition=\"\"\n",
+    "    # Loop through pairs and build the condition\n",
+    "    for i in range(len(transactions)):\n",
+    "      # Append each pair to the condition with an OR if not the first pair\n",
+    "      if i !=0:\n",
+    "        condition+=\" OR \"\n",
+    "      # Add the pair condition\n",
+    "      condition+=f\"(transaction_id = {transactions[i][0]} AND statement_id = {transactions[i][1]})\"\n",
+    "    # Formulate the SQL query with the constructed condition\n",
+    "    query = f\"SELECT MIN(start_timestamp) FROM query_requests WHERE {condition};\"\n",
+    "    res= vp._utils._sql._sys._executeSQL(query, method = \"fetchall\")\n",
+    "    query_min_time = res[0][0]\n",
+    "    if isinstance(res[0][0], type(None)):\n",
+    "        raise ValueError(f\"The queries with transaction id/statement ids '{transactions}' not found int the QUERY_REQUESTS table.\")\n",
+    "\n",
+    "    important_table_list = [\n",
+    "        # ['execution_engine_profiles', None], # This is crucial to have data. But do we have warning if there is no data?\n",
+    "        ['v_internal.dc_explain_plans', 'time'],\n",
+    "        ['v_internal.dc_query_executions', 'time'],\n",
+    "        ['v_internal.dc_requests_issued', 'time'],\n",
+    "        ['v_internal.dc_plan_steps', 'time'],\n",
+    "        ['v_monitor.query_events', 'event_timestamp'],\n",
+    "        # ['host_resources', None],\n",
+    "        ['v_monitor.query_consumption', 'start_time'],\n",
+    "        # ['v_monitor.query_plan_profiles', None],\n",
+    "        ['query_profiles', 'query_start'],\n",
+    "        # ['resource_pool_status', None],\n",
+    "        ['v_monitor.resource_acquisitions' , 'queue_entry_timestamp'],\n",
+    "        ['v_internal.dc_plan_activities', 'start_time']\n",
+    "    ]\n",
+    "    for table, table_time in important_table_list:\n",
+    "        res= vp._utils._sql._sys._executeSQL(f\"SELECT MIN({table_time}) from {table};\", method = \"fetchall\")\n",
+    "        table_min_time = res[0][0]\n",
+    "        if isinstance(table_min_time, str):\n",
+    "            date_string = res[0][0]\n",
+    "            # Replace '+00' with '+0000' to match the expected format\n",
+    "            try:\n",
+    "                date_string = date_string.replace('+00', '+0000')\n",
+    "                table_min_time = datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S.%f%z')\n",
+    "            except:\n",
+    "                if '+' in date_string or '-' in date_string:\n",
+    "                    # Locate the position of '+' or '-' for timezone\n",
+    "                    split_pos = max(date_string.rfind('+'), date_string.rfind('-'))\n",
+    "                    if len(date_string) - split_pos == 3:  # Handle offsets like '-05'\n",
+    "                        date_string = date_string[:split_pos] + date_string[split_pos:] + ':00'\n",
+    "                table_min_time = datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S.%f%z')\n",
+    "        print(f\"The table '{table}' has entries going far back as time {res[0][0]}\")\n",
+    "        if query_min_time<table_min_time:\n",
+    "            raise ValueError(\"Retention issues. The table '{table}' does not seem to have old enough data to capture the query profile. Please re-profile your query\")"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -636,6 +697,7 @@
     "            key_val_2 = key_2.value\n",
     "            transaction_val = transaction.value\n",
     "            statement_val = statement.value\n",
+    "            check_rentention(transactions = [(transaction_val,statement_val),])\n",
     "            logging.info(f\"[Query Profile Main Page] For create option, saved key value as :{key_val_2}, target schema as {target_schema_val_2}, transaction id as {transaction_val}, statement id as {statement_val}.\")\n",
     "            reset_values(statement_val = statement_val, transaction_val = transaction_val, target_schema_val_2 = target_schema_val_2, key_val_2 = key_val_2)\n",
     "\n",