Merge pull request #126 from andrewm4894/fasthtml

Fasthtml

.example.env

@@ -19,10 +19,10 @@ ANOMSTACK_AWS_ACCESS_KEY_ID=
 ANOMSTACK_AWS_SECRET_ACCESS_KEY=
 
 # local duckdb path for testing/dev quickstart
-ANOMSTACK_DUCKDB_PATH=tmpdata/anomstack.db
+ANOMSTACK_DUCKDB_PATH=tmpdata/anomstack-duckdb.db
 
 # local sqlite path for testing/dev quickstart
-ANOMSTACK_SQLITE_PATH=tmpdata/anomstack.db
+ANOMSTACK_SQLITE_PATH=tmpdata/anomstack-sqlite.db
 
 # table id to store metrics in
 ANOMSTACK_TABLE_KEY=tmp.metrics

.gitignore

@@ -178,3 +178,6 @@ models/*
 # tmp
 tmp
 tmpdata
+.sesskey
+.telemetry/id.yaml
+.sesskey

.vscode/settings.json

@@ -15,8 +15,8 @@
         {
             "previewLimit": 50,
             "driver": "SQLite",
-            "database": "${workspaceFolder:anomstack}/tmpdata/anomstack.db",
-            "name": "tmpdata/anomstack.db"
+            "database": "${workspaceFolder:anomstack}/tmpdata/anomstack-sqlite.db",
+            "name": "tmpdata/anomstack-sqlite.db"
         }
     ],
     "sqltools.useNodeRuntime": true

Makefile

@@ -1,7 +1,7 @@
 SHELL=/bin/bash
 
-.PHONY: dashboard
-.PHONY: dashboardd
+.PHONY: streamlit
+.PHONY: streamlitd
 .PHONY: local
 .PHONY: locald
 .PHONY: docker
@@ -10,16 +10,18 @@ SHELL=/bin/bash
 .PHONY: docs
 .PHONY: requirements
 .PHONY: kill-locald
-.PHONY: kill-dashboardd
+.PHONY: kill-streamlitd
 .PHONY: ps-locald
+.PHONY: fasthtml
+.PHONY: fasthtmld
 
 # start streamlit dashboard
-dashboard:
-	streamlit run ./dashboard.py --server.port 8501
+streamlit:
+	streamlit run ./dashboard/streamlit-dashboard.py --server.port 8501
 
 # start streamlit dashboard as a daemon with no log file
-dashboardd:
-	nohup streamlit run ./dashboard.py --server.port 8501 > /dev/null 2>&1 &
+streamlitd:
+	nohup streamlit run ./dashboard/streamlit-dashboard.py --server.port 8501 > /dev/null 2>&1 &
 
 # start dagster locally
 local:
@@ -34,7 +36,7 @@ kill-locald:
 	kill -9 $(shell ps aux | grep dagster | grep -v grep | awk '{print $$2}')
 
 # kill any running streamlit process
-kill-dashboardd:
+kill-streamlitd:
 	kill -9 $(shell ps aux | grep streamlit | grep -v grep | awk '{print $$2}')
 
 # list any running dagster process
@@ -62,3 +64,9 @@ docs:
 
 requirements:
 	pip-compile requirements.compile
+
+fasthtml:
+	python dashboard/fasthtml-dashboard.py
+
+fasthtmld:
+	nohup python dashboard/fasthtml-dashboard.py > /dev/null 2>&1 &

anomstack/external/duckdb/duckdb.py

@@ -22,7 +22,7 @@ def read_sql_duckdb(sql: str) -> pd.DataFrame:
 
     logger = get_dagster_logger()
 
-    duckdb_path = os.environ.get("ANOMSTACK_DUCKDB_PATH", "tmpdata/anomstack.db")
+    duckdb_path = os.environ.get("ANOMSTACK_DUCKDB_PATH", "tmpdata/anomstack-duckdb.db")
     logger.info(f"duckdb_path:{duckdb_path}")
 
     os.makedirs(os.path.dirname(duckdb_path), exist_ok=True)
@@ -47,7 +47,7 @@ def save_df_duckdb(df: pd.DataFrame, table_key: str) -> pd.DataFrame:
 
     logger = get_dagster_logger()
 
-    duckdb_path = os.environ.get("ANOMSTACK_DUCKDB_PATH", "tmpdata/anomstack.db")
+    duckdb_path = os.environ.get("ANOMSTACK_DUCKDB_PATH", "tmpdata/anomstack-duckdb.db")
     logger.info(f"duckdb_path:{duckdb_path}")
 
     os.makedirs(os.path.dirname(duckdb_path), exist_ok=True)
@@ -77,7 +77,7 @@ def run_sql_duckdb(sql: str) -> None:
     """
     logger = get_dagster_logger()
 
-    duckdb_path = os.environ.get("ANOMSTACK_DUCKDB_PATH", "tmpdata/anomstack.db")
+    duckdb_path = os.environ.get("ANOMSTACK_DUCKDB_PATH", "tmpdata/anomstack-duckdb.db")
     logger.info(f"duckdb_path: {duckdb_path}")
 
     os.makedirs(os.path.dirname(duckdb_path), exist_ok=True)

anomstack/external/sqlite/sqlite.py

@@ -24,7 +24,7 @@ def read_sql_sqlite(sql: str) -> pd.DataFrame:
         pd.DataFrame: The result of the SQL query as a pandas DataFrame.
     """
     logger = get_dagster_logger()
-    sqlite_path = os.environ.get("ANOMSTACK_SQLITE_PATH", "tmpdata/anomstack.db")
+    sqlite_path = os.environ.get("ANOMSTACK_SQLITE_PATH", "tmpdata/anomstack-sqlite.db")
     logger.info(f"sqlite_path: {sqlite_path}")
     os.makedirs(os.path.dirname(sqlite_path), exist_ok=True)
 
@@ -66,7 +66,7 @@ def save_df_sqlite(df: pd.DataFrame, table_key: str) -> pd.DataFrame:
         pd.DataFrame: The input DataFrame.
     """
     logger = get_dagster_logger()
-    sqlite_path = os.environ.get("ANOMSTACK_SQLITE_PATH", "tmpdata/anomstack.db")
+    sqlite_path = os.environ.get("ANOMSTACK_SQLITE_PATH", "tmpdata/anomstack-sqlite.db")
     logger.info(f"sqlite_path: {sqlite_path}")
     os.makedirs(os.path.dirname(sqlite_path), exist_ok=True)
 
@@ -103,7 +103,7 @@ def run_sql_sqlite(sql: str) -> None:
         None
     """
     logger = get_dagster_logger()
-    sqlite_path = os.environ.get("ANOMSTACK_SQLITE_PATH", "tmpdata/anomstack.db")
+    sqlite_path = os.environ.get("ANOMSTACK_SQLITE_PATH", "tmpdata/anomstack-sqlite.db")
     logger.info(f"sqlite_path: {sqlite_path}")
     os.makedirs(os.path.dirname(sqlite_path), exist_ok=True)
 

dashboard/README.md

@@ -0,0 +1,6 @@
+# Dashboard
+
+Some simple local dashboard options live here.
+
+- [`streamlit-dashboard.py`](streamlit-dashboard.py): [Streamlit](https://streamlit.io/) dashboard for the project.
+- [`fasthtml-dashboard.py`](fasthtml-dashboard.py): [FastHTML](https://fastht.ml/) dashboard for the project (WIP).

dashboard/fasthtml-dashboard.py

@@ -0,0 +1,54 @@
+import logging
+
+from fasthtml.common import Title, fast_app, serve
+from fh_plotly import plotly2fasthtml, plotly_headers
+from utils import get_enabled_dagster_jobs, plot_time_series
+
+from anomstack.config import specs
+from anomstack.jinja.render import render
+from anomstack.sql.read import read_sql
+
+log = logging.getLogger("fasthtml")
+
+app, rt = fast_app(hdrs=(plotly_headers,))
+
+
+@app.get('/')
+def home():
+
+    # get enabled jobs from Dagster
+    enabled_jobs = get_enabled_dagster_jobs()
+
+    figs = []
+
+    # loop through metric batches
+    for i, metric_batch in enumerate(specs, start=1):
+
+        if i > 3:
+            break
+
+        # check if the job is enabled
+        if f'{metric_batch}_ingest' in enabled_jobs:
+
+            spec = specs[metric_batch]
+            sql = render("dashboard_sql", spec, params={"alert_max_n": 90})
+            db = spec["db"]
+            df = read_sql(sql, db=db)
+
+            for metric_name in df["metric_name"].unique():
+
+                metric_df = df[df["metric_name"] == metric_name]
+                fig = plotly2fasthtml(plot_time_series(metric_df, metric_name))
+                figs.append(fig)
+
+        else:
+            log.debug(f"job {metric_batch}_ingest is not enabled.")
+
+    return (
+        Title("Anomstack"),
+        figs
+        )
+
+
+
+serve()