Merge pull request #43 from andrewm4894/pre-commit-initial-changes

add pre-commit changes

.devcontainer/devcontainer.json

@@ -3,4 +3,4 @@
     "forwardPorts": [3000],
     "remoteUser": "root",
     "postCreateCommand": "bash .devcontainer/post_create_command.sh"
-}
+}

.devcontainer/post_create_command.sh

@@ -10,4 +10,4 @@ cp .example.env .env
 # docker compose up
 docker compose up -d
 
-echo "done post create command"
+echo "done post create command"

.github/workflows/branch_deployments.yml

@@ -2,7 +2,7 @@ name: Serverless Branch Deployments
 on:
   pull_request:
     types: [opened, synchronize, reopened, closed]
-    
+
 concurrency:
   # Cancel in-progress deploys to same branch
   group: ${{ github.ref }}/branch_deployments
@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-20.04
     outputs:
       build_info: ${{ steps.parse-workspace.outputs.build_info }}
-    
+
     steps:
       - name: Prerun Checks
         id: prerun
@@ -39,7 +39,7 @@ jobs:
         with:
           ref: ${{ github.head_ref }}
           path: project-repo
-          
+
       - name: Python Executable Deploy
         if: steps.prerun.outputs.result == 'pex-deploy'
         uses: dagster-io/dagster-cloud-action/actions/[email protected]
@@ -75,4 +75,3 @@ jobs:
           organization_id: ${{ secrets.ORGANIZATION_ID }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-

.github/workflows/deploy.yml

@@ -4,7 +4,7 @@ on:
     branches:
       - "main"
       - "master"
-    
+
 concurrency:
   # Cancel in-progress deploys to same branch
   group: ${{ github.ref }}/deploy
@@ -22,7 +22,7 @@ jobs:
     runs-on: ubuntu-20.04
     outputs:
       build_info: ${{ steps.parse-workspace.outputs.build_info }}
-    
+
     steps:
       - name: Prerun Checks
         id: prerun
@@ -41,7 +41,7 @@ jobs:
         with:
           ref: ${{ github.head_ref }}
           path: project-repo
-          
+
       - name: Python Executable Deploy
         if: steps.prerun.outputs.result == 'pex-deploy'
         uses: dagster-io/dagster-cloud-action/actions/[email protected]
@@ -77,4 +77,3 @@ jobs:
           organization_id: ${{ secrets.ORGANIZATION_ID }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-

.github/workflows/pre-commit.yaml

@@ -0,0 +1,35 @@
+name: pre-commit
+
+on:
+  workflow_call:
+  pull_request_target:
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+      # in case of PR, check out the PR's head branch
+      - uses: actions/checkout@v3
+        if: github.event_name == 'pull_request_target'
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      # in case of push, check out the main branch
+      - uses: actions/checkout@v3
+        if: github.event_name != 'pull_request_target'
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+          cache-dependency-path: "**/requirements*.txt"
+      - uses: pre-commit/[email protected]
+      - name: Post PR comment on failure
+        if: failure() && github.event_name == 'pull_request_target'
+        uses: peter-evans/create-or-update-comment@v2
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          body: |
+            :x: **pre-commit** failed.
+            Please run `pre-commit run --all-files` locally and commit the changes.
+            Find more information in the repository's CONTRIBUTING.md

.gitignore

@@ -176,4 +176,4 @@ models/*
 
 # tmp
 tmp
-tmpdata
+tmpdata

.streamlit/config.toml

@@ -2,4 +2,4 @@
 primaryColor="#ff4b4b"
 backgroundColor="#0e1117"
 secondaryBackgroundColor="#262730"
-textColor="#fafafa"
+textColor="#fafafa"

.vscode/settings.json

@@ -8,4 +8,4 @@
         "llmalert",
         "openai"
     ]
-}
+}

Dockerfile.anomstack_code

@@ -17,4 +17,4 @@ EXPOSE 4000
 
 # CMD allows this to be overridden from run launchers or executors that want
 # to run other commands against your repository
-CMD ["dagster", "api", "grpc", "-h", "0.0.0.0", "-p", "4000", "-f", "anomstack/main.py"]
+CMD ["dagster", "api", "grpc", "-h", "0.0.0.0", "-p", "4000", "-f", "anomstack/main.py"]

Dockerfile.anomstack_dashboard

@@ -25,4 +25,4 @@ COPY .streamlit /opt/dagster/app/.streamlit
 
 EXPOSE 8501
 
-ENTRYPOINT ["streamlit", "run", "/opt/dagster/app/dashboard.py", "--server.port=8501", "--server.address=0.0.0.0"]
+ENTRYPOINT ["streamlit", "run", "/opt/dagster/app/dashboard.py", "--server.port=8501", "--server.address=0.0.0.0"]

Dockerfile.dagster

@@ -22,4 +22,4 @@ COPY dagster_docker.yaml $DAGSTER_HOME/dagster.yaml
 
 COPY workspace.yaml $DAGSTER_HOME
 
-WORKDIR $DAGSTER_HOME
+WORKDIR $DAGSTER_HOME

Makefile

@@ -79,5 +79,3 @@ show-help:
 		printf "\n"; \
 	}' \
 	| more $(shell test $(shell uname) == Darwin && echo '--no-init --raw-control-chars')
-
-

anomstack/alerts/asciiart.py

@@ -1,13 +1,15 @@
 from __future__ import unicode_literals
 
+import copy
+import re
+import sys
+
+import numpy as np
+import pandas as pd
+
 # copied from: https://raw.githubusercontent.com/kakwa/py-ascii-graph/09ca5901be94ec3563bdcc25d6396e18fd8ca5df/ascii_graph/__init__.py
 # copied from: https://raw.githubusercontent.com/nyurik/py-ascii-graph/fix-python310/ascii_graph/__init__.py
 
-import pandas as pd
-import numpy as np
-import sys
-import re
-import copy
 
 if (
     sys.version < "3"
@@ -515,7 +517,9 @@ def make_alert_message(
     df_alert_metric = df_alert_metric.sort_values(
         by="metric_timestamp", ascending=False
     ).dropna()
-    df_alert_metric["metric_timestamp"] = pd.to_datetime(df_alert_metric["metric_timestamp"])
+    df_alert_metric["metric_timestamp"] = pd.to_datetime(
+        df_alert_metric["metric_timestamp"]
+    )
     x = df_alert_metric["metric_value"].round(2).values.tolist()
     metric_batch = df_alert_metric["metric_batch"].unique()[0]
     metric_name = df_alert_metric["metric_name"].unique()[0]

anomstack/alerts/email.py

@@ -5,14 +5,16 @@
 import os
 import smtplib
 import ssl
+import tempfile
+from email import encoders
+from email.mime.base import MIMEBase
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
-from email.mime.base import MIMEBase
-from email import encoders
-import tempfile
-from anomstack.plots.plot import make_alert_plot
+
 from dagster import get_dagster_logger
 
+from anomstack.plots.plot import make_alert_plot
+
 
 def send_email_with_plot(
     df, metric_name, subject, body, attachment_name, threshold=0.8

anomstack/alerts/send.py

@@ -2,11 +2,12 @@
 Helper functions to send alerts.
 """
 
-from dagster import get_dagster_logger
 import pandas as pd
+from dagster import get_dagster_logger
+
 from anomstack.alerts.asciiart import make_alert_message
-from anomstack.alerts.slack import send_alert_slack
 from anomstack.alerts.email import send_email_with_plot
+from anomstack.alerts.slack import send_alert_slack
 
 
 def send_alert(

anomstack/alerts/slack.py

@@ -2,9 +2,10 @@
 Helper functions for sending alerts via Slack.
 """
 
-import requests
 import json
 import os
+
+import requests
 from dagster import get_dagster_logger
 
 

anomstack/config.py

@@ -3,9 +3,10 @@
 """
 
 import os
-import yaml
 from pathlib import Path
 
+import yaml
+
 # environment variables that can be used to override the configuration.
 env_vars = ["ANOMSTACK_GCP_PROJECT_ID", "ANOMSTACK_MODEL_PATH", "ANOMSTACK_TABLE_KEY"]
 
@@ -28,7 +29,7 @@ def process_yaml_file(yaml_file):
     None
     """
 
-    with open(yaml_file, "r", encoding='utf-8') as f:
+    with open(yaml_file, "r", encoding="utf-8") as f:
         metric_specs = yaml.safe_load(f)
         metric_batch = metric_specs["metric_batch"]
         merged_specs = {**defaults, **metric_specs}
@@ -45,7 +46,7 @@ def process_yaml_file(yaml_file):
 
 
 # load defaults
-with open(defaults_dir / "defaults.yaml", "r", encoding='utf-8') as file:
+with open(defaults_dir / "defaults.yaml", "r", encoding="utf-8") as file:
     defaults = yaml.safe_load(file)
 
 # load all the YAML files

anomstack/df/resample.py

@@ -2,14 +2,13 @@
 
 
 def resample(df, freq, freq_agg):
-
     logger = get_dagster_logger()
 
-    df = df.set_index('metric_timestamp')
-    if freq_agg == 'mean':
-        df = df.groupby(['metric_batch', 'metric_name']).resample(freq).mean()
-    elif freq_agg == 'sum':
-        df = df.groupby(['metric_batch', 'metric_name']).resample(freq).sum()
+    df = df.set_index("metric_timestamp")
+    if freq_agg == "mean":
+        df = df.groupby(["metric_batch", "metric_name"]).resample(freq).mean()
+    elif freq_agg == "sum":
+        df = df.groupby(["metric_batch", "metric_name"]).resample(freq).sum()
     else:
         raise ValueError(f"Unsupported aggregation method: {freq_agg}")
     df = df.reset_index()

anomstack/df/save.py

@@ -6,12 +6,13 @@
 """
 
 import pandas as pd
-from anomstack.external.gcp.bigquery import save_df_bigquery
+
 from anomstack.external.duckdb.duckdb import save_df_duckdb
+from anomstack.external.gcp.bigquery import save_df_bigquery
 from anomstack.external.snowflake.snowflake import save_df_snowflake
 
 
-def save_df(df, db, table_key, if_exists='append') -> pd.DataFrame:
+def save_df(df, db, table_key, if_exists="append") -> pd.DataFrame:
     """
     Save a Pandas DataFrame to a database.
 
@@ -24,13 +25,13 @@ def save_df(df, db, table_key, if_exists='append') -> pd.DataFrame:
     Returns:
     - The Pandas DataFrame that was saved.
     """
-    if db=='bigquery':
+    if db == "bigquery":
         df = save_df_bigquery(df, table_key, if_exists)
-    elif db=='snowflake':
+    elif db == "snowflake":
         df = save_df_snowflake(df, table_key)
-    elif db=='duckdb':
+    elif db == "duckdb":
         df = save_df_duckdb(df, table_key)
     else:
-        raise ValueError(f'Unknown db: {db}')
+        raise ValueError(f"Unknown db: {db}")
 
     return df

anomstack/df/wrangle.py

@@ -28,7 +28,9 @@ def wrangle_df(df: pd.DataFrame, rounding: int = 4) -> pd.DataFrame:
 
     # if we have any nan metric_values then drop them and log how many nan rows we dropped
     if df["metric_value"].isnull().sum() > 0:
-        logger.warning(f"dropping {df['metric_value'].isnull().sum()} nan metric_value rows")
+        logger.warning(
+            f"dropping {df['metric_value'].isnull().sum()} nan metric_value rows"
+        )
         df = df[~df["metric_value"].isnull()]
 
     # round metric_value

anomstack/external/aws/credentials.py

@@ -2,13 +2,12 @@
 
 
 def get_aws_credentials():
-
-    aws_access_key_id = os.getenv('ANOMSTACK_AWS_ACCESS_KEY_ID')
-    aws_secret_access_key = os.getenv('ANOMSTACK_AWS_SECRET_ACCESS_KEY')
+    aws_access_key_id = os.getenv("ANOMSTACK_AWS_ACCESS_KEY_ID")
+    aws_secret_access_key = os.getenv("ANOMSTACK_AWS_SECRET_ACCESS_KEY")
 
     aws_credentials = {
-        'aws_access_key_id': aws_access_key_id,
-        'aws_secret_access_key': aws_secret_access_key
+        "aws_access_key_id": aws_access_key_id,
+        "aws_secret_access_key": aws_secret_access_key,
     }
 
     return aws_credentials