Add Sentry as supported incident system.

bq-workers/sentry-parser/Dockerfile

@@ -0,0 +1,39 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Use the official Python image.
+# https://hub.docker.com/_/python
+FROM python:3.10
+
+# Allow statements and log messages to immediately appear in the Cloud Run logs
+ENV PYTHONUNBUFFERED True
+
+# Copy application dependency manifests to the container image.
+# Copying this separately prevents re-running pip install on every code change.
+COPY requirements.txt .
+
+# Install production dependencies.
+RUN pip install -r requirements.txt
+
+# Copy local code to the container image.
+ENV APP_HOME /app
+WORKDIR $APP_HOME
+COPY . .
+
+# Run the web service on container startup.
+# Use gunicorn webserver with one worker process and 8 threads.
+# For environments with multiple CPU cores, increase the number of workers
+# to be equal to the cores available.
+CMD exec gunicorn --bind :$PORT --workers 1 --threads 8 --timeout 0 main:app

bq-workers/sentry-parser/cloudbuild.yaml

@@ -0,0 +1,40 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+steps:
+- # Build sentry-parser image
+  name: gcr.io/cloud-builders/docker:latest
+  args: ['build', 
+         '--tag=gcr.io/$PROJECT_ID/sentry-parser:${_TAG}', '.']
+  id: build
+
+- # Push the container image to Artifact Registry
+  name: gcr.io/cloud-builders/docker
+  args: ['push', 'gcr.io/$PROJECT_ID/sentry-parser:${_TAG}']
+  waitFor: build
+  id: push
+
+- # Deploy to Cloud Run
+  name: google/cloud-sdk
+  args: ['gcloud', 'run', 'deploy', 'sentry-parser',
+         '--image', 'gcr.io/$PROJECT_ID/sentry-parser:${_TAG}',
+         '--region', '${_REGION}',
+         '--platform', 'managed'
+  ]
+  id: deploy
+  waitFor: push
+
+images: [
+  'gcr.io/$PROJECT_ID/sentry-parser:${_TAG}'
+]

bq-workers/sentry-parser/main.py

@@ -0,0 +1,93 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import os
+import json
+
+import shared
+
+from flask import Flask, request
+
+app = Flask(__name__)
+
+
+@app.route("/", methods=["POST"])
+def index():
+    """
+    Receives messages from a push subscription from Pub/Sub.
+    Parses the message, and inserts it into BigQuery.
+    """
+    event = None
+    # Check request for JSON
+    if not request.is_json:
+        raise Exception("Expecting JSON payload")
+    envelope = request.get_json()
+
+    # Check that message is a valid pub/sub message
+    if "message" not in envelope:
+        raise Exception("Not a valid Pub/Sub Message")
+    msg = envelope["message"]
+
+    if "attributes" not in msg:
+        raise Exception("Missing pubsub attributes")
+
+    try:
+        attr = msg["attributes"]
+
+        if "headers" in attr:
+            headers = json.loads(attr["headers"])
+
+            if "Sentry-Hook-Resource" in headers:
+                event = process_sentry_event(headers, msg)
+
+        shared.insert_row_into_bigquery(event)
+
+    except Exception as e:
+        entry = {
+            "severity": "WARNING",
+            "msg": "Data not saved to BigQuery",
+            "errors": str(e),
+            "json_payload": envelope
+        }
+        print(json.dumps(entry))
+
+    return "", 204
+
+
+def process_sentry_event(headers, msg):
+    event_type = headers["Sentry-Hook-Resource"]
+    metadata = json.loads(base64.b64decode(msg["data"]).decode("utf-8").strip())
+    types = {"issue"}
+
+    if event_type not in types:
+        raise Exception("Unsupported Sentry event: '%s'" % event_type)
+
+    return {
+        "event_type": event_type,
+        "id": metadata["data"]["issue"]["id"],
+        "metadata": json.dumps(metadata),
+        "time_created": headers["Sentry-Hook-Timestamp"],
+        "signature": headers["Sentry-Hook-Signature"],
+        "msg_id": msg["message_id"],
+        "source": "sentry",
+    }
+
+
+if __name__ == "__main__":
+    PORT = int(os.getenv("PORT")) if os.getenv("PORT") else 8080
+
+    # This is used when running locally. Gunicorn is used to run the
+    # application on Cloud Run. See entrypoint in Dockerfile.
+    app.run(host="127.0.0.1", port=PORT, debug=True)

bq-workers/sentry-parser/main_test.py

@@ -0,0 +1,190 @@
+# Copyright 2020 Google, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import json
+
+import main
+import shared
+
+import mock
+import pytest
+
+
+@pytest.fixture
+def client():
+    main.app.testing = True
+    return main.app.test_client()
+
+
+def test_not_json(client):
+    with pytest.raises(Exception) as e:
+        client.post("/", data="foo")
+
+    assert "Expecting JSON payload" in str(e.value)
+
+
+def test_not_pubsub_message(client):
+    with pytest.raises(Exception) as e:
+        client.post(
+            "/",
+            data=json.dumps({"foo": "bar"}),
+            headers={"Content-Type": "application/json"},
+        )
+
+    assert "Not a valid Pub/Sub Message" in str(e.value)
+
+
+def test_missing_msg_attributes(client):
+    with pytest.raises(Exception) as e:
+        client.post(
+            "/",
+            data=json.dumps({"message": "bar"}),
+            headers={"Content-Type": "application/json"},
+        )
+
+    assert "Missing pubsub attributes" in str(e.value)
+
+
+def test_sentry_event_processed(client):
+    data = json.dumps({
+        "action": "created",
+        "installation": {
+            "uuid": "95cc9015-1456-4656-83fb-df3fa930348e"
+        },
+        "data": {
+            "issue": {
+                "id": "3359227809",
+                "shareId": None,
+                "shortId": "PROJECT-NAME",
+                "title": "Your exception: Cannot continue.",
+                "culprit": "com.google.common.base.Preconditions in checkState",
+                "permalink": None,
+                "logger": "com.logger.name",
+                "level": "warning",
+                "status": "unresolved",
+                "statusDetails": {},
+                "isPublic": False,
+                "platform": "rust",
+                "project": {
+                    "id": "5303212330",
+                    "name": "name",
+                    "slug": "name",
+                    "platform": "rust"
+                },
+                "type": "error",
+                "metadata": {
+                    "value": "Cannot continue.",
+                    "type": "YourException",
+                    "filename": "Preconditions.java",
+                    "function": "checkState",
+                    "display_title_with_tree_label": False
+                },
+                "numComments": 0,
+                "assignedTo": None,
+                "isBookmarked": False,
+                "isSubscribed": False,
+                "subscriptionDetails": None,
+                "hasSeen": False,
+                "annotations": [],
+                "isUnhandled": False,
+                "count": "1",
+                "userCount": 0,
+                "firstSeen": "2022-06-01T00:50:02.780000Z",
+                "lastSeen": "2022-06-18T13:50:02.780000Z"
+            }
+        },
+        "actor": {
+            "type": "application",
+            "id": "sentry",
+            "name": "Sentry"
+        }
+    }).encode("utf-8")
+    pubsub_msg = {
+        "message": {
+            "data": base64.b64encode(data).decode("utf-8"),
+            "attributes": {
+                "headers": json.dumps({
+                    "Content-Type": "application/json",
+                    "Request-ID": "ef0dd634-e830-4aa4-a690-5fca54a11a18",
+                    "Sentry-Hook-Resource": "issue",
+                    "Sentry-Hook-Timestamp": "2022-06-20 12:49:49 UTC",
+                    "Sentry-Hook-Signature": "<generated_signature>"
+                })
+            },
+            "message_id": "foobar",
+        },
+    }
+
+    event = {
+        "event_type": "issue",
+        "id": "3359227809",
+        "metadata": data.decode("utf-8"),
+        "time_created": "2022-06-20 12:49:49 UTC",
+        "signature": "<generated_signature>",
+        "msg_id": "foobar",
+        "source": "sentry",
+    }
+
+    shared.insert_row_into_bigquery = mock.MagicMock()
+
+    r = client.post(
+        "/",
+        data=json.dumps(pubsub_msg),
+        headers={"Content-Type": "application/json"},
+    )
+
+    shared.insert_row_into_bigquery.assert_called_with(event)
+    assert r.status_code == 204
+
+
+def test_unsupported_sentry_event_processed(client):
+    data = json.dumps({
+        "action": "created",
+        "data": {
+            "comment": "adding a comment",
+            "project_slug": "sentry",
+            "comment_id": 1234,
+            "issue_id": 100,
+            "timestamp": "2022-03-02T21:51:44.118160Z"
+        },
+        "installation": {"uuid": "eac5a0ae-60ec-418f-9318-46dc5e7e52ec"},
+        "actor": {"type": "user", "id": 1, "name": "Rikkert"}
+    }).encode("utf-8")
+    pubsub_msg = {
+        "message": {
+            "data": base64.b64encode(data).decode("utf-8"),
+            "attributes": {
+                "headers": json.dumps({
+                    "Content-Type": "application/json",
+                    "Request-ID": "ef0dd634-e830-4aa4-a690-5fca54a11a18",
+                    "Sentry-Hook-Resource": "comment",
+                    "Sentry-Hook-Timestamp": "2022-06-20 12:49:49 UTC",
+                    "Sentry-Hook-Signature": "<generated_signature>"
+                })
+            },
+            "message_id": "foobar",
+        },
+    }
+
+    shared.insert_row_into_bigquery = mock.MagicMock()
+
+    r = client.post(
+        "/",
+        data=json.dumps(pubsub_msg),
+        headers={"Content-Type": "application/json"},
+    )
+
+    shared.insert_row_into_bigquery.assert_not_called()
+    assert r.status_code == 204

bq-workers/sentry-parser/requirements-test.txt

@@ -0,0 +1,2 @@
+-r requirements.txt
+pytest==7.1.2

bq-workers/sentry-parser/requirements.txt

@@ -0,0 +1,4 @@
+Flask==2.1.2
+gunicorn==20.1.0
+google-cloud-bigquery==3.2.0
+git+https://github.com/GoogleCloudPlatform/fourkeys.git#egg=shared&subdirectory=shared

event_handler/Dockerfile

@@ -15,7 +15,7 @@
 
 # Use the official Python image.
 # https://hub.docker.com/_/python
-FROM python:3.7-slim
+FROM python:3.10-slim
 
 # Allow statements and log messages to immediately appear in the Cloud Run logs
 ENV PYTHONUNBUFFERED True