faucetsdn · noursaidi · Jan 14, 2025
diff --git a/misc/automated_sequencer/README.md b/misc/automated_sequencer/README.md
@@ -0,0 +1,113 @@
+# Automated Sequencer Reporting
+
+Automated sequencer running and reporting using GKE, GCS and Pub/Sub
+
+- `sequencer` is a utility which:
+    - clones the UDMI repository
+    - builds sequencer
+    - runs sequencer against a given device
+    - uploads results to GCS
+    - publishes a Pub/Sub message to 
+- `reporter` is a utility which:
+    - receieves a Pub/Sub message
+    - downloads results from GCS
+    - generates a report
+    - uploads report to GCS
+
+## Example Kubernetes Deployments
+
+### Reporter
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: reporter
+  namespace: sequencer
+  labels:
+    app: reporter
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: reporter
+  template:
+    metadata:
+      labels:
+        app: reporter
+    spec:
+      containers:
+      - name: reporter
+        image: reporter
+        env:
+        - name: PUBSUB
+          value: "udmi-sequencer-notification"
+        - name: GCS_BUCKET
+          value: "gcs-bucket-id"
+        - name: PROJECT_ID
+          value: "gcp-project-id"
+```
+
+### Sequencer
+
+```yaml
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: sequencer-ddc-8
+  namespace: sequencer
+spec:
+  concurrencyPolicy: Forbid
+  schedule: "36 1 * * *"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: sequencer
+            image: sequencer
+            resources:
+              requests:
+                cpu: 1
+                memory: 1000Mi
+            volumeMounts:
+            - name: tmp
+              mountPath: /tmp
+            env:
+            - name: PROJECT_ID
+              value: "gcp-project-id"
+            - name: SEQUENCER_PROJECT_ID
+              value: "//gref/gcp-project-id+ddc-8"
+            - name: SITE_PROJECT
+              value: "gcp-project-id"
+            - name: SITE_NAME_REPO
+              value: "UK-LON-GLAB"
+            - name: BRANCH
+              value: "main"
+            - name: DEVICE_ID
+              value: "DDC-8"
+            - name: GCS_BUCKET
+              value: "gcs-bucket-id"
+            - name: SITE_MODEL_SUBDIR
+              value: "udmi"
+            - name: PUBSUB
+              value: "udmi-sequencer-notification"
+            - name: PUBSUB_PROJECT
+              value: "gcp-project-id"
+            - name: UDMI_BRANCH
+              valueFrom:
+                configMapKeyRef:
+                  name: data
+                  key: udmi_branch
+            - name: UDMI_REPO
+              valueFrom:
+                configMapKeyRef:
+                  name: data
+                  key: udmi_repo
+          restartPolicy: Never
+          volumes:
+          - name: tmp
+            emptyDir:
+              medium: Memory
+
+```
diff --git a/misc/automated_sequencer/reporter/Dockerfile b/misc/automated_sequencer/reporter/Dockerfile
@@ -0,0 +1,16 @@
+FROM eu.gcr.io/gke-release/debian-base:latest
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Fix for installing java but we also need bash anyway as a dependancy of the UDMI tooling
+# https://groups.google.com/g/linux.debian.bugs.dist/c/2zq4rjzg7Fs
+RUN apt update && apt install -y --allow-change-held-packages python3 python3-pip python3-venv bash curl gnupg libcap2 hxtools
+
+WORKDIR /
+
+COPY / /
+RUN apt update && apt install -y jq
+RUN pip install --no-cache-dir -r /python/requirements.txt
+RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg  add - && apt-get update -y && apt-get install google-cloud-cli -y
+
+ENTRYPOINT [ "/listener.sh" ]
diff --git a/misc/automated_sequencer/reporter/build.sh b/misc/automated_sequencer/reporter/build.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+tag=latest
+image_name=autosequencer-reporter
+PROJECT_ID=bos-platform-artifacts
+REGISTRY=us-central1-docker.pkg.dev/$PROJECT_ID/udmi
+
+docker build --no-cache -t $image_name:$tag .
+docker tag $image_name:$tag $REGISTRY/$image_name:$tag
+docker push $REGISTRY/$image_name:$tag
diff --git a/misc/automated_sequencer/reporter/config.sh b/misc/automated_sequencer/reporter/config.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+## Fixed Config
+######
+
+# Name of run information (timestamp, UDMI version)
+export VERSION_FILE_NAME=version.json
+
+# Directory to put summary (result.log and run information) into
+export GCS_SUMMARY_SUBDIR=summary
+
+# Directory to put gzipped results into
+export GCS_RESULTS_SUBDIR=results
+
+# Subscription to receieve notifications of completed run
+export NOTIFICATION_SUBSCRIPTION=sequencer_notification
+
+# Subscription to PUBLISH notifications of completed run
+export NOTIFICATION_TOPIC=sequencer_notification
+
+# directory to save results downloaded from GCS into (results deleted before each run)
+export LOCAL_RESULTS_DIR=results
+
+# Name of device report (saved in local device folder)
+export DEVICE_REPORT_NAME=report.html
+
+# Name of master report (saved in root of GCS bucket)
+export MASTER_REPORT_NAME=index.html
+
diff --git a/misc/automated_sequencer/reporter/generate_device_report b/misc/automated_sequencer/reporter/generate_device_report
@@ -0,0 +1,26 @@
+#!/bin/bash -e
+ROOT_DIR=$(realpath $(dirname $0))
+
+# FIXED CONFIG VARIABLES
+source config.sh
+
+# ARGS
+if [[ $# -ne 3 ]]; then
+    echo $0 GCS_BUCKET DEVICE_PATH REPORT_PATH
+    exit 1
+fi
+
+GCS_BUCKET=$1
+DEVICE_PATH=$2
+REPORT_PATH=$3
+
+rm -rf $LOCAL_RESULTS_DIR/$DEVICE_PATH
+mkdir -p $LOCAL_RESULTS_DIR/$DEVICE_PATH
+
+# /.. because gsutil will copy into /device_id/device_id
+gsutil -m cp -r gs://$GCS_BUCKET/$GCS_SUMMARY_SUBDIR/$DEVICE_PATH $LOCAL_RESULTS_DIR/$DEVICE_PATH/..
+
+python3 $ROOT_DIR/python/generate_report.py \
+    $LOCAL_RESULTS_DIR/$DEVICE_PATH $DEVICE_PATH $REPORT_PATH
+
+echo done generating device report
diff --git a/misc/automated_sequencer/reporter/index_template.html b/misc/automated_sequencer/reporter/index_template.html
@@ -0,0 +1,23 @@
+<!doctype html>
+<html>
+<head>
+    <link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-rbsA2VBKQhggwzxH7pPCaAqO46MgnOM80zW1RWuH61DGLwZJEdK2Kadq2F9CUG65" crossorigin="anonymous">
+    <style type="text/css">
+
+    </style>
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/font/bootstrap-icons.css">
+    <script src="https://code.jquery.com/jquery-3.7.0.slim.min.js" crossorigin="anonymous"></script>
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js" crossorigin="anonymous"></script>
+    <script>
+        $(function () {
+            $('[data-bs-toggle="tooltip"]').tooltip({
+                container: 'body'
+        })
+        })
+    </script>
+</head>
+<body>
+    <!--START_test-->
+    <!-- INSERT_HERE -->
+</body>
+</html>
diff --git a/misc/automated_sequencer/reporter/listener.sh b/misc/automated_sequencer/reporter/listener.sh
@@ -0,0 +1,27 @@
+#!/bin/bash -ex
+ROOT_DIR=$(realpath $(dirname $0))
+echo listening
+source $ROOT_DIR/config.sh
+while true
+do
+	pubsub=$(gcloud pubsub subscriptions pull $PUBSUB --limit=1 --format=json --project=$PROJECT_ID)
+    if [[ $pubsub != '[]' ]]; then
+        ACK_ID=$(echo $pubsub | jq -r '.[0].ackId')
+        PAYLOAD=$(echo $pubsub | jq -r '.[0].message.data' | base64 -d)
+
+        DEVICE_PATH=$(echo $PAYLOAD | jq -r '.device_path')
+        echo "processing $DEVICE_PATH"
+
+        $ROOT_DIR/generate_device_report $GCS_BUCKET $DEVICE_PATH report.html
+        # Upload report
+        gsutil cp report.html gs://$GCS_BUCKET/$GCS_SUMMARY_SUBDIR/$DEVICE_PATH/report.html
+
+        # Generate master report
+        gsutil cp gs://$GCS_BUCKET/$MASTER_REPORT_NAME index.html || cp $ROOT_DIR/index_template.html index.html
+        python3 $ROOT_DIR/python/merge_master.py $MASTER_REPORT_NAME $DEVICE_REPORT_NAME
+        gsutil cp index.html gs://$GCS_BUCKET/$MASTER_REPORT_NAME
+
+        gcloud pubsub subscriptions ack $PUBSUB --ack-ids=$ACK_ID --project=$PROJECT_ID
+    fi
+    sleep 1
+done
diff --git a/misc/automated_sequencer/reporter/python/device_runs.py b/misc/automated_sequencer/reporter/python/device_runs.py
@@ -0,0 +1,59 @@
+import itertools
+import json
+import os
+from pathlib import Path
+from run_results import RunResult
+from test_result import TestResult
+
+METADATA_FILE = 'version.json'
+NO_RESULT = ''
+
+
+class DeviceRuns:
+  """All sequencer runs for a given device loaded from directory"""
+
+  def __init__(self, results_dir):
+    """Load results
+
+    Arguments: results_dir path to directory of results, where each directory
+    must be named by a timestamp
+    """
+    self.run_dates = []
+    self.encountered_test = []  # list of all tests encountered
+    self.test_matrix = {}
+    self.runs = {}
+
+    self.results_dir = results_dir
+    self.load_results(results_dir)
+    self.create_test_matrix()
+
+  def load_results(self, path):
+    """Load results from given path to directory of results direcotires
+
+    IMPORTANT! This method must load the results in the correct order!
+    """
+    results_dir = Path(path)
+    results_subdirs = [int(x.stem) for x in results_dir.iterdir() if x.is_dir()]
+    results_subdirs.sort()
+    # Show only latest 30 results to avoid horizontal scroll
+    for rdir in results_subdirs[-30:]:
+      full_path = os.path.join(path, str(rdir))
+      self.run_dates.append(rdir)
+      print(f'{full_path} is full path, {rdir} is rdir')
+      results = RunResult(rdir, full_path)
+      self.runs[f'{rdir}'] = results
+
+  def create_test_matrix(self):
+    # load all unique test names (rows)
+    self.all_results = [
+        x for x in itertools.chain(*[a.results for a in self.runs.values()])
+    ]
+    encountered_tests = set([t.name for t in self.all_results])
+    self.encountered_tests = list(encountered_tests)
+    self.encountered_tests.sort()
+
+    for test in self.encountered_tests:
+      self.test_matrix[test] = []
+      for run in self.runs.values():
+        result = run.tests[test].result if test in run.tests else NO_RESULT
+        self.test_matrix[test].append(result)
diff --git a/misc/automated_sequencer/reporter/python/device_template.html b/misc/automated_sequencer/reporter/python/device_template.html
@@ -0,0 +1,48 @@
+<!doctype html>
+<html>
+<head>
+    <link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-rbsA2VBKQhggwzxH7pPCaAqO46MgnOM80zW1RWuH61DGLwZJEdK2Kadq2F9CUG65" crossorigin="anonymous">
+    <style type="text/css">
+
+    </style>
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/font/bootstrap-icons.css">
+    <script src="https://code.jquery.com/jquery-3.7.0.slim.min.js" crossorigin="anonymous"></script>
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js" crossorigin="anonymous"></script>
+    <script>
+        $(function () {
+            $('[data-bs-toggle="tooltip"]').tooltip({
+                container: 'body'
+        })
+        })
+    </script>
+</head>
+<body>
+    <!-- START: {{device_path}} -->
+    <h1>{{device_path}}</h1>
+    <table id="table" class="table w-auto table-sm table-bordered">
+        <thead>
+            <tr>
+                <th>Test Name</th>
+                {% for run in runs.values() %}
+                    <th class="{{run.udmi_hash|commit_background}}"  data-bs-toggle="tooltip" data-bs-placement="top" data-bs-title="{{run.udmi_hash}}">
+                        {{run.timestamp|header_date}} <a href="https://pantheon.corp.google.com/storage/browser/_details/{{gcs_bucket}}/{{device_path}}/{{run.timestamp}}.tar.gz" class="bi bi-download"></a>
+                    </th>
+                {% endfor %}
+            </tr>
+        </thead>
+        <tbody>
+            {% for test_name, test_results in results.items() %}
+            <tr>
+                <td>{{test_name}}</td>
+                {% for result in test_results %}
+                    <td class="{{result|result_background}}">{{ result }}</td>
+                {% endfor %}
+            </tr>
+            {% endfor %}
+        </tbody>
+
+    </table>
+    <!-- END: {{device_path}} -->
+
+</body>
+</html>