feat: setup devenv observability

Makefile

@@ -166,10 +166,10 @@ $(EMILY_OPENAPI_SPEC): $(INSTALL_TARGET) $(EMILY_HANDLER_SOURCE_FILES)
 # ----------------------------------------------------
 
 devenv-no-sbtc-up:
-	docker compose -f docker/docker-compose.yml --profile default --profile bitcoin-mempool up
+	docker compose -f docker/docker-compose.yml --profile default --profile bitcoin-mempool --profile observability up
 
 devenv-no-sbtc-down:
-	docker compose -f docker/docker-compose.yml --profile default --profile bitcoin-mempool down
+	docker compose -f docker/docker-compose.yml --profile default --profile bitcoin-mempool --profile observability down
 
 devenv-up:
 	docker compose -f docker/docker-compose.yml --profile default --profile bitcoin-mempool --profile sbtc-signer up --detach

docker/docker-compose.yml

@@ -30,6 +30,10 @@ x-common-vars:
   - &REWARD_RECIPIENT ${REWARD_RECIPIENT:-STQM73RQC4EX0A07KWG1J5ECZJYBZS4SJ4ERC6WN} # priv: 6ad9cadb42d4edbfbe0c5bfb3b8a4125ddced021c4174f829b714ccbf527f02001
   - &EXIT_FROM_MONITOR 1 # set to "1" to automatically shut down via monitor.ts
 
+volumes:
+  # prometheus-data: {}
+  grafana-data1: {}
+
 # Templates.
 # ------------------------------------------------------------------------------
 x-postgres: &postgres
@@ -610,3 +614,54 @@ services:
     command: "./wait-for-it.sh mariadb:3306 --timeout=720 --strict -- ./start.sh"
     profiles:
       - bitcoin-mempool
+
+  prometheus:
+    image: prom/prometheus:v2.45.2
+    restart: always
+    volumes:
+      - ./local-prometheus.yml:/etc/prometheus/prometheus.yml
+      # - prometheus-data:/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
+      - '--web.console.templates=/usr/share/prometheus/consoles'
+    ports:
+      - 9090:9090
+    depends_on:
+      - vector
+    profiles:
+      - observability
+
+  grafana:
+    image: grafana/grafana:10.2.3-ubuntu
+    user: '472'
+    restart: always
+    environment:
+      GF_INSTALL_PLUGINS: 'grafana-clock-panel,grafana-simple-json-datasource'
+      GF_SECURITY_ADMIN_USER: admin
+      GF_SECURITY_ADMIN_PASSWORD: admin
+      GF_USERS_ALLOW_SIGN_UP: 'false'
+    volumes:
+      - grafana-data1:/var/lib/grafana
+      - ./grafana/provisioning/:/etc/grafana/provisioning/
+    ports:
+      - 3030:3000
+    depends_on:
+      - prometheus
+    profiles:
+      - observability
+
+  vector:
+    image: timberio/vector:0.34.2-distroless-static
+    command: --config /etc/vector/vector.toml
+    restart: always
+    ports:
+      - 9598
+    volumes: 
+      - ./vector.toml:/etc/vector/vector.toml:ro
+    # This is needed for localhost networking
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    profiles:
+      - observability

docker/grafana/dashboards/dashboard.yml

@@ -0,0 +1,11 @@
+apiVersion: 1
+
+providers:
+- name: 'Prometheus'
+  orgId: 1
+  folder: ''
+  type: file
+  disableDeletion: false
+  editable: true
+  options:
+    path: /etc/grafana/provisioning/dashboards

docker/grafana/datasources/datasource.yml

@@ -0,0 +1,50 @@
+# config file version
+apiVersion: 1
+
+# list of datasources that should be deleted from the database
+deleteDatasources:
+  - name: Prometheus
+    orgId: 1
+
+# list of datasources to insert/update depending
+# whats available in the database
+datasources:
+  # <string, required> name of the datasource. Required
+- name: Prometheus
+  # <string, required> datasource type. Required
+  type: prometheus
+  # <string, required> access mode. direct or proxy. Required
+  access: proxy
+  # <int> org id. will default to orgId 1 if not specified
+  orgId: 1
+  # <string> url
+  url: http://prometheus:9090
+  # <string> database password, if used
+  password:
+  # <string> database user, if used
+  user:
+  # <string> database name, if used
+  database:
+  # <bool> enable/disable basic auth
+  basicAuth: false
+  # <string> basic auth username, if used
+  basicAuthUser:
+  # <string> basic auth password, if used
+  basicAuthPassword:
+  # <bool> enable/disable with credentials headers
+  withCredentials:
+  # <bool> mark as default datasource. Max one per org
+  isDefault: true
+  # <map> fields that will be converted to json and stored in json_data
+  jsonData:
+     graphiteVersion: "1.1"
+     tlsAuth: false
+     tlsAuthWithCACert: false
+  # <string> json object of data that will be encrypted.
+  secureJsonData:
+    tlsCACert: "..."
+    tlsClientCert: "..."
+    tlsClientKey: "..."
+  version: 1
+  # <bool> allow users to edit datasources from the UI.
+  editable: true

docker/local-prometheus.yml

@@ -0,0 +1,18 @@
+global:
+  scrape_interval:     15s # By default, scrape targets every 15 seconds.
+  evaluation_interval: 15s # By default, evaluate targets every 15 seconds.
+
+rule_files:
+  # - "first.rules"
+  # - "second.rules"
+
+scrape_configs:
+  - job_name: prometheus
+    scrape_interval: 5s
+    static_configs:
+      - targets: ['localhost:9090']
+
+  - job_name: vector
+    scrape_interval: 5s
+    static_configs:
+      - targets: ['vector:9598']

docker/vector.toml

@@ -0,0 +1,27 @@
+## Prometheus Host metrics
+
+[sources.host_metrics]
+type = "host_metrics"
+collectors = ["cpu", "disk", "load", "filesystem", "host", "memory", "network"]
+scrape_interval_secs = 15
+
+[sources.host_metrics.filesystem.mountpoints]
+includes = ["/"]
+
+[sources.host_metrics.disk.devices]
+includes = ["/dev/root", "nvme*", "tmpfs"]
+
+[sources.app]
+type = "prometheus_scrape"
+endpoints = [
+    "http://host.docker.internal:9181",
+    "http://host.docker.internal:9182",
+    "http://host.docker.internal:9183",
+]
+
+## App Metric Sinks
+
+[sinks.host_metrics_sink]
+type = "prometheus_exporter"
+inputs = ["host_metrics", "app"]
+address = "0.0.0.0:9598"

envs/signer-1.env

@@ -12,6 +12,7 @@ export SIGNER_SIGNER__SIGNER_ROUND_MAX_DURATION=5
 export SIGNER_SIGNER__BITCOIN_PRESIGN_REQUEST_MAX_DURATION=5
 export SIGNER_SIGNER__DKG_MAX_DURATION=5
 export SIGNER_SIGNER__DKG_BEGIN_PAUSE=1
+export SIGNER_SIGNER__PROMETHEUS_EXPORTER_ENDPOINT=0.0.0.0:9181
 
 export SIGNER_SIGNER__BOOTSTRAP_SIGNING_SET=035249137286c077ccee65ecc43e724b9b9e5a588e3d7f51e3b62f9624c2a49e46,031a4d9f4903da97498945a4e01a5023a1d53bc96ad670bfe03adf8a06c52e6380,02007311430123d4cad97f4f7e86e023b28143130a18099ecf094d36fef0f6135c
 

envs/signer-2.env

@@ -12,6 +12,7 @@ export SIGNER_SIGNER__SIGNER_ROUND_MAX_DURATION=5
 export SIGNER_SIGNER__BITCOIN_PRESIGN_REQUEST_MAX_DURATION=5
 export SIGNER_SIGNER__DKG_MAX_DURATION=5
 export SIGNER_SIGNER__DKG_BEGIN_PAUSE=1
+export SIGNER_SIGNER__PROMETHEUS_EXPORTER_ENDPOINT=0.0.0.0:9182
 
 export SIGNER_SIGNER__BOOTSTRAP_SIGNING_SET=035249137286c077ccee65ecc43e724b9b9e5a588e3d7f51e3b62f9624c2a49e46,031a4d9f4903da97498945a4e01a5023a1d53bc96ad670bfe03adf8a06c52e6380,02007311430123d4cad97f4f7e86e023b28143130a18099ecf094d36fef0f6135c
 

envs/signer-3.env

@@ -12,6 +12,7 @@ export SIGNER_SIGNER__SIGNER_ROUND_MAX_DURATION=5
 export SIGNER_SIGNER__BITCOIN_PRESIGN_REQUEST_MAX_DURATION=5
 export SIGNER_SIGNER__DKG_MAX_DURATION=5
 export SIGNER_SIGNER__DKG_BEGIN_PAUSE=1
+export SIGNER_SIGNER__PROMETHEUS_EXPORTER_ENDPOINT=0.0.0.0:9183
 
 export SIGNER_SIGNER__BOOTSTRAP_SIGNING_SET=035249137286c077ccee65ecc43e724b9b9e5a588e3d7f51e3b62f9624c2a49e46,031a4d9f4903da97498945a4e01a5023a1d53bc96ad670bfe03adf8a06c52e6380,02007311430123d4cad97f4f7e86e023b28143130a18099ecf094d36fef0f6135c
 

signers.sh

@@ -67,7 +67,7 @@ exec_run() {
       . "$ENV_PATH/signer-$i.env" \
       && export RUST_LOG="$LOG_SETTINGS" \
       && export SIGNER_SIGNER__BOOTSTRAP_SIGNING_SET="$BOOTSTRAP_SIGNER_SET" \
-      && cargo run --bin signer -- --config "$SIGNER_CONFIG" --migrate-db > "$PWD/target/signer-$i.log" 2>&1 \
+      && cargo run --bin signer -- --config "$SIGNER_CONFIG" --migrate-db -o json > "$PWD/target/signer-$i.log" 2>&1 \
       &
     )
     i=$((i + 1))