Support triton metrics

Dockerfile

@@ -4,7 +4,7 @@ FROM alpine:3.6
 # artisanally hand-rolling curl and the rest of our stack we'll just use
 # Alpine so we can use `docker build`.
 
-RUN apk add --update curl
+RUN apk add --update curl bash
 
 # add Prometheus. alas, the Prometheus developers provide no checksum
 RUN export PROM_VERSION=1.7.1 \
@@ -22,22 +22,31 @@ RUN export PROM_VERSION=1.7.1 \
     && ln -s /usr/share/prometheus/console_libraries /usr/share/prometheus/consoles/ /etc/prometheus/ \
     && rm /tmp/prometheus-${PROM_VERSION}.linux-amd64.tar.gz
 
-# get consul-template
-RUN curl -Lso /tmp/consul-template_0.14.0_linux_amd64.zip https://releases.hashicorp.com/consul-template/0.14.0/consul-template_0.14.0_linux_amd64.zip \
-    && echo "7c70ea5f230a70c809333e75fdcff2f6f1e838f29cfb872e1420a63cdf7f3a78  /tmp/consul-template_0.14.0_linux_amd64.zip" | sha256sum -c \
-    && unzip /tmp/consul-template_0.14.0_linux_amd64.zip \
-    && mv consul-template /bin \
-    && rm /tmp/consul-template_0.14.0_linux_amd64.zip
-
-# get consul-agent
-RUN export CONSUL_VERSION=0.7.0 \
-    && export CONSUL_CHECKSUM=b350591af10d7d23514ebaa0565638539900cdb3aaa048f077217c4c46653dd8 \
+# Install Consul
+# Releases at https://releases.hashicorp.com/consul
+RUN set -ex \
+    && export CONSUL_VERSION=0.7.5 \
+    && export CONSUL_CHECKSUM=40ce7175535551882ecdff21fdd276cef6eaab96be8a8260e0599fadb6f1f5b8 \
     && curl --retry 7 --fail -vo /tmp/consul.zip "https://releases.hashicorp.com/consul/${CONSUL_VERSION}/consul_${CONSUL_VERSION}_linux_amd64.zip" \
     && echo "${CONSUL_CHECKSUM}  /tmp/consul.zip" | sha256sum -c \
     && unzip /tmp/consul -d /usr/local/bin \
     && rm /tmp/consul.zip \
+    # Create empty directories for Consul config and data \
+    && mkdir -p /etc/consul \
+    && mkdir -p /var/lib/consul \
     && mkdir /config
 
+
+# Install Consul template
+# Releases at https://releases.hashicorp.com/consul-template/
+RUN set -ex \
+    && export CONSUL_TEMPLATE_VERSION=0.18.0 \
+    && export CONSUL_TEMPLATE_CHECKSUM=f7adf1f879389e7f4e881d63ef3b84bce5bc6e073eb7a64940785d32c997bc4b \
+    && curl --retry 7 --fail -Lso /tmp/consul-template.zip "https://releases.hashicorp.com/consul-template/${CONSUL_TEMPLATE_VERSION}/consul-template_${CONSUL_TEMPLATE_VERSION}_linux_amd64.zip" \
+    && echo "${CONSUL_TEMPLATE_CHECKSUM}  /tmp/consul-template.zip" | sha256sum -c \
+    && unzip /tmp/consul-template.zip -d /usr/local/bin \
+    && rm /tmp/consul-template.zip
+
 # Add Containerpilot and set its configuration
 ENV CONTAINERPILOT_VERSION 3.0.0
 ENV CONTAINERPILOT /etc/containerpilot.json
@@ -58,6 +67,7 @@ ENV CONTAINERPILOT /etc/containerpilot.json
 # ref https://prometheus.io/docs/operating/configuration/
 # for details on building your own config
 COPY etc/prometheus.yml.ctmpl /etc/prometheus/prometheus.yml.ctmpl
+COPY bin /bin
 
 # Override the entrypoint to include Containerpilot
 WORKDIR /prometheus

bin/prestart.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Do we have env vars for Triton discovery?
+# Copy creds from env vars to files on disk
+if [ -n ${!TRITON_CREDS_PATH} ] \
+    && [ -n ${!TRITON_CA} ] \
+    && [ -n ${!TRITON_CERT} ] \
+    && [ -n ${!TRITON_KEY} ]
+then
+    mkdir -p ${TRITON_CREDS_PATH}
+    echo -e "${TRITON_CA}" | tr '#' '\n' > ${TRITON_CREDS_PATH}/ca.pem
+    echo -e "${TRITON_CERT}" | tr '#' '\n' > ${TRITON_CREDS_PATH}/cert.pem
+    echo -e "${TRITON_KEY}" | tr '#' '\n' > ${TRITON_CREDS_PATH}/key.pem
+fi
+
+# Are we on Triton? Do we _not_ have a user-defined DC?
+# Set the DC automatically from mdata
+if [ -n ${TRITON_DC} ] \
+    && [ -f "/native/usr/sbin/mdata-get" ]
+then
+    export TRITON_DC=$(/native/usr/sbin/mdata-get sdc:datacenter_name)
+fi
+
+# Create Prometheus config
+consul-template -once -consul-addr ${CONSUL}:8500 -template /etc/prometheus/prometheus.yml.ctmpl:/etc/prometheus/prometheus.yml

etc/containerpilot.json

@@ -3,10 +3,8 @@
   "jobs": [
     {
       "name": "setup-config",
-      "exec": [
-        "consul-template", "-once", "-consul", "{{ if .CONSUL_AGENT }}localhost{{ else }}{{ .CONSUL | default "consul" }}{{ end }}:8500",
-        "-template", "/etc/prometheus/prometheus.yml.ctmpl:/etc/prometheus/prometheus.yml"
-      ]{{ if .CONSUL_AGENT }},
+      "exec": "/bin/prestart.sh"
+      {{ if .CONSUL_AGENT }},
       "when": {
         "source": "consul-agent",
         "once": "healthy"
@@ -23,7 +21,7 @@
         "-web.console.templates=/etc/prometheus/consoles"
       ],
       "health": {
-        "exec": "curl -so /dev/null http://localhost:9090/metrics",
+        "exec": "curl -fso /dev/null http://localhost:9090/metrics",
         "interval": 10,
         "ttl": 25
       },

etc/prometheus.yml.ctmpl

@@ -1,7 +1,7 @@
 # my global config
 global:
   scrape_interval:     15s # By default, scrape targets every 15 seconds.
-  evaluation_interval: 15s # By default, scrape targets every 15 seconds.
+  evaluation_interval: 15s # By default, evaluate rules every 15 seconds.
   # scrape_timeout is set to the global default (10s).
 
   # Attach these labels to any time series or alerts when communicating with
@@ -26,11 +26,37 @@ scrape_configs:
     metrics_path: /metrics
     # scheme defaults to 'http'.
 
+    {{ $consul_agent := env "CONSUL_AGENT" }}
+    {{ $consul := env "CONSUL" }}
     consul_sd_configs:
-     - server: '{{ if .CONSUL_AGENT }}localhost{{ else }}{{ if .CONSUL }}{{ .CONSUL }}{{ else }}consul{{ end }}{{ end }}:8500'
+     - server: '{{ if $consul_agent }}localhost{{ else }}{{ if $consul }}{{ $consul }}{{ else }}consul{{ end }}{{ end }}:8500'
        services: ['containerpilot']
 
   - job_name: 'prometheus'
     metrics_path: /metrics
     static_configs:
      - targets: ['localhost:9090']
+
+  {{ $triton_tls_configured := env "TRITON_CREDS_PATH" }}
+  {{ if $triton_tls_configured }}
+  - job_name: 'triton'
+    scheme: https
+    tls_config:
+      ca_file: '{{env "TRITON_CA_PATH"}}'
+      cert_file: '{{env "TRITON_CERT_PATH"}}'
+      key_file: '{{env "TRITON_KEY_PATH"}}'
+      insecure_skip_verify: true
+    triton_sd_configs:
+      - account: '{{env "TRITON_ACCOUNT"}}'
+        dns_suffix: 'cmon.{{env "TRITON_DC"}}.triton.zone'
+        endpoint: 'cmon.{{env "TRITON_DC"}}.triton.zone'
+        version: 1
+        tls_config:
+          ca_file: '{{env "TRITON_CA_PATH"}}'
+          cert_file: '{{env "TRITON_CERT_PATH"}}'
+          key_file: '{{env "TRITON_KEY_PATH"}}'
+          insecure_skip_verify: true
+    relabel_configs:
+      - source_labels: [__meta_triton_machine_alias]
+        target_label: instance
+  {{ end }}

examples/compose/README.md

@@ -0,0 +1,7 @@
+# Autopilot Pattern Prometheus on local Docker
+
+To launch Prometheus locally (on Docker for Mac as an example):
+
+```bash
+$ docker-compose -p prometheus up -d
+```

docker-compose.yml → examples/compose/docker-compose.yml

@@ -1,33 +1,32 @@
+version: '2.1'
 # Prometheus demonstration of the autopilot pattern
 
-prometheus:
-    image: autopilotpattern/prometheus:0.17.0-r1
+services:
+  prometheus:
+    build: ../../
     mem_limit: 1g
     restart: always
-    labels:
-      - triton.cns.services=prometheus
+    dns:
+      - 127.0.0.1
     ports:
       - 9090
-    env_file: _env
+    environment:
+      - CONSUL=consul
+      - CONSUL_AGENT=1
+    links:
+      - consul:consul
 
 
-# Start with a single host which will bootstrap the cluster.
-# In production we'll want to use an HA cluster.
-consul:
+  # Start with a single host which will bootstrap the cluster.
+  # In production we'll want to use an HA cluster.
+  consul:
     image: autopilotpattern/consul:0.7.2-r0.8
     restart: always
     mem_limit: 128m
-    expose:
-      - 53
-      - 8300
-      - 8301
-      - 8302
-      - 8400
+    ports:
       - 8500
     dns:
-       - 127.0.0.1
-    labels:
-      - triton.cns.services=consul
+      - 127.0.0.1
     command: >
       /usr/local/bin/containerpilot
       /bin/consul agent -server

examples/triton/README.md

@@ -0,0 +1,20 @@
+# Autopilot Pattern Prometheus on Triton
+
+1. [Get a Joyent account](https://my.joyent.com/landing/signup/) and [add your SSH key](https://docs.joyent.com/public-cloud/getting-started).
+2. Install [Docker](https://docs.docker.com/docker-for-mac/install/) on your laptop or other environment, as well as the [Joyent Triton CLI](https://www.joyent.com/blog/introducing-the-triton-command-line-tool).
+3. Install the [Triton Docker CLI helper](https://github.com/joyent/triton-docker-cli).
+
+Check that everything is configured correctly by running the `setup.sh` script. This will check that your environment is setup correctly and create an `_env` file that includes environment variables with reasonable defaults, if not, run `eval "$(triton env)"`.
+
+```bash
+$ setup.sh
+$ vim _env
+```
+
+See the [README](../../README.md) for details on environment variables in `_env`.
+
+Start everything:
+
+```bash
+triton-docker up -d
+```

examples/triton/docker-compose.yml

@@ -0,0 +1,46 @@
+version: '2.1'
+# Prometheus demonstration of the autopilot pattern
+
+services:
+  prometheus:
+    image: autopilotpattern/prometheus:${TAG:-latest}
+    # Joyent recommends setting instances to always restart on Triton
+    restart: always
+    labels:
+      # This label sets the CNS name, Triton's automatic DNS
+      # Learn more at https://docs.joyent.com/public-cloud/network/cns
+      - triton.cns.services=prometheus
+      # This label selects the proper Joyent resource package
+      # https://www.joyent.com/blog/optimizing-docker-on-triton#ram-cpu-and-disk-resources-for-your-containers
+      - com.joyent.package=g4-highcpu-1G
+    network_mode: bridge
+    ports:
+      # You may not want these port declarations for production. Without them, Prometheus will only
+      # listen on the private network. This will also result in a public prometheus CNS record being created,
+      # in the triton.zone domain.
+      - 9090
+    env_file: _env
+    environment:
+      - CONSUL_AGENT=1
+      - CONSUL=pc.svc.${TRITON_CNS_SEARCH_DOMAIN_PRIVATE}
+
+
+  # Consul is the service catalog
+  consul:
+    image: autopilotpattern/consul:0.7.2-r0.8
+    command: >
+      /usr/local/bin/containerpilot
+      /bin/consul agent -server
+        -bootstrap-expect 1
+        -config-dir=/etc/consul
+        -ui-dir /ui
+        # Change "-bootstrap" to "-bootstrap-expect 3", then scale to 3 or more to
+        # turn this into an HA Consul raft.
+    restart: always
+    mem_limit: 128m
+    ports:
+      # As above, this port delcaration should not be made for production.
+      - 8500
+    labels:
+      - triton.cns.services=pc
+    network_mode: bridge

setup.sh → examples/triton/setup.sh

@@ -97,7 +97,22 @@ check() {
         exit 1
     fi
 
-    echo CONSUL=consul.svc.${TRITON_ACCOUNT}.${TRITON_DC}.cns.joyent.com > _env
+
+    echo '# Prometheus discovery for Triton' > _env
+    echo TRITON_ACCOUNT=${TRITON_ACCOUNT} >> _env
+    echo '#TRITON_DC= # Leave empty and Autopilot Pattern Prometheus will automatically detect the DC' >> _env
+    echo >> _env
+
+    echo '# Prometheus authentication for Triton' >> _env
+    TRITON_CREDS_PATH=/root/.triton
+    echo TRITON_CREDS_PATH=${TRITON_CREDS_PATH} >> _env
+    echo TRITON_CA=$(cat "${DOCKER_CERT_PATH}"/ca.pem | tr '\n' '#') >> _env
+    echo TRITON_CA_PATH=${TRITON_CREDS_PATH}/ca.pem >> _env
+    echo TRITON_KEY=$(cat "${DOCKER_CERT_PATH}"/key.pem | tr '\n' '#') >> _env
+    echo TRITON_KEY_PATH=${TRITON_CREDS_PATH}/key.pem >> _env
+    echo TRITON_CERT=$(cat "${DOCKER_CERT_PATH}"/cert.pem | tr '\n' '#') >> _env
+    echo TRITON_CERT_PATH=${TRITON_CREDS_PATH}/cert.pem >> _env
+    echo >> _env
 }
 
 # ---------------------------------------------------