From c19497ff05cc77b7bc701af37b0324467a795a93 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Wed, 1 Apr 2020 13:49:06 +0100 Subject: [PATCH 1/2] HKG->AMS replication over VPN --- ansible/roles/plpsql/templates/pg_hba.conf | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ansible/roles/plpsql/templates/pg_hba.conf b/ansible/roles/plpsql/templates/pg_hba.conf index 6a855223..f16f72e0 100644 --- a/ansible/roles/plpsql/templates/pg_hba.conf +++ b/ansible/roles/plpsql/templates/pg_hba.conf @@ -94,7 +94,13 @@ host all all ::1/128 trust # NB: `ssl` is neither supported nor enforced. {% if 'db_active' in group_names %} -host replication amsrepl {{ lookup('dig', 'amsmetadb.ooni.nu/A') }}/32 md5 + +# Run active->standby replication over VPN +host replication amsrepl 10.1.0.1/32 md5 + +# Run active->standby replication without VPN - unused +#host replication amsrepl {{ lookup('dig', 'amsmetadb.ooni.nu/A') }}/32 md5 + {% endif %} host all all all md5 From 58b5bd918a725b8843f8bbeacd1f28709fa788be Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Thu, 2 Apr 2020 14:25:17 +0100 Subject: [PATCH 2/2] Add replication monitoring --- .../files/db_replication_mon_active | 22 +++++++++++++++ .../files/db_replication_mon_standby | 22 +++++++++++++++ ansible/roles/node_exporter/handlers/main.yml | 8 ++++++ ansible/roles/node_exporter/tasks/main.yml | 20 ++++++++++++++ .../templates/db_replication_mon.service | 27 +++++++++++++++++++ ansible/roles/plpsql/README.adoc | 8 ++++++ 6 files changed, 107 insertions(+) create mode 100644 ansible/roles/node_exporter/files/db_replication_mon_active create mode 100755 ansible/roles/node_exporter/files/db_replication_mon_standby create mode 100644 ansible/roles/node_exporter/templates/db_replication_mon.service create mode 100644 ansible/roles/plpsql/README.adoc diff --git a/ansible/roles/node_exporter/files/db_replication_mon_active b/ansible/roles/node_exporter/files/db_replication_mon_active new file mode 100644 index 00000000..4f0407ce --- /dev/null +++ b/ansible/roles/node_exporter/files/db_replication_mon_active @@ -0,0 +1,22 @@ +#!/bin/bash +# Deployed by ansible +# See roles/node_exporter/files/db_replication_mon_active +# active database --> standby +# +# Errors are reported as RTT=0 + +set -u +FN=/run/nodeexp/db_replication_socket.prom +while true; do + socket_rtt=$(ss -ntpi state established 'dst 37.218.242.175' | tr -s " " "\n" | grep ^rtt: | cut -c5- | cut -d'/' -f1) + socket_rtt=${socket_rtt:-0} + ping_rtt=$(ping -w 1 -c 1 10.1.0.1 | tail -n1 | cut -d'/' -f5) + ping_rtt=${ping_rtt:-0} + + cat < $FN.tmp +db_replication_rtt{role="active"} $socket_rtt +ping_rtt{role="active"} $ping_rtt +EOF + mv $FN.tmp $FN + sleep 1 +done diff --git a/ansible/roles/node_exporter/files/db_replication_mon_standby b/ansible/roles/node_exporter/files/db_replication_mon_standby new file mode 100755 index 00000000..415fb030 --- /dev/null +++ b/ansible/roles/node_exporter/files/db_replication_mon_standby @@ -0,0 +1,22 @@ +#!/bin/bash +# Deployed by ansible +# See roles/node_exporter/files/db_replication_mon_standby +# standby database --> active +# +# Errors are reported as RTT=0 + +set -u +FN=/run/nodeexp/db_replication_socket.prom +while true; do + socket_rtt=$(ss -ntpi state established 'dst 37.218.240.56 dport = 5432' | tr -s " " "\n" | grep ^rtt: | cut -c5- | cut -d'/' -f1) + socket_rtt=${socket_rtt:-0} + ping_rtt=$(ping -w 1 -c 1 10.1.0.2 | tail -n1 | cut -d'/' -f5) + ping_rtt=${ping_rtt:-0} + + cat < $FN.tmp +db_replication_rtt{role="standby"} $socket_rtt +ping_rtt{role="standby"} $ping_rtt +EOF + mv $FN.tmp $FN + sleep 1 +done diff --git a/ansible/roles/node_exporter/handlers/main.yml b/ansible/roles/node_exporter/handlers/main.yml index 029cc475..3780fb2e 100644 --- a/ansible/roles/node_exporter/handlers/main.yml +++ b/ansible/roles/node_exporter/handlers/main.yml @@ -3,14 +3,22 @@ service: name: node_exporter state: restarted + - name: restart seeksample service: name: seeksample state: restarted + - name: restart tcpmetrics service: name: tcpmetrics state: restarted + +- name: restart db_replication_mon + service: + name: db_replication_mon + state: restarted + - name: restart collector-metrics service: name: collector-metrics diff --git a/ansible/roles/node_exporter/tasks/main.yml b/ansible/roles/node_exporter/tasks/main.yml index a3477c68..e6416c55 100644 --- a/ansible/roles/node_exporter/tasks/main.yml +++ b/ansible/roles/node_exporter/tasks/main.yml @@ -89,6 +89,26 @@ dest: /etc/systemd/system/tcpmetrics.service when: "'have_tcpmetrics' in group_names" +- name: Install db_replication_mon on active DB + copy: src=db_replication_mon_active dest=/srv/db_replication_mon mode=0555 + when: "'db_active' in group_names" + tags: db_replication_mon + +- name: Install db_replication_mon on standby DB + copy: src=db_replication_mon_standby dest=/srv/db_replication_mon mode=0555 + when: "'db_standby' in group_names" + tags: db_replication_mon + +- name: Install db_replication_mon systemd service file + notify: + - systemctl daemon-reload + - restart db_replication_mon + template: + src: db_replication_mon.service + dest: /etc/systemd/system/db_replication_mon.service + when: "'db_active' in group_names or 'db_standby' in group_names" + tags: db_replication_mon + - name: Install collector-metrics systemd service file notify: - restart collector-metrics diff --git a/ansible/roles/node_exporter/templates/db_replication_mon.service b/ansible/roles/node_exporter/templates/db_replication_mon.service new file mode 100644 index 00000000..2c7b559d --- /dev/null +++ b/ansible/roles/node_exporter/templates/db_replication_mon.service @@ -0,0 +1,27 @@ +# Deployed by ansible +# see roles/node_exporter/templates/db_replication_mon.service +[Unit] +Description=DB replication monitor + +[Service] +User=root +Group=root + +ExecStartPre=/usr/bin/install --owner root --group root --mode 0755 --directory {{ node_exporter_textfiles_path }} +ExecStart=/srv/db_replication_mon +KillMode=process +Restart=always + +# Hardening +CapabilityBoundingSet=CAP_SETUID CAP_SETGID CAP_NET_RAW +SystemCallFilter=~@clock @debug @cpu-emulation @keyring @module @mount @obsolete @raw-io @reboot @swap +NoNewPrivileges=yes +PrivateDevices=yes +PrivateTmp=yes +ProtectHome=yes +ProtectSystem=full +ProtectKernelModules=yes +ProtectKernelTunables=yes + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/plpsql/README.adoc b/ansible/roles/plpsql/README.adoc new file mode 100644 index 00000000..ef1a7915 --- /dev/null +++ b/ansible/roles/plpsql/README.adoc @@ -0,0 +1,8 @@ +PostgreSQL metadb database + +Deploys a temporary traffic forwarder using A VPN + HKG -> mia-ps-test.ooni.nu -> AMS + +Runs db_replication_mon.service on HKG and AMS db to monitor the replication +sudo systemctl status db_replication_mon.service +sudo journalctl -f -u db_replication_mon