From 99519e9b1139d95020c56fbc65d6b42f2a26b334 Mon Sep 17 00:00:00 2001 From: Pavel Kim Date: Tue, 23 Nov 2021 09:31:02 +0300 Subject: [PATCH] [Add] Prometheus metrics export implemented --- .github/workflows/pr_to_master.yml | 7 ++- .github/workflows/push_to_dev.yml | 4 +- .github/workflows/push_to_master.yml | 4 +- .version | 2 +- README.md | 31 ++++++++++-- check_certificates.sh | 73 ++++++++++++++++++++++++---- 6 files changed, 104 insertions(+), 17 deletions(-) diff --git a/.github/workflows/pr_to_master.yml b/.github/workflows/pr_to_master.yml index e195641..0a7c548 100644 --- a/.github/workflows/pr_to_master.yml +++ b/.github/workflows/pr_to_master.yml @@ -25,11 +25,16 @@ jobs: run: echo "${{steps.readversionfile.outputs.desiredversion}}" - name: Test run - run: bash check_certificates.sh -i test/inputfile.txt --only-alerting --alert-limit 5 | tee test_run_result.txt + run: | + export PROMETHEUS_EXPORT_FILENAME="./metrics" + bash check_certificates.sh -G -i test/inputfile.txt --only-alerting --alert-limit 5 | tee test_run_result.txt - name: Test run results verification run: diff -u test_run_result.txt test/expected_result.txt + - name: Test run results verification (metrics) + run: cat ./metrics + - name: Lookup planned tag id: tagexists run: | diff --git a/.github/workflows/push_to_dev.yml b/.github/workflows/push_to_dev.yml index de4c569..d2ee7af 100644 --- a/.github/workflows/push_to_dev.yml +++ b/.github/workflows/push_to_dev.yml @@ -25,7 +25,9 @@ jobs: run: echo "${{steps.readversionfile.outputs.desiredversion}}" - name: Test run - run: bash check_certificates.sh -i test/inputfile.txt --only-alerting --alert-limit 5 + run: | + export PROMETHEUS_EXPORT_FILENAME="./metrics" + bash check_certificates.sh -G -i test/inputfile.txt --only-alerting --alert-limit 5 - name: Lookup planned tag id: tagexists diff --git a/.github/workflows/push_to_master.yml b/.github/workflows/push_to_master.yml index 6c5189c..f858cae 100644 --- a/.github/workflows/push_to_master.yml +++ b/.github/workflows/push_to_master.yml @@ -25,7 +25,9 @@ jobs: run: echo "${{steps.readversionfile.outputs.desiredversion}}" - name: Test run - run: bash check_certificates.sh -i test/inputfile.txt --only-alerting --alert-limit 5 + run: | + export PROMETHEUS_EXPORT_FILENAME="./metrics" + bash check_certificates.sh -G -i test/inputfile.txt --only-alerting --alert-limit 5 - name: Lookup planned tag id: tagexists diff --git a/.version b/.version index 8e03717..ce6a70b 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -1.5.1 \ No newline at end of file +1.6.0 \ No newline at end of file diff --git a/README.md b/README.md index 911bcdf..31c8f64 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Latest release: [Download](https://github.com/pavelkim/check_certificates/releas The script takes on input a file with a list of hostnames: ```bash -Usage: check_certificates.sh [-h] [-v] [-s] [-l] [-n] [-A n] -i input_filename -d domain_name -b backend_name +Usage: check_certificates.sh [-h] [-v] [-s] [-l] [-n] [-A n] [-G] -i input_filename -d domain_name -b backend_name -b, --backend-name Domain list backend name (pastebin, gcs, etc.) -i, --input-filename Path to the list of domains to check @@ -19,19 +19,20 @@ Usage: check_certificates.sh [-h] [-v] [-s] [-l] [-n] [-A n] -i input_filename - -l, --only-alerting Show only alerting domains (expiring soon and erroneous) -n, --only-names Show only domain names instead of the full table -A, --alert-limit Set threshold of upcoming expiration alert to n days + -G, --generate-metrics Generates a Prometheus metrics file to be served by nginx -v, --verbose Enable debug output - -h, --help Enable debug output + -h, --help Show help ``` # Supported domain list backends Domain list backends allow you to manage configuration in a centralised manner. -## PasteBin +## PasteBin source You can use a PasteBin paste as a source of domain names to be checked. We encorage you to register on PasteBin and create all your pastes related to `check_certificates` as Private or at least as Unlisted. -1. Create a paste with a valid structure [example](https://pastebin.com/FJFvdiPg) +1. Create a paste with a valid structure ([example](https://pastebin.com/FJFvdiPg)) 1. Obtain devkey and userkey ([documentation](https://pastebin.com/doc_api#7)) 1. Fill out variables in `.config` file @@ -87,6 +88,28 @@ imaginary-domain-9000.com google.com ``` +# Prometheus metrics + +The script can generate a file with Prometheus metrics that is to be served by an external web server (eg. nginx or httpd). + +Use `-G` or `--generate-metrics` parameters to enable this feature. + +## .config file variables + +```bash +PROMETHEUS_EXPORT_FILENAME="/path/to/htdocs/metrics" +``` + +## Metrics example + +```prometheus +# HELP check_certificates_expiration Days until HTTPs SSL certificate expires (skipped on error) +# TYPE check_certificates_expiration counter +check_certificates_expiration{domain="example.com"} 20 +check_certificates_expiration{domain="example.de"} 193 +check_certificates_expiration{domain="imaginary-domain-9000.com"} -1 +``` + # Application examples ## Monitoring in Cron diff --git a/check_certificates.sh b/check_certificates.sh index 00834d4..a86f88d 100644 --- a/check_certificates.sh +++ b/check_certificates.sh @@ -6,6 +6,7 @@ set -o pipefail VERSION="DEV" +TODAY_TIMESTAMP="$(date "+%s")" [[ -f ".config" ]] && source .config || : @@ -15,7 +16,7 @@ usage() { SSL Certificate checker Version: ${VERSION} -Usage: $0 [-h] [-v] [-s] [-l] [-n] [-A n] -i input_filename -d domain_name -b backend_name +Usage: $0 [-h] [-v] [-s] [-l] [-n] [-A n] [-G] -i input_filename -d domain_name -b backend_name -b, --backend-name Domain list backend name (pastebin, gcs, etc.) -i, --input-filename Path to the list of domains to check @@ -24,8 +25,9 @@ Usage: $0 [-h] [-v] [-s] [-l] [-n] [-A n] -i input_filename -d domain_name -b ba -l, --only-alerting Show only alerting domains (expiring soon and erroneous) -n, --only-names Show only domain names instead of the full table -A, --alert-limit Set threshold of upcoming expiration alert to n days + -G, --generate-metrics Generates a Prometheus metrics file to be served by nginx -v, --verbose Enable debug output - -h, --help Enable debug output + -h, --help Show help EOF @@ -36,10 +38,10 @@ timestamp() { } error() { - + [[ ! -z "${1}" ]] && msg="ERROR: ${1}" || msg="ERROR!" [[ ! -z "${2}" ]] && rc="${2}" || rc=1 - + echo "[$(timestamp)] ${BASH_SOURCE[1]}: line ${BASH_LINENO[0]}: ${FUNCNAME[1]}: ${msg}" >&2 exit "${rc}" } @@ -124,6 +126,32 @@ backend_read_pastebin() { } +generate_prometheus_metrics() { + + [[ -z "${PROMETHEUS_EXPORT_FILENAME}" ]] && error "PROMETHEUS_EXPORT_FILENAME not set!" + [[ -z "${TODAY_TIMESTAMP}" ]] && error "TODAY_TIMESTAMP not set!" + + local metrics_name='check_certificates_expiration' + + [[ ! -z "$*" ]] && full_result=( "$@" ) || error "Formatted result list not set!" + + info "Exporting Prometheus metrics into file '${PROMETHEUS_EXPORT_FILENAME}'" + + info "Writing Prometheus metrics header (overwriting)" + echo "# HELP check_certificates_expiration Days until HTTPs SSL certificate expires (skipped on error)" > "${PROMETHEUS_EXPORT_FILENAME}" + echo "# TYPE check_certificates_expiration counter" >> "${PROMETHEUS_EXPORT_FILENAME}" + + for full_result_item in "${full_result[@]}"; do + full_result_item_parts=( ${full_result_item} ) + metrics_item="${metrics_name}{domain=\"${full_result_item_parts[0]}\"} $(( (${full_result_item_parts[2]} - ${TODAY_TIMESTAMP}) / 86400 ))" + info "Writing metrics item '${metrics_item}'" + echo "${metrics_item}" >> "${PROMETHEUS_EXPORT_FILENAME}" + done + + info "Finished Prometheus metrics export" + +} + backend_read() { local backend_name @@ -212,6 +240,7 @@ main() { local CLI_ALERT_LIMIT local CLI_ONLY_NAMES local CLI_SENSOR_MODE + local CLI_GENERATE_METRICS local CLI_RETRIES local CLI_VERBOSE @@ -219,8 +248,8 @@ main() { local error_result=( ) local formatted_result=( ) local sorted_result=( ) - local today_timestamp local input_filename + local formatted_result_item while [[ "$#" -gt 0 ]]; do case "${1}" in @@ -245,6 +274,9 @@ main() { -A|--alert-limit) [[ -z "${CLI_ALERT_LIMIT}" ]] && CLI_ALERT_LIMIT="${2}" || error "Argument already set: -A"; shift; shift;; + -G|--generate-metrics) + [[ -z "${CLI_GENERATE_METRICS}" ]] && CLI_GENERATE_METRICS="1" || error "Parameter already set: -G"; shift;; + -R|--retries) [[ -z "${CLI_RETRIES}" ]] && CLI_RETRIES="${2}" || error "Argument already set: -R"; shift; shift;; @@ -266,6 +298,16 @@ main() { error "Error! Only one parameter is allowed: input file or domain" fi + if [[ "${CLI_GENERATE_METRICS}" == "1" ]] && [[ -z "${PROMETHEUS_EXPORT_FILENAME}" ]]; then + error "Error! PROMETHEUS_EXPORT_FILENAME is not set" + else + if ! touch "${PROMETHEUS_EXPORT_FILENAME}"; then + error "Can't create Prometheus metrics file '${PROMETHEUS_EXPORT_FILENAME}'" + else + info "Prometheus metrics file touched: '${PROMETHEUS_EXPORT_FILENAME}'" + fi + fi + if [[ ! -z "${CLI_INPUT_FILENAME}" ]]; then [[ -f "${CLI_INPUT_FILENAME}" ]] || error "Can't open input file: '${CLI_INPUT_FILENAME}'" input_filename="${CLI_INPUT_FILENAME}" @@ -279,7 +321,7 @@ main() { backend_read "${CLI_BACKEND_NAME}" "${input_filename}" fi - today_timestamp="$(date "+%s")" + while IFS= read -r remote_hostname; do @@ -293,13 +335,18 @@ main() { warning "Skipping '${remote_hostname}'" error_result+=( "${remote_hostname}" ) else + info "Adding item into full_result: '${current_result}'" full_result+=( "${current_result}" ) fi + info "Finished processing '${remote_hostname}'" + done < "${input_filename}" if [[ "${#full_result[@]}" -le "0" ]]; then warning "Couldn't process anything from '${input_filename}'" + else + info "Processed '${#full_result[@]}' items from '${input_filename}'" fi if [[ "${#error_result[@]}" -gt "0" ]]; then @@ -308,18 +355,26 @@ main() { done fi + if [[ "${CLI_GENERATE_METRICS}" == "1" ]]; then + info "Generating Prometheus metrics" + generate_prometheus_metrics "${full_result[@]}" + fi + for result_item in "${full_result[@]}"; do result_item_parts=( ${result_item} ) - + info "Result item split into ${#result_item_parts[@]} parts: ${result_item_parts[*]}" + if [[ "${CLI_ONLY_ALERTING}" == "1" ]]; then - if [[ "$(( (result_item_parts[2] - today_timestamp) / 86400 ))" -gt "${CLI_ALERT_LIMIT}" ]]; then + if [[ "$(( (result_item_parts[2] - TODAY_TIMESTAMP) / 86400 ))" -gt "${CLI_ALERT_LIMIT}" ]]; then info "Certificate on ${result_item_parts[0]} expiring later than alert limit (${CLI_ALERT_LIMIT} day(s))." continue fi fi - formatted_result+=( "${result_item_parts[0]} $(epoch_to_date "${result_item_parts[1]}" "+%F %T") $(epoch_to_date "${result_item_parts[2]}" "+%F %T") $(( (result_item_parts[2] - today_timestamp) / 86400 ))" ) + formatted_result_item="${result_item_parts[0]} $(epoch_to_date "${result_item_parts[1]}" "+%F %T") $(epoch_to_date "${result_item_parts[2]}" "+%F %T") $(( (result_item_parts[2] - TODAY_TIMESTAMP) / 86400 ))" + info "Rendering a formatted result item: '${formatted_result_item}'" + formatted_result+=( "${formatted_result_item}" ) done while read formatted_item; do