From 6a14bc6fb870eaaa7456b07208c47a0f92e41dab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CArslan?= Date: Sat, 5 Oct 2024 08:38:35 +0000 Subject: [PATCH 1/2] Added base core-booking rps metrics --- core-booking/build.gradle.kts | 6 +- .../src/main/resources/application.yaml | 19 +- .../grafana/provisioning/dashboards/all.yml | 10 + .../provisioning/dashboards/core-booking.json | 505 ++++++++++++++++++ .../grafana/provisioning/datasources/all.yml | 8 + metrics/config/prometheus.yaml | 10 +- metrics/docker-compose.yaml | 36 +- 7 files changed, 564 insertions(+), 30 deletions(-) create mode 100644 metrics/config/grafana/provisioning/dashboards/all.yml create mode 100644 metrics/config/grafana/provisioning/dashboards/core-booking.json create mode 100644 metrics/config/grafana/provisioning/datasources/all.yml diff --git a/core-booking/build.gradle.kts b/core-booking/build.gradle.kts index 700bb90..7fc7612 100644 --- a/core-booking/build.gradle.kts +++ b/core-booking/build.gradle.kts @@ -26,9 +26,9 @@ dependencies { implementation("com.fasterxml.jackson.module:jackson-module-kotlin") implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310") implementation("org.springframework.kafka:spring-kafka") - implementation("io.micrometer:micrometer-core:1.13.5") - implementation("io.micrometer:micrometer-registry-prometheus:1.13.5") - implementation("org.springframework.boot:spring-boot-starter-actuator:3.3.4") + implementation("io.micrometer:micrometer-core") + implementation("io.micrometer:micrometer-registry-prometheus") + implementation("org.springframework.boot:spring-boot-starter-actuator:3.3.0") } tasks.test { diff --git a/core-booking/src/main/resources/application.yaml b/core-booking/src/main/resources/application.yaml index 4756923..f5dcb63 100644 --- a/core-booking/src/main/resources/application.yaml +++ b/core-booking/src/main/resources/application.yaml @@ -30,11 +30,14 @@ management: endpoints: web: exposure: - include: health,prometheus - metrics: - export: - prometheus: - enabled: true - distribution: - percentiles-histogram: - "[http.server.requests]": true \ No newline at end of file + include: 'prometheus,health' + endpoint: + prometheus: + enabled: true + metrics: + export: + prometheus: + enabled: true + distribution: + percentiles-histogram: + "[http.server.requests]": true \ No newline at end of file diff --git a/metrics/config/grafana/provisioning/dashboards/all.yml b/metrics/config/grafana/provisioning/dashboards/all.yml new file mode 100644 index 0000000..1c2155f --- /dev/null +++ b/metrics/config/grafana/provisioning/dashboards/all.yml @@ -0,0 +1,10 @@ +apiVersion: 1 +providers: + - name: 'default' + folder: 'default' + type: file + allowUiUpdates: true + updateIntervalSeconds: 30 + options: + path: /etc/grafana/provisioning/dashboards + foldersFromFilesStructure: true \ No newline at end of file diff --git a/metrics/config/grafana/provisioning/dashboards/core-booking.json b/metrics/config/grafana/provisioning/dashboards/core-booking.json new file mode 100644 index 0000000..4e1af59 --- /dev/null +++ b/metrics/config/grafana/provisioning/dashboards/core-booking.json @@ -0,0 +1,505 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Core-booking metrics dashboard", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 5373, + "graphTooltip": 0, + "id": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], + "title": "Status and throughput", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "text": "DOWN" + }, + "1": { + "text": "UP" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "text": "DOWN" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 4, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "up{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 8, + "y": 1 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum(rate(http_server_requests_seconds_count{instance=~\"$instance\"}[1m]))", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "intervalFactor": 1, + "legendFormat": "{{method}}-{{status}}-{{uri}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Requests per second", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 6 + }, + "id": 18, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "process_uptime_seconds{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Up Time", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 19, + "panels": [], + "title": "JVM", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance) (jvm_memory_used_bytes{instance=~\"$instance\"})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Total jvm memory usage", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "core-booking-1:8080", + "value": "core-booking-2:8080" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(up{instance=~\"core-booking-1:8080|core-booking-2:8080\"},instance)", + "hide": 0, + "includeAll": false, + "label": "instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(up{instance=~\"core-booking-1:8080|core-booking-2:8080\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Core-booking metrics", + "uid": "twqdYjziz", + "version": 10, + "weekStart": "" +} \ No newline at end of file diff --git a/metrics/config/grafana/provisioning/datasources/all.yml b/metrics/config/grafana/provisioning/datasources/all.yml new file mode 100644 index 0000000..40b9aef --- /dev/null +++ b/metrics/config/grafana/provisioning/datasources/all.yml @@ -0,0 +1,8 @@ +apiVersion: 1 +datasources: + - name: Prometheus + label: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true \ No newline at end of file diff --git a/metrics/config/prometheus.yaml b/metrics/config/prometheus.yaml index b7c9f93..54198bf 100644 --- a/metrics/config/prometheus.yaml +++ b/metrics/config/prometheus.yaml @@ -9,7 +9,13 @@ scrape_configs: - job_name: 'core-booking-1' scrape_interval: 1s - metrics_path: '/actuator/metrics' + metrics_path: '/actuator/prometheus' static_configs: - - targets: ['core-booking-core-booking-1-1:8080'] + - targets: ['core-booking-1:8080'] + + - job_name: 'core-booking-2' + scrape_interval: 1s + metrics_path: '/actuator/prometheus' + static_configs: + - targets: ['core-booking-2:8080'] \ No newline at end of file diff --git a/metrics/docker-compose.yaml b/metrics/docker-compose.yaml index 0a742c9..ce000e4 100644 --- a/metrics/docker-compose.yaml +++ b/metrics/docker-compose.yaml @@ -1,17 +1,19 @@ services: - # grafana: - # build: './config/grafana' - # ports: - # - 3000:3000 - # volumes: - # - ./grafana:/var/lib/grafana - # environment: - # - GF_SECURITY_ADMIN_USER=admin - # - GF_SECURITY_ADMIN_PASSWORD=admin - # networks: - # monitoring: - # aliases: - # - grafana + grafana: + build: './config/grafana' + container_name: 'grafana' + hostname: grafana + user: root + ports: + - 3001:3000 + volumes: + - ./grafana:/var/lib/grafana + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=admin + networks: + - shared-network + prometheus: image: prom/prometheus container_name: 'prometheus' @@ -27,10 +29,10 @@ services: networks: - shared-network +volumes: + prometheus: + networks: monitoring: shared-network: - external: true - -volumes: - prometheus: {} \ No newline at end of file + external: true \ No newline at end of file From adb1f0bd94510eb426f6ee7e1debaf0496bfeba4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CArslan?= Date: Sat, 5 Oct 2024 17:59:45 +0000 Subject: [PATCH 2/2] Added business metric on terminal booking status from core-rcatalog kafka --- .../kotlin/service/BookingMetricsService.kt | 27 +++++++++++++++++++ .../stream/kafka/CoreCatalogEventsListener.kt | 4 +++ 2 files changed, 31 insertions(+) create mode 100644 core-booking/src/main/kotlin/service/BookingMetricsService.kt diff --git a/core-booking/src/main/kotlin/service/BookingMetricsService.kt b/core-booking/src/main/kotlin/service/BookingMetricsService.kt new file mode 100644 index 0000000..02c94ae --- /dev/null +++ b/core-booking/src/main/kotlin/service/BookingMetricsService.kt @@ -0,0 +1,27 @@ +package org.example.service + +import io.micrometer.core.instrument.Counter +import io.micrometer.core.instrument.MeterRegistry +import org.example.model.Reservation +import org.springframework.stereotype.Component + +@Component +class BookingMetricsService(meterRegistry: MeterRegistry) { + private val successCounter: Counter = Counter.builder("booking_status_success_count") + .description("Count of successful bookings") + .tags("status", "SUCCESS") + .register(meterRegistry) + + private val failedCounter: Counter = Counter.builder("booking_status_failed_count") + .description("Count of failed bookings") + .tags("status", "FAILED") + .register(meterRegistry) + + internal fun incrementBookingMetric(status: Reservation.ReservationStatus) { + when (status) { + Reservation.ReservationStatus.SUCCESS -> successCounter.increment() + Reservation.ReservationStatus.FAILED -> failedCounter.increment() + else -> {} + } + } +} \ No newline at end of file diff --git a/core-booking/src/main/kotlin/stream/kafka/CoreCatalogEventsListener.kt b/core-booking/src/main/kotlin/stream/kafka/CoreCatalogEventsListener.kt index 71429a1..192817e 100644 --- a/core-booking/src/main/kotlin/stream/kafka/CoreCatalogEventsListener.kt +++ b/core-booking/src/main/kotlin/stream/kafka/CoreCatalogEventsListener.kt @@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.databind.ObjectMapper import org.example.model.Reservation import org.example.model.ReservationRepository +import org.example.service.BookingMetricsService import org.springframework.kafka.annotation.KafkaListener import org.springframework.stereotype.Service import java.time.Instant @@ -13,6 +14,7 @@ import java.util.UUID class CoreCatalogEventsListener( private val reservationRepository: ReservationRepository, private val objectMapper: ObjectMapper, + private val bookingMetricsService: BookingMetricsService, ) { @KafkaListener(topics = ["\${spring.kafka.consumer.topic.name}"], groupId = "\${spring.kafka.consumer.group-id}") @@ -30,6 +32,8 @@ class CoreCatalogEventsListener( reason = reservationEvent.reason, expectedStatus = Reservation.ReservationStatus.IN_PROGRESS, ) + + bookingMetricsService.incrementBookingMetric(updatedStatus) } internal fun fromCoreCatalogStatus(status: String) =