From 8b809c5a89c848e8862e81fc95f4bb7ba08d7964 Mon Sep 17 00:00:00 2001 From: Jonathan VUILLEMIN Date: Wed, 28 Feb 2024 14:45:22 +0100 Subject: [PATCH] feat(httpclient): Added metrics transport (#108) --- httpclient/README.md | 56 +++++++++++-- httpclient/go.mod | 12 ++- httpclient/go.sum | 30 +++++-- httpclient/status/status.go | 17 ++++ httpclient/status/status_test.go | 31 +++++++ httpclient/transport/metrics.go | 116 +++++++++++++++++++++++++++ httpclient/transport/metrics_test.go | 105 ++++++++++++++++++++++++ 7 files changed, 354 insertions(+), 13 deletions(-) create mode 100644 httpclient/status/status.go create mode 100644 httpclient/status/status_test.go create mode 100644 httpclient/transport/metrics.go create mode 100644 httpclient/transport/metrics_test.go diff --git a/httpclient/README.md b/httpclient/README.md index b29b16c..35ec93a 100644 --- a/httpclient/README.md +++ b/httpclient/README.md @@ -9,14 +9,13 @@ > Http client module based on [net/http](https://pkg.go.dev/net/http). - * [Installation](#installation) * [Documentation](#documentation) - * [Requests](#requests) - * [Transports](#transports) - * [BaseTransport](#basetransport) - * [LoggerTransport](#loggertransport) - + * [Requests](#requests) + * [Transports](#transports) + * [BaseTransport](#basetransport) + * [LoggerTransport](#loggertransport) + * [MetricsTransport](#metricstransport) ## Installation @@ -166,3 +165,48 @@ var client, _ = httpclient.NewDefaultHttpClientFactory().Create( ``` Note: if no transport is provided for decoration in `transport.NewLoggerTransport(nil)`, the [BaseTransport](transport/base.go) will be used as base transport. + +#### MetricsTransport + +This module provide a [MetricsTransport](transport/metrics.go), able to decorate any `http.RoundTripper` to add metrics: + +- about requests total count (labelled by url, http method and status code) +- about requests duration (labelled by url and http method) + +To use it: + +```go +package main + +import ( + "github.com/ankorstore/yokai/httpclient" + "github.com/ankorstore/yokai/httpclient/transport" + "github.com/prometheus/client_golang/prometheus" + "github.com/rs/zerolog" +) + +var client, _ = httpclient.NewDefaultHttpClientFactory().Create( + httpclient.WithTransport(transport.NewMetricsTransport(nil)), +) + +// equivalent to: +var client, _ = httpclient.NewDefaultHttpClientFactory().Create( + httpclient.WithTransport( + transport.NewMetricsTransportWithConfig( + transport.NewBaseTransport(), + &transport.MetricsTransportConfig{ + Registry: prometheus.DefaultRegisterer, // metrics registry + Namespace: "", // metrics namespace + Subsystem: "", // metrics subsystem + Buckets: prometheus.DefBuckets, // metrics duration buckets + NormalizeHTTPStatus: true, // normalize the response HTTP code (ex: 201 => 2xx) + }, + ), + ), +) +``` + +Notes: + +- if no transport is provided for decoration in `transport.NewMetricsTransport(nil)`, the [BaseTransport](transport/base.go) will be used as base transport +- if no registry is provided in the `config` in `transport.NewMetricsTransportWithConfig(nil, config)`, the `prometheus.DefaultRegisterer` will be used a metrics registry diff --git a/httpclient/go.mod b/httpclient/go.mod index 032c066..f9375f3 100644 --- a/httpclient/go.mod +++ b/httpclient/go.mod @@ -3,18 +3,26 @@ module github.com/ankorstore/yokai/httpclient go 1.20 require ( - github.com/ankorstore/yokai/log v1.0.0 + github.com/ankorstore/yokai/log v1.1.0 + github.com/prometheus/client_golang v1.19.0 github.com/rs/zerolog v1.29.1 github.com/stretchr/testify v1.8.4 ) require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/kr/text v0.2.0 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.48.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect go.opentelemetry.io/otel v1.16.0 // indirect go.opentelemetry.io/otel/trace v1.16.0 // indirect - golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 // indirect + golang.org/x/sys v0.16.0 // indirect + google.golang.org/protobuf v1.32.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/httpclient/go.sum b/httpclient/go.sum index 68d2d88..3485379 100644 --- a/httpclient/go.sum +++ b/httpclient/go.sum @@ -1,10 +1,18 @@ -github.com/ankorstore/yokai/log v1.0.0 h1:9NsM0J+1O028WuNDW7vr0yeUdWDX1JKYTkuz7hiYCSs= -github.com/ankorstore/yokai/log v1.0.0/go.mod h1:lyBRVA8VkrmlNjaR2jVTH9XjV06ioolWTuDVN6wF0vk= +github.com/ankorstore/yokai/log v1.1.0 h1:7+kkmbGMtpfkEEaWSZ4/4Yp+dieVoolOVG24NpEJDO4= +github.com/ankorstore/yokai/log v1.1.0/go.mod h1:lyBRVA8VkrmlNjaR2jVTH9XjV06ioolWTuDVN6wF0vk= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= @@ -12,6 +20,15 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= +github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc= github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= @@ -23,9 +40,12 @@ go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeH go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 h1:foEbQz/B0Oz6YIqu/69kfXPYeFQAuuMYFkjaqXzl5Wo= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/httpclient/status/status.go b/httpclient/status/status.go new file mode 100644 index 0000000..8fe1ee8 --- /dev/null +++ b/httpclient/status/status.go @@ -0,0 +1,17 @@ +package status + +// NormalizeHTTPStatus normalizes an HTTP status code. +func NormalizeHTTPStatus(status int) string { + switch { + case status < 200: + return "1xx" + case status >= 200 && status < 300: + return "2xx" + case status >= 300 && status < 400: + return "3xx" + case status >= 400 && status < 500: + return "4xx" + default: + return "5xx" + } +} diff --git a/httpclient/status/status_test.go b/httpclient/status/status_test.go new file mode 100644 index 0000000..b9db0ce --- /dev/null +++ b/httpclient/status/status_test.go @@ -0,0 +1,31 @@ +package status_test + +import ( + "testing" + + "github.com/ankorstore/yokai/httpclient/status" +) + +func TestNormalizeHTTPStatus(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + code int + want string + }{ + {"1xx normalization", 101, "1xx"}, + {"2xx normalization", 202, "2xx"}, + {"3xx normalization", 303, "3xx"}, + {"4xx normalization", 404, "4xx"}, + {"5xx normalization", 505, "5xx"}, + } + + for _, tt := range tests { + got := status.NormalizeHTTPStatus(tt.code) + + if got != tt.want { + t.Errorf("expected %s, got %s", tt.want, got) + } + } +} diff --git a/httpclient/transport/metrics.go b/httpclient/transport/metrics.go new file mode 100644 index 0000000..ed39b10 --- /dev/null +++ b/httpclient/transport/metrics.go @@ -0,0 +1,116 @@ +package transport + +import ( + "net/http" + "strconv" + + "github.com/ankorstore/yokai/httpclient/status" + "github.com/prometheus/client_golang/prometheus" +) + +const ( + HttpClientMetricsRequestsCount = "client_requests_total" + HttpClientMetricsRequestsDuration = "client_request_duration_seconds" +) + +// MetricsTransport is a wrapper around [http.RoundTripper] with some [MetricsTransportConfig] configuration. +type MetricsTransport struct { + transport http.RoundTripper + config *MetricsTransportConfig + requestsCounter *prometheus.CounterVec + requestsDuration *prometheus.HistogramVec +} + +// MetricsTransportConfig is the configuration of the [MetricsTransport]. +type MetricsTransportConfig struct { + Registry prometheus.Registerer + Namespace string + Subsystem string + Buckets []float64 + NormalizeHTTPStatus bool +} + +// NewMetricsTransport returns a [MetricsTransport] instance with default [MetricsTransportConfig] configuration. +func NewMetricsTransport(base http.RoundTripper) *MetricsTransport { + return NewMetricsTransportWithConfig( + base, + &MetricsTransportConfig{ + Registry: prometheus.DefaultRegisterer, + Namespace: "", + Subsystem: "", + Buckets: prometheus.DefBuckets, + NormalizeHTTPStatus: true, + }, + ) +} + +// NewMetricsTransportWithConfig returns a [MetricsTransport] instance for a provided [MetricsTransportConfig] configuration. +func NewMetricsTransportWithConfig(base http.RoundTripper, config *MetricsTransportConfig) *MetricsTransport { + if base == nil { + base = NewBaseTransport() + } + + if config.Registry == nil { + config.Registry = prometheus.DefaultRegisterer + } + + requestsCounter := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: config.Namespace, + Subsystem: config.Subsystem, + Name: HttpClientMetricsRequestsCount, + Help: "Number of performed HTTP requests", + }, + []string{ + "url", + "method", + "status", + }, + ) + + requestsDuration := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: config.Namespace, + Subsystem: config.Subsystem, + Name: HttpClientMetricsRequestsDuration, + Help: "Time spent performing HTTP requests", + Buckets: config.Buckets, + }, + []string{ + "url", + "method", + }, + ) + + config.Registry.MustRegister(requestsCounter, requestsDuration) + + return &MetricsTransport{ + transport: base, + config: config, + requestsCounter: requestsCounter, + requestsDuration: requestsDuration, + } +} + +// Base returns the wrapped [http.RoundTripper]. +func (t *MetricsTransport) Base() http.RoundTripper { + return t.transport +} + +// RoundTrip performs a request / response round trip, based on the wrapped [http.RoundTripper]. +func (t *MetricsTransport) RoundTrip(req *http.Request) (*http.Response, error) { + timer := prometheus.NewTimer(t.requestsDuration.WithLabelValues(req.URL.String(), req.Method)) + resp, err := t.transport.RoundTrip(req) + timer.ObserveDuration() + + respStatus := "" + if t.config.NormalizeHTTPStatus { + respStatus = status.NormalizeHTTPStatus(resp.StatusCode) + } else { + respStatus = strconv.Itoa(resp.StatusCode) + } + + t.requestsCounter.WithLabelValues(req.URL.String(), req.Method, respStatus).Inc() + + return resp, err +} diff --git a/httpclient/transport/metrics_test.go b/httpclient/transport/metrics_test.go new file mode 100644 index 0000000..92e5cd5 --- /dev/null +++ b/httpclient/transport/metrics_test.go @@ -0,0 +1,105 @@ +package transport_test + +import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/ankorstore/yokai/httpclient/transport" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" +) + +func TestMetricsTransportRoundTrip(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + + req := httptest.NewRequest(http.MethodGet, server.URL, nil) + + trans := transport.NewMetricsTransport(nil) + assert.IsType(t, &transport.MetricsTransport{}, trans) + assert.Implements(t, (*http.RoundTripper)(nil), trans) + + resp, err := trans.RoundTrip(req) + assert.NoError(t, err) + + err = resp.Body.Close() + assert.NoError(t, err) + + assert.Equal(t, http.StatusInternalServerError, resp.StatusCode) + + // requests counter assertions + expectedCounterMetric := fmt.Sprintf(` + # HELP client_requests_total Number of performed HTTP requests + # TYPE client_requests_total counter + client_requests_total{method="GET",status="5xx",url="%s"} 1 + `, + server.URL, + ) + + err = testutil.GatherAndCompare( + prometheus.DefaultGatherer, + strings.NewReader(expectedCounterMetric), + "client_requests_total", + ) + assert.NoError(t, err) +} + +func TestMetricsTransportRoundTripWithBaseAndConfig(t *testing.T) { + t.Parallel() + + registry := prometheus.NewPedanticRegistry() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + req := httptest.NewRequest(http.MethodGet, server.URL, nil) + + base := &http.Transport{} + + trans := transport.NewMetricsTransportWithConfig( + base, + &transport.MetricsTransportConfig{ + Registry: registry, + Namespace: "foo", + Subsystem: "bar", + Buckets: []float64{1, 2, 3}, + NormalizeHTTPStatus: false, + }, + ) + + assert.Equal(t, base, trans.Base()) + + resp, err := trans.RoundTrip(req) + assert.NoError(t, err) + + err = resp.Body.Close() + assert.NoError(t, err) + + assert.Equal(t, http.StatusNoContent, resp.StatusCode) + + // requests counter assertions + expectedCounterMetric := fmt.Sprintf(` + # HELP foo_bar_client_requests_total Number of performed HTTP requests + # TYPE foo_bar_client_requests_total counter + foo_bar_client_requests_total{method="GET",status="204",url="%s"} 1 + `, + server.URL, + ) + + err = testutil.GatherAndCompare( + registry, + strings.NewReader(expectedCounterMetric), + "foo_bar_client_requests_total", + ) + assert.NoError(t, err) +}