Use Otel Collector (#192)

.github/workflows/k8s-snap-integration.yaml

@@ -14,7 +14,7 @@ concurrency:
 
 jobs:
   build:
-    name: Build K8-dqlite
+    name: K8s-snap Integration Test
     runs-on: ubuntu-20.04
 
     steps:

cmd/tracing.go

@@ -4,15 +4,16 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"time"
 
 	"github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
-	"go.opentelemetry.io/otel/exporters/stdout/stdoutmetric"
 	"go.opentelemetry.io/otel/sdk/metric"
+	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
 	"go.opentelemetry.io/otel/sdk/resource"
 	"go.opentelemetry.io/otel/sdk/trace"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
@@ -51,7 +52,7 @@ func setupOTelSDK(ctx context.Context, otelEndpoint string) (shutdown func(conte
 	tracerProvider := newTraceProvider(traceExporter, res)
 	otel.SetTracerProvider(tracerProvider)
 
-	meterProvider, err := newMeterProvider(res)
+	meterExporter, err := newMeterExporter(ctx, conn)
 	if err != nil {
 		var shutdownErrs error
 		shutdownErr := tracerProvider.Shutdown(ctx)
@@ -67,6 +68,7 @@ func setupOTelSDK(ctx context.Context, otelEndpoint string) (shutdown func(conte
 			return nil, fmt.Errorf("failed to create meter provider: %w", err)
 		}
 	}
+	meterProvider, err := newMeterProvider(meterExporter, res)
 	otel.SetMeterProvider(meterProvider)
 
 	shutdown = func(ctx context.Context) error {
@@ -110,25 +112,27 @@ func newTraceExporter(ctx context.Context, conn *grpc.ClientConn) (trace.SpanExp
 }
 
 func newTraceProvider(traceExporter trace.SpanExporter, res *resource.Resource) *trace.TracerProvider {
-	traceProvider := trace.NewTracerProvider(
-		trace.WithBatcher(traceExporter,
-			trace.WithBatchTimeout(time.Second),
-		),
-		trace.WithResource(res),
+	bsp := sdktrace.NewBatchSpanProcessor(traceExporter)
+	traceProvider := sdktrace.NewTracerProvider(
+		sdktrace.WithSampler(sdktrace.AlwaysSample()),
+		sdktrace.WithResource(res),
+		sdktrace.WithSpanProcessor(bsp),
 	)
 	return traceProvider
 }
 
-func newMeterProvider(res *resource.Resource) (*metric.MeterProvider, error) {
-	metricExporter, err := stdoutmetric.New()
+func newMeterExporter(ctx context.Context, conn *grpc.ClientConn) (*otlpmetricgrpc.Exporter, error) {
+	metricExporter, err := otlpmetricgrpc.New(ctx, otlpmetricgrpc.WithGRPCConn(conn))
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to create metric exporter: %w", err)
 	}
+	return metricExporter, nil
+}
 
-	meterProvider := metric.NewMeterProvider(
-		metric.WithReader(metric.NewPeriodicReader(metricExporter,
-			metric.WithInterval(30*time.Second))),
-		metric.WithResource(res),
+func newMeterProvider(metricExporter *otlpmetricgrpc.Exporter, res *resource.Resource) (*metric.MeterProvider, error) {
+	meterProvider := sdkmetric.NewMeterProvider(
+		sdkmetric.WithReader(sdkmetric.NewPeriodicReader(metricExporter)),
+		sdkmetric.WithResource(res),
 	)
 	return meterProvider, nil
 }

docs/configuration.md

@@ -40,7 +40,9 @@ With k8s-dqlite `v1.2.0` you will need to enable the `metrics` endpoint first be
 
 Starting with k8s-dqlite `v1.2.0`, [Otel](https://opentelemetry.io/) can be used to gather insights on
 traces on queries to Dqlite using a tool like Jaeger.
-To gather insights on traces locally, you can start the Jaeger all-in-one container by running the script under `k8s-dqlite/hack/jaeger.sh` and navigate to `http://localhost:16686`.
+To gather insights on traces and metrics locally, run `docker-compose up` in the `./hack/otel` directory.
+This sets up the Otel collector, Jaeger, and Prometheus. Navigate to `http://localhost:16686` to view the traces
+in Jaeger and to `http://localhost:9090` to view the metrics in Prometheus.
 
 ## Connection Pool Configuration
 

go.mod

@@ -1,6 +1,8 @@
 module github.com/canonical/k8s-dqlite
 
-go 1.21
+go 1.22
+
+toolchain go1.22.8
 
 require (
 	github.com/canonical/go-dqlite v1.22.0
@@ -14,15 +16,15 @@ require (
 	go.etcd.io/etcd/client/pkg/v3 v3.5.12
 	go.etcd.io/etcd/client/v3 v3.5.12
 	go.etcd.io/etcd/server/v3 v3.5.12
-	go.opentelemetry.io/otel v1.28.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0
-	go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0
-	go.opentelemetry.io/otel/metric v1.28.0
-	go.opentelemetry.io/otel/sdk v1.28.0
-	go.opentelemetry.io/otel/sdk/metric v1.28.0
-	go.opentelemetry.io/otel/trace v1.28.0
-	golang.org/x/sys v0.21.0
-	google.golang.org/grpc v1.65.0
+	go.opentelemetry.io/otel v1.31.0
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.31.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.31.0
+	go.opentelemetry.io/otel/metric v1.31.0
+	go.opentelemetry.io/otel/sdk v1.31.0
+	go.opentelemetry.io/otel/sdk/metric v1.31.0
+	go.opentelemetry.io/otel/trace v1.31.0
+	golang.org/x/sys v0.26.0
+	google.golang.org/grpc v1.67.1
 	gopkg.in/yaml.v2 v2.4.0
 )
 
@@ -47,7 +49,7 @@ require (
 	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jonboulle/clockwork v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
@@ -65,19 +67,19 @@ require (
 	go.etcd.io/etcd/pkg/v3 v3.5.12 // indirect
 	go.etcd.io/etcd/raft/v3 v3.5.12 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
-	golang.org/x/crypto v0.24.0 // indirect
-	golang.org/x/net v0.26.0 // indirect
-	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/crypto v0.28.0 // indirect
+	golang.org/x/net v0.30.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/text v0.19.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect
-	google.golang.org/protobuf v1.34.2 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9 // indirect
+	google.golang.org/protobuf v1.35.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect