feat: Improve OpenTelemetry trace instrumentation

caddytest/integration/caddyfile_adapt/tracing.caddyfiletest

@@ -33,4 +33,4 @@
 			}
 		}
 	}
-}
+}

modules/caddyhttp/reverseproxy/reverseproxy.go

@@ -32,6 +32,7 @@ import (
 	"sync"
 	"time"
 
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 	"golang.org/x/net/http/httpguts"
@@ -810,8 +811,10 @@ func (h *Handler) reverseProxy(rw http.ResponseWriter, req *http.Request, origRe
 
 	// do the round-trip; emit debug log with values we know are
 	// safe, or if there is no error, emit fuller log entry
+	// we are wrapping the RoundTripper to make it observable
 	start := time.Now()
-	res, err := h.Transport.RoundTrip(req)
+	otelTransport := otelhttp.NewTransport(h.Transport)
+	res, err := otelTransport.RoundTrip(req)
 	duration := time.Since(start)
 	logger := h.logger.With(
 		zap.String("upstream", di.Upstream.String()),

modules/caddyhttp/tracing/module.go

@@ -25,7 +25,8 @@ func init() {
 type Tracing struct {
 	// SpanName is a span name. It should follow the naming guidelines here:
 	// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#span
-	SpanName string `json:"span"`
+	SpanName                 string `json:"span"`
+	InjectServerTimingHeader bool   `json:"server_timing,omitempty"`
 
 	// otel implements opentelemetry related logic.
 	otel openTelemetryWrapper
@@ -46,7 +47,7 @@ func (ot *Tracing) Provision(ctx caddy.Context) error {
 	ot.logger = ctx.Logger()
 
 	var err error
-	ot.otel, err = newOpenTelemetryWrapper(ctx, ot.SpanName)
+	ot.otel, err = newOpenTelemetryWrapper(ctx, ot.SpanName, ot.InjectServerTimingHeader)
 
 	return err
 }
@@ -68,6 +69,7 @@ func (ot *Tracing) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyh
 // UnmarshalCaddyfile sets up the module from Caddyfile tokens. Syntax:
 //
 //	tracing {
+//	    [injectServerTimingHeader <bool>]
 //	    [span <span_name>]
 //	}
 func (ot *Tracing) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
@@ -94,12 +96,19 @@ func (ot *Tracing) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 	}
 
 	for d.NextBlock(0) {
-		if dst, ok := paramsMap[d.Val()]; ok {
-			if err := setParameter(d, dst); err != nil {
-				return err
+		switch d.Val() {
+		case "server_timing":
+			if d.NextArg() {
+				ot.InjectServerTimingHeader = true
+			}
+		default:
+			if dst, ok := paramsMap[d.Val()]; ok {
+				if err := setParameter(d, dst); err != nil {
+					return err
+				}
+			} else {
+				return d.ArgErr()
 			}
-		} else {
-			return d.ArgErr()
 		}
 	}
 	return nil

modules/caddyhttp/tracing/tracer.go

@@ -3,6 +3,7 @@ package tracing
 import (
 	"context"
 	"fmt"
+	"go.opentelemetry.io/otel"
 	"net/http"
 
 	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
@@ -37,20 +38,19 @@ type openTelemetryWrapper struct {
 
 	handler http.Handler
 
-	spanName string
+	spanName                 string
+	injectServerTimingHeader bool
 }
 
 // newOpenTelemetryWrapper is responsible for the openTelemetryWrapper initialization using provided configuration.
-func newOpenTelemetryWrapper(
-	ctx context.Context,
-	spanName string,
-) (openTelemetryWrapper, error) {
+func newOpenTelemetryWrapper(ctx context.Context, spanName string, injectServerTimingHeader bool) (openTelemetryWrapper, error) {
 	if spanName == "" {
 		spanName = defaultSpanName
 	}
 
 	ot := openTelemetryWrapper{
-		spanName: spanName,
+		injectServerTimingHeader: injectServerTimingHeader,
+		spanName:                 spanName,
 	}
 
 	version, _ := caddy.Version()
@@ -64,7 +64,9 @@ func newOpenTelemetryWrapper(
 		return ot, fmt.Errorf("creating trace exporter error: %w", err)
 	}
 
-	ot.propagators = autoprop.NewTextMapPropagator()
+	prop := autoprop.NewTextMapPropagator()
+	otel.SetTextMapPropagator(prop)
+	ot.propagators = prop
 
 	tracerProvider := globalTracerProvider.getTracerProvider(
 		sdktrace.WithBatcher(traceExporter),
@@ -84,15 +86,24 @@ func newOpenTelemetryWrapper(
 // serveHTTP injects a tracing context and call the next handler.
 func (ot *openTelemetryWrapper) serveHTTP(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
-	ot.propagators.Inject(ctx, propagation.HeaderCarrier(r.Header))
+
 	spanCtx := trace.SpanContextFromContext(ctx)
 	if spanCtx.IsValid() {
 		traceID := spanCtx.TraceID().String()
+		spanID := spanCtx.SpanID().String()
 		// Add a trace_id placeholder, accessible via `{http.vars.trace_id}`.
 		caddyhttp.SetVar(ctx, "trace_id", traceID)
+		// Add a span_id placeholder, accessible via `{http.vars.span_id}`.
+		caddyhttp.SetVar(ctx, "span_id", spanID)
 		// Add the trace id to the log fields for the request.
 		if extra, ok := ctx.Value(caddyhttp.ExtraLogFieldsCtxKey).(*caddyhttp.ExtraLogFields); ok {
 			extra.Add(zap.String("traceID", traceID))
+			extra.Add(zap.String("spanID", spanID))
+		}
+
+		// Add the server-timing header so clients can make the connection
+		if ot.injectServerTimingHeader {
+			w.Header().Set("server-timing", fmt.Sprintf("traceparent;desc=\"00-%s-%s-%s\"", traceID, spanID, spanCtx.TraceFlags().String()))
 		}
 	}
 	next := ctx.Value(nextCallCtxKey).(*nextCall)

modules/caddyhttp/tracing/tracer_test.go

@@ -14,9 +14,7 @@ func TestOpenTelemetryWrapper_newOpenTelemetryWrapper(t *testing.T) {
 	var otw openTelemetryWrapper
 	var err error
 
-	if otw, err = newOpenTelemetryWrapper(ctx,
-		"",
-	); err != nil {
+	if otw, err = newOpenTelemetryWrapper(ctx, "", false); err != nil {
 		t.Errorf("newOpenTelemetryWrapper() error = %v", err)
 		t.FailNow()
 	}