From a61b6b70ba3f9239e5321909fd2b05e6dff18a88 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Tue, 19 Sep 2023 10:11:06 -0600 Subject: [PATCH 1/4] Implement grace period, but probably needs sync --- modules/caddyhttp/reverseproxy/upstreams.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/modules/caddyhttp/reverseproxy/upstreams.go b/modules/caddyhttp/reverseproxy/upstreams.go index 528e2c5ef72..8c4d9dde040 100644 --- a/modules/caddyhttp/reverseproxy/upstreams.go +++ b/modules/caddyhttp/reverseproxy/upstreams.go @@ -49,6 +49,13 @@ type SRVUpstreams struct { // Results are cached between lookups. Default: 1m Refresh caddy.Duration `json:"refresh,omitempty"` + // If > 0 and there is an error with the lookup, + // continue to use the cached results for up to + // this long before trying again, (even though they + // are stale) instead of returning an error to the + // client. Default: 0s. + GracePeriod caddy.Duration `json:"grace_period,omitempty"` + // Configures the DNS resolver used to resolve the // SRV address to SRV records. Resolver *UpstreamResolver `json:"resolver,omitempty"` @@ -140,6 +147,11 @@ func (su SRVUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) { // out and an error will be returned alongside the remaining results, if any." Thus, we // only return an error if no records were also returned. if len(records) == 0 { + if su.GracePeriod > 0 { + su.logger.Error("SRV lookup failed; using previously cached", zap.Error(err)) + cached.freshness = time.Now() // TODO: This is probably racey... + return allNew(cached.upstreams), nil + } return nil, err } su.logger.Warn("SRV records filtered", zap.Error(err)) From 91e5a338216eb693e9c48de33b5d4eeb3bea28b3 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Thu, 21 Sep 2023 11:36:26 -0600 Subject: [PATCH 2/4] Update cached freshness value --- modules/caddyhttp/reverseproxy/upstreams.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/caddyhttp/reverseproxy/upstreams.go b/modules/caddyhttp/reverseproxy/upstreams.go index 8c4d9dde040..87b45316dc6 100644 --- a/modules/caddyhttp/reverseproxy/upstreams.go +++ b/modules/caddyhttp/reverseproxy/upstreams.go @@ -149,7 +149,8 @@ func (su SRVUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) { if len(records) == 0 { if su.GracePeriod > 0 { su.logger.Error("SRV lookup failed; using previously cached", zap.Error(err)) - cached.freshness = time.Now() // TODO: This is probably racey... + cached.freshness = time.Now() + srvs[suAddr] = cached return allNew(cached.upstreams), nil } return nil, err From f759a12888afb1eb7d91c0e9289b19f9ee1bee68 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Thu, 21 Sep 2023 11:47:15 -0600 Subject: [PATCH 3/4] D'oh, actually use the grace period --- modules/caddyhttp/reverseproxy/upstreams.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/caddyhttp/reverseproxy/upstreams.go b/modules/caddyhttp/reverseproxy/upstreams.go index 87b45316dc6..756e36bb331 100644 --- a/modules/caddyhttp/reverseproxy/upstreams.go +++ b/modules/caddyhttp/reverseproxy/upstreams.go @@ -149,7 +149,7 @@ func (su SRVUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) { if len(records) == 0 { if su.GracePeriod > 0 { su.logger.Error("SRV lookup failed; using previously cached", zap.Error(err)) - cached.freshness = time.Now() + cached.freshness = time.Now().Add(-time.Duration(su.Refresh) - time.Duration(su.GracePeriod)) srvs[suAddr] = cached return allNew(cached.upstreams), nil } From 5880c3b801d5199843fa488db3347c3aa98e6dc2 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Tue, 5 Mar 2024 11:44:09 -0700 Subject: [PATCH 4/4] Fix freshness math --- modules/caddyhttp/reverseproxy/upstreams.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/caddyhttp/reverseproxy/upstreams.go b/modules/caddyhttp/reverseproxy/upstreams.go index 2fb916b4d78..46e45c64652 100644 --- a/modules/caddyhttp/reverseproxy/upstreams.go +++ b/modules/caddyhttp/reverseproxy/upstreams.go @@ -149,7 +149,7 @@ func (su SRVUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) { if len(records) == 0 { if su.GracePeriod > 0 { su.logger.Error("SRV lookup failed; using previously cached", zap.Error(err)) - cached.freshness = time.Now().Add(-time.Duration(su.Refresh) - time.Duration(su.GracePeriod)) + cached.freshness = time.Now().Add(time.Duration(su.GracePeriod) - time.Duration(su.Refresh)) srvs[suAddr] = cached return allNew(cached.upstreams), nil }