reverseproxy: Fix retries on "upstreams unavailable" error (#5841)

This commit is contained in:
Francis Lavoie 2023-10-10 18:07:20 -04:00 committed by GitHub
parent df99502977
commit 2a6859a5e4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 17 additions and 9 deletions

View File

@ -357,7 +357,7 @@ func (h *Handler) Provision(ctx caddy.Context) error {
// set defaults on passive health checks, if necessary // set defaults on passive health checks, if necessary
if h.HealthChecks.Passive != nil { if h.HealthChecks.Passive != nil {
h.HealthChecks.Passive.logger = h.logger.Named("health_checker.passive") h.HealthChecks.Passive.logger = h.logger.Named("health_checker.passive")
if h.HealthChecks.Passive.FailDuration > 0 && h.HealthChecks.Passive.MaxFails == 0 { if h.HealthChecks.Passive.MaxFails == 0 {
h.HealthChecks.Passive.MaxFails = 1 h.HealthChecks.Passive.MaxFails = 1
} }
} }
@ -480,7 +480,7 @@ func (h *Handler) proxyLoopIteration(r *http.Request, origReq *http.Request, w h
upstream := h.LoadBalancing.SelectionPolicy.Select(upstreams, r, w) upstream := h.LoadBalancing.SelectionPolicy.Select(upstreams, r, w)
if upstream == nil { if upstream == nil {
if proxyErr == nil { if proxyErr == nil {
proxyErr = caddyhttp.Error(http.StatusServiceUnavailable, fmt.Errorf("no upstreams available")) proxyErr = caddyhttp.Error(http.StatusServiceUnavailable, noUpstreamsAvailable)
} }
if !h.LoadBalancing.tryAgain(h.ctx, start, retries, proxyErr, r) { if !h.LoadBalancing.tryAgain(h.ctx, start, retries, proxyErr, r) {
return true, proxyErr return true, proxyErr
@ -1010,19 +1010,25 @@ func (lb LoadBalancing) tryAgain(ctx caddy.Context, start time.Time, retries int
// should be safe to retry, since without a connection, no // should be safe to retry, since without a connection, no
// HTTP request can be transmitted; but if the error is not // HTTP request can be transmitted; but if the error is not
// specifically a dialer error, we need to be careful // specifically a dialer error, we need to be careful
if _, ok := proxyErr.(DialError); proxyErr != nil && !ok { if proxyErr != nil {
_, isDialError := proxyErr.(DialError)
herr, isHandlerError := proxyErr.(caddyhttp.HandlerError)
// if the error occurred after a connection was established, // if the error occurred after a connection was established,
// we have to assume the upstream received the request, and // we have to assume the upstream received the request, and
// retries need to be carefully decided, because some requests // retries need to be carefully decided, because some requests
// are not idempotent // are not idempotent
if !isDialError && !(isHandlerError && errors.Is(herr, noUpstreamsAvailable)) {
if lb.RetryMatch == nil && req.Method != "GET" { if lb.RetryMatch == nil && req.Method != "GET" {
// by default, don't retry requests if they aren't GET // by default, don't retry requests if they aren't GET
return false return false
} }
if !lb.RetryMatch.AnyMatch(req) { if !lb.RetryMatch.AnyMatch(req) {
return false return false
} }
} }
}
// fast path; if the interval is zero, we don't need to wait // fast path; if the interval is zero, we don't need to wait
if lb.TryInterval == 0 { if lb.TryInterval == 0 {
@ -1421,6 +1427,8 @@ func (c ignoreClientGoneContext) Err() error {
// from the proxy handler. // from the proxy handler.
const proxyHandleResponseContextCtxKey caddy.CtxKey = "reverse_proxy_handle_response_context" const proxyHandleResponseContextCtxKey caddy.CtxKey = "reverse_proxy_handle_response_context"
var noUpstreamsAvailable = fmt.Errorf("no upstreams available")
// Interface guards // Interface guards
var ( var (
_ caddy.Provisioner = (*Handler)(nil) _ caddy.Provisioner = (*Handler)(nil)