// The MIT License // // Copyright (c) 2020 Temporal Technologies Inc. All rights reserved. // // Copyright (c) 2020 Uber Technologies, Inc. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal // in the Software without restriction, including without limitation the rights // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell // copies of the Software, and to permit persons to whom the Software is // furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. package retry import ( "context" "math" "time" "github.com/gogo/status" grpc_retry "github.com/grpc-ecosystem/go-grpc-middleware/retry" "github.com/grpc-ecosystem/go-grpc-middleware/util/backoffutils" "google.golang.org/grpc" "google.golang.org/grpc/codes" ) const ( // UnlimitedMaximumAttempts when maximum attempts is set to this special value, then the number of attempts is unlimited. UnlimitedMaximumAttempts = 0 // UnlimitedInterval when maximum interval is set to this special value, then there is no upper bound on the retry delay. // Should not be used together with unlimited attempts as resulting retry interval can grow to unreasonable values. UnlimitedInterval = 0 // DefaultBackoffCoefficient is default backOffCoefficient for retryPolicy DefaultBackoffCoefficient = 2.0 // DefaultMaximumInterval is default maximum amount of time for an individual retry. DefaultMaximumInterval = 10 * time.Second // DefaultExpirationInterval is default expiration time for all retry attempts. DefaultExpirationInterval = time.Minute // DefaultMaximumAttempts is default maximum number of attempts. DefaultMaximumAttempts = UnlimitedMaximumAttempts // DefaultJitter is a default jitter applied on the backoff interval for delay randomization. DefaultJitter = 0.2 ) type ( // GrpcRetryConfig defines required configuration for exponential backoff function that is supplied to gRPC retrier. GrpcRetryConfig struct { initialInterval time.Duration backoffCoefficient float64 maximumInterval time.Duration expirationInterval time.Duration jitter float64 maximumAttempts int } contextKey struct{} ) func (ck contextKey) String() string { return "RetryConfig" } // SetBackoffCoefficient sets rate at which backoff coefficient will change. func (g *GrpcRetryConfig) SetBackoffCoefficient(backoffCoefficient float64) { g.backoffCoefficient = backoffCoefficient } // SetMaximumInterval defines maximum amount of time between attempts. func (g *GrpcRetryConfig) SetMaximumInterval(maximumInterval time.Duration) { g.maximumInterval = maximumInterval } // SetExpirationInterval defines total amount of time that can be used for all retry attempts. // Note that this value is ignored if deadline is set on the context. func (g *GrpcRetryConfig) SetExpirationInterval(expirationInterval time.Duration) { g.expirationInterval = expirationInterval } // SetJitter defines level of randomization for each delay interval. For example 0.2 would mex target +- 20% func (g *GrpcRetryConfig) SetJitter(jitter float64) { g.jitter = jitter } // SetMaximumAttempts defines maximum total number of retry attempts. func (g *GrpcRetryConfig) SetMaximumAttempts(maximumAttempts int) { g.maximumAttempts = maximumAttempts } // NewGrpcRetryConfig creates new retry config with specified initial interval and defaults for other parameters. // Use SetXXX functions on this config in order to customize values. func NewGrpcRetryConfig(initialInterval time.Duration) *GrpcRetryConfig { return &GrpcRetryConfig{ initialInterval: initialInterval, backoffCoefficient: DefaultBackoffCoefficient, maximumInterval: DefaultMaximumInterval, expirationInterval: DefaultExpirationInterval, jitter: DefaultJitter, maximumAttempts: DefaultMaximumAttempts, } } var ( // ConfigKey context key for GrpcRetryConfig ConfigKey = contextKey{} // gRPC response codes that represent retryable errors. // The following status codes are never generated by the library: // INVALID_ARGUMENT, NOT_FOUND, ALREADY_EXISTS, FAILED_PRECONDITION, ABORTED, OUT_OF_RANGE, DATA_LOSS retryableCodes = []codes.Code{codes.Aborted, codes.DeadlineExceeded, codes.Internal, codes.ResourceExhausted, codes.Unavailable, codes.Unknown} ) // NewRetryOptionsInterceptor creates a new gRPC interceptor that populates retry options for each call based on values // provided in the context. func NewRetryOptionsInterceptor() grpc.UnaryClientInterceptor { return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { if rc, ok := ctx.Value(ConfigKey).(*GrpcRetryConfig); ok { if _, ok := ctx.Deadline(); !ok { deadlineCtx, cancel := context.WithDeadline(ctx, time.Now().Add(rc.expirationInterval)) defer cancel() ctx = deadlineCtx } // Populate backoff function, which provides retrier with the delay for each attempt. opts = append(opts, grpc_retry.WithBackoff(func(attempt uint) time.Duration { next := float64(rc.initialInterval) * math.Pow(rc.backoffCoefficient, float64(attempt)) if rc.maximumInterval != UnlimitedInterval { next = math.Min(next, float64(rc.maximumInterval)) } return backoffutils.JitterUp(time.Duration(next), rc.jitter) })) // Max attempts is a required parameter in grpc retry interceptor, // if it's set to zero then no retries will be made. if rc.maximumAttempts != UnlimitedMaximumAttempts { opts = append(opts, grpc_retry.WithMax(uint(rc.maximumAttempts))) } else { opts = append(opts, grpc_retry.WithMax(math.MaxUint32)) } // We have to deal with plain gRPC error codes instead of service errors here as actual error translation // happens after invoker is called below and invoker must have correct retry options right away in order to // supply them to the gRPC retrier. opts = append(opts, grpc_retry.WithCodes(retryableCodes...)) } else { // Do not retry if retry config is not set. opts = append(opts, grpc_retry.Disable()) } return invoker(ctx, method, req, reply, cc, opts...) } } // IsStatusCodeRetryable returns true if error code in the status is retryable. func IsStatusCodeRetryable(status *status.Status) bool { for _, retryable := range retryableCodes { if retryable == status.Code() { return true } } return false }