From d1e5426bc8a9b08abf2937ab50698ad36b7afcfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Sat, 14 Mar 2026 18:26:51 +0800 Subject: [PATCH] ccm,ocm: add exponential backoff with cap for poll retry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace flat 1-minute poll retry interval with exponential backoff (1m → 2m → 4m → 5m cap). Suppress error logs after reaching the cap. --- service/ccm/credential_external.go | 17 +++++++++++++++-- service/ccm/credential_state.go | 22 +++++++++++++++++++--- service/ocm/credential_external.go | 17 +++++++++++++++-- service/ocm/credential_state.go | 22 +++++++++++++++++++--- 4 files changed, 68 insertions(+), 10 deletions(-) diff --git a/service/ccm/credential_external.go b/service/ccm/credential_external.go index 8d1c1a08d..e8fe31799 100644 --- a/service/ccm/credential_external.go +++ b/service/ccm/credential_external.go @@ -529,7 +529,9 @@ func (c *externalCredential) pollUsage(ctx context.Context) { return request, nil }) if err != nil { - c.logger.Error("poll usage for ", c.tag, ": ", err) + if !c.isPollBackoffAtCap() { + c.logger.Error("poll usage for ", c.tag, ": ", err) + } c.incrementPollFailures() return } @@ -609,7 +611,18 @@ func (c *externalCredential) pollBackoff(baseInterval time.Duration) time.Durati if failures <= 0 { return baseInterval } - return failedPollRetryInterval + backoff := failedPollRetryInterval * time.Duration(1<<(failures-1)) + if backoff > httpRetryMaxBackoff { + return httpRetryMaxBackoff + } + return backoff +} + +func (c *externalCredential) isPollBackoffAtCap() bool { + c.stateMutex.RLock() + defer c.stateMutex.RUnlock() + failures := c.state.consecutivePollFailures + return failures > 0 && failedPollRetryInterval*time.Duration(1<<(failures-1)) >= httpRetryMaxBackoff } func (c *externalCredential) incrementPollFailures() { diff --git a/service/ccm/credential_state.go b/service/ccm/credential_state.go index 6b1a766f2..81d559f3c 100644 --- a/service/ccm/credential_state.go +++ b/service/ccm/credential_state.go @@ -29,6 +29,7 @@ import ( const ( defaultPollInterval = 60 * time.Minute failedPollRetryInterval = time.Minute + httpRetryMaxBackoff = 5 * time.Minute ) const ( @@ -583,7 +584,18 @@ func (c *defaultCredential) pollBackoff(baseInterval time.Duration) time.Duratio if failures <= 0 { return baseInterval } - return failedPollRetryInterval + backoff := failedPollRetryInterval * time.Duration(1<<(failures-1)) + if backoff > httpRetryMaxBackoff { + return httpRetryMaxBackoff + } + return backoff +} + +func (c *defaultCredential) isPollBackoffAtCap() bool { + c.stateMutex.RLock() + defer c.stateMutex.RUnlock() + failures := c.state.consecutivePollFailures + return failures > 0 && failedPollRetryInterval*time.Duration(1<<(failures-1)) >= httpRetryMaxBackoff } func (c *defaultCredential) earliestReset() time.Time { @@ -616,7 +628,9 @@ func (c *defaultCredential) pollUsage(ctx context.Context) { accessToken, err := c.getAccessToken() if err != nil { - c.logger.Error("poll usage for ", c.tag, ": get token: ", err) + if !c.isPollBackoffAtCap() { + c.logger.Error("poll usage for ", c.tag, ": get token: ", err) + } c.incrementPollFailures() return } @@ -638,7 +652,9 @@ func (c *defaultCredential) pollUsage(ctx context.Context) { return request, nil }) if err != nil { - c.logger.Error("poll usage for ", c.tag, ": ", err) + if !c.isPollBackoffAtCap() { + c.logger.Error("poll usage for ", c.tag, ": ", err) + } c.incrementPollFailures() return } diff --git a/service/ocm/credential_external.go b/service/ocm/credential_external.go index f09716db3..0b60cff1e 100644 --- a/service/ocm/credential_external.go +++ b/service/ocm/credential_external.go @@ -568,7 +568,9 @@ func (c *externalCredential) pollUsage(ctx context.Context) { return request, nil }) if err != nil { - c.logger.Error("poll usage for ", c.tag, ": ", err) + if !c.isPollBackoffAtCap() { + c.logger.Error("poll usage for ", c.tag, ": ", err) + } c.incrementPollFailures() return } @@ -648,7 +650,18 @@ func (c *externalCredential) pollBackoff(baseInterval time.Duration) time.Durati if failures <= 0 { return baseInterval } - return failedPollRetryInterval + backoff := failedPollRetryInterval * time.Duration(1<<(failures-1)) + if backoff > httpRetryMaxBackoff { + return httpRetryMaxBackoff + } + return backoff +} + +func (c *externalCredential) isPollBackoffAtCap() bool { + c.stateMutex.RLock() + defer c.stateMutex.RUnlock() + failures := c.state.consecutivePollFailures + return failures > 0 && failedPollRetryInterval*time.Duration(1<<(failures-1)) >= httpRetryMaxBackoff } func (c *externalCredential) incrementPollFailures() { diff --git a/service/ocm/credential_state.go b/service/ocm/credential_state.go index d8f2e826a..58b734465 100644 --- a/service/ocm/credential_state.go +++ b/service/ocm/credential_state.go @@ -29,6 +29,7 @@ import ( const ( defaultPollInterval = 60 * time.Minute failedPollRetryInterval = time.Minute + httpRetryMaxBackoff = 5 * time.Minute ) const ( @@ -597,7 +598,18 @@ func (c *defaultCredential) pollBackoff(baseInterval time.Duration) time.Duratio if failures <= 0 { return baseInterval } - return failedPollRetryInterval + backoff := failedPollRetryInterval * time.Duration(1<<(failures-1)) + if backoff > httpRetryMaxBackoff { + return httpRetryMaxBackoff + } + return backoff +} + +func (c *defaultCredential) isPollBackoffAtCap() bool { + c.stateMutex.RLock() + defer c.stateMutex.RUnlock() + failures := c.state.consecutivePollFailures + return failures > 0 && failedPollRetryInterval*time.Duration(1<<(failures-1)) >= httpRetryMaxBackoff } func (c *defaultCredential) earliestReset() time.Time { @@ -633,7 +645,9 @@ func (c *defaultCredential) pollUsage(ctx context.Context) { accessToken, err := c.getAccessToken() if err != nil { - c.logger.Error("poll usage for ", c.tag, ": get token: ", err) + if !c.isPollBackoffAtCap() { + c.logger.Error("poll usage for ", c.tag, ": get token: ", err) + } c.incrementPollFailures() return } @@ -663,7 +677,9 @@ func (c *defaultCredential) pollUsage(ctx context.Context) { return request, nil }) if err != nil { - c.logger.Error("poll usage for ", c.tag, ": ", err) + if !c.isPollBackoffAtCap() { + c.logger.Error("poll usage for ", c.tag, ": ", err) + } c.incrementPollFailures() return }