diff --git a/service/ccm/credential.go b/service/ccm/credential.go index d5cae9e1e..b73261717 100644 --- a/service/ccm/credential.go +++ b/service/ccm/credential.go @@ -60,6 +60,7 @@ type credentialState struct { remotePlanWeight float64 lastUpdated time.Time consecutivePollFailures int + usageAPIRetryDelay time.Duration unavailable bool lastCredentialLoadAttempt time.Time lastCredentialLoadError string diff --git a/service/ccm/credential_default.go b/service/ccm/credential_default.go index c44ec4103..a6ed7ec87 100644 --- a/service/ccm/credential_default.go +++ b/service/ccm/credential_default.go @@ -436,8 +436,12 @@ func (c *defaultCredential) incrementPollFailures() { func (c *defaultCredential) pollBackoff(baseInterval time.Duration) time.Duration { c.stateAccess.RLock() failures := c.state.consecutivePollFailures + retryDelay := c.state.usageAPIRetryDelay c.stateAccess.RUnlock() if failures <= 0 { + if retryDelay > 0 { + return retryDelay + } return baseInterval } backoff := failedPollRetryInterval * time.Duration(1<<(failures-1)) @@ -518,7 +522,18 @@ func (c *defaultCredential) pollUsage(ctx context.Context) { if response.StatusCode != http.StatusOK { if response.StatusCode == http.StatusTooManyRequests { - c.logger.Warn("poll usage for ", c.tag, ": rate limited") + retryDelay := time.Minute + if retryAfter := response.Header.Get("Retry-After"); retryAfter != "" { + seconds, err := strconv.ParseInt(retryAfter, 10, 64) + if err == nil && seconds > 0 { + retryDelay = time.Duration(seconds) * time.Second + } + } + c.logger.Warn("poll usage for ", c.tag, ": usage API rate limited, retry in ", log.FormatDuration(retryDelay)) + c.stateAccess.Lock() + c.state.usageAPIRetryDelay = retryDelay + c.stateAccess.Unlock() + return } body, _ := io.ReadAll(response.Body) c.logger.Debug("poll usage for ", c.tag, ": status ", response.StatusCode, " ", string(body)) @@ -548,6 +563,7 @@ func (c *defaultCredential) pollUsage(ctx context.Context) { oldFiveHour := c.state.fiveHourUtilization oldWeekly := c.state.weeklyUtilization c.state.consecutivePollFailures = 0 + c.state.usageAPIRetryDelay = 0 c.state.fiveHourUtilization = usageResponse.FiveHour.Utilization if !usageResponse.FiveHour.ResetsAt.IsZero() { c.state.fiveHourReset = usageResponse.FiveHour.ResetsAt diff --git a/service/ocm/credential.go b/service/ocm/credential.go index e0ad9f565..80c094cdb 100644 --- a/service/ocm/credential.go +++ b/service/ocm/credential.go @@ -62,6 +62,7 @@ type credentialState struct { remotePlanWeight float64 lastUpdated time.Time consecutivePollFailures int + usageAPIRetryDelay time.Duration unavailable bool lastCredentialLoadAttempt time.Time lastCredentialLoadError string diff --git a/service/ocm/credential_default.go b/service/ocm/credential_default.go index b82af9d20..70b6eb6c1 100644 --- a/service/ocm/credential_default.go +++ b/service/ocm/credential_default.go @@ -493,8 +493,12 @@ func (c *defaultCredential) incrementPollFailures() { func (c *defaultCredential) pollBackoff(baseInterval time.Duration) time.Duration { c.stateAccess.RLock() failures := c.state.consecutivePollFailures + retryDelay := c.state.usageAPIRetryDelay c.stateAccess.RUnlock() if failures <= 0 { + if retryDelay > 0 { + return retryDelay + } return baseInterval } backoff := failedPollRetryInterval * time.Duration(1<<(failures-1)) @@ -618,7 +622,18 @@ func (c *defaultCredential) pollUsage(ctx context.Context) { if response.StatusCode != http.StatusOK { if response.StatusCode == http.StatusTooManyRequests { - c.logger.Warn("poll usage for ", c.tag, ": rate limited") + retryDelay := time.Minute + if retryAfter := response.Header.Get("Retry-After"); retryAfter != "" { + seconds, err := strconv.ParseInt(retryAfter, 10, 64) + if err == nil && seconds > 0 { + retryDelay = time.Duration(seconds) * time.Second + } + } + c.logger.Warn("poll usage for ", c.tag, ": usage API rate limited, retry in ", log.FormatDuration(retryDelay)) + c.stateAccess.Lock() + c.state.usageAPIRetryDelay = retryDelay + c.stateAccess.Unlock() + return } body, _ := io.ReadAll(response.Body) c.logger.Debug("poll usage for ", c.tag, ": status ", response.StatusCode, " ", string(body)) @@ -649,6 +664,7 @@ func (c *defaultCredential) pollUsage(ctx context.Context) { oldFiveHour := c.state.fiveHourUtilization oldWeekly := c.state.weeklyUtilization c.state.consecutivePollFailures = 0 + c.state.usageAPIRetryDelay = 0 if usageResponse.RateLimit != nil { if w := usageResponse.RateLimit.PrimaryWindow; w != nil { c.state.fiveHourUtilization = w.UsedPercent