feat(v2): webhook 投递硬化——指数退避 + 最大次数死信 + 告警钩子(替裸 60s 猛敲)

This commit is contained in:
wangjia
2026-07-10 17:25:21 +08:00
parent 6427e8c848
commit c73f7ffaa9
5 changed files with 227 additions and 16 deletions
+60 -2
View File
@@ -6,6 +6,7 @@ import (
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/wangjia/pay/config"
"github.com/wangjia/pay/internal/model"
@@ -72,6 +73,60 @@ func TestNotifierDeliversSignedEvent(t *testing.T) {
}
}
// 业务方持续 500:每次失败按指数退避重排;时钟推进后才重投;达上限标死信 + 告警。
func TestNotifierBackoffThenDeadWithAlert(t *testing.T) {
var hits int
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits++
w.WriteHeader(http.StatusInternalServerError)
}))
defer srv.Close()
clk := time.Date(2026, 7, 10, 12, 0, 0, 0, time.UTC)
nowFn := func() time.Time { return clk }
var alerted []string
ws := store.NewWebhookStore(model.OpenTestDB(t))
n := webhook.NewNotifier(ws,
func(string) (config.BizSystemConfig, bool) {
return config.BizSystemConfig{CallbackURL: srv.URL, Secret: "x"}, true
},
func(string) (bool, error) { return true, nil },
webhook.WithClock(nowFn),
webhook.WithBaseBackoff(time.Second),
webhook.WithMaxBackoff(4*time.Second),
webhook.WithMaxAttempts(3),
webhook.WithAlerter(func(d *store.WebhookDeliveryRow, reason string) { alerted = append(alerted, d.OutTradeNo) }),
)
_ = n.Enqueue("PAY-D", "pangolin", "payment.succeeded", "", map[string]any{"event_type": "payment.succeeded"})
// 尝试 1:失败 → attempts=1,退避到 +1s。
if sent, _ := n.DeliverPending(10); sent != 0 || hits != 1 {
t.Fatalf("try1 sent=%d hits=%d", sent, hits)
}
// 退避窗内不投。
if sent, _ := n.DeliverPending(10); sent != 0 || hits != 1 {
t.Fatalf("退避窗内不应再敲, hits=%d", hits)
}
// 推进越过退避;尝试 2 失败 → attempts=2,退避到 +2s。
clk = clk.Add(2 * time.Second)
if sent, _ := n.DeliverPending(10); sent != 0 || hits != 2 {
t.Fatalf("try2 hits=%d", hits)
}
// 推进;尝试 3 失败 → attempts 达 maxAttempts(3)→ 死信 + 告警。
clk = clk.Add(4 * time.Second)
if sent, _ := n.DeliverPending(10); sent != 0 || hits != 3 {
t.Fatalf("try3 hits=%d", hits)
}
if len(alerted) != 1 || alerted[0] != "PAY-D" {
t.Fatalf("死信应触发告警一次, got %+v", alerted)
}
// 已死信:无论时钟怎么走都不再投。
clk = clk.Add(time.Hour)
if sent, _ := n.DeliverPending(10); sent != 0 || hits != 3 {
t.Fatalf("死信后不应再投, hits=%d", hits)
}
}
func TestNotifierRetriesOnFailure(t *testing.T) {
var hits int
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -80,20 +135,23 @@ func TestNotifierRetriesOnFailure(t *testing.T) {
}))
defer srv.Close()
clk := time.Date(2026, 7, 10, 12, 0, 0, 0, time.UTC)
ws := store.NewWebhookStore(model.OpenTestDB(t))
n := webhook.NewNotifier(ws, func(string) (config.BizSystemConfig, bool) {
return config.BizSystemConfig{CallbackURL: srv.URL, Secret: "x"}, true
}, func(string) (bool, error) { return true, nil })
}, func(string) (bool, error) { return true, nil },
webhook.WithClock(func() time.Time { return clk }),
webhook.WithBaseBackoff(time.Second))
_ = n.Enqueue("PAY-3", "pangolin", "payment.succeeded", "", map[string]any{"event_type": "payment.succeeded"})
if sent, _ := n.DeliverPending(10); sent != 0 {
t.Fatalf("失败不应算投递成功, got %d", sent)
}
// 仍待投递,可被下一轮重试兜底
pend, _ := ws.ListUndelivered(10)
if len(pend) != 1 || pend[0].Attempts != 1 {
t.Fatalf("失败后应留队重试, got %+v", pend)
}
clk = clk.Add(2 * time.Second) // 越过退避窗
if _, _ = n.DeliverPending(10); hits < 2 {
t.Fatalf("应重试第二次, hits=%d", hits)
}