feat(backend): 公开接口反爬轻量版——per-IP 日配额 + 列表收紧(todo #3)
DailyQuota 二级闸(单品 1000/日、店铺列表 300/日,0=关;出错放行不误伤); 店铺列表 page_size 上限 50→20(客户端固定传 20 无破坏);公开响应敏感字段 (cost/purchase_price/profit)零暴露回归测试。 明确不做:签名链接(QR 已印刷+UUIDv4 不可枚举)、滑块(杀零门槛分享)、登录墙。 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -232,7 +232,8 @@ func (h *PublicHandler) ListShopProducts(c *gin.Context) {
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
if pageSize < 1 || pageSize > 50 {
|
||||
// 上限 20(2026-07 反爬收紧:客户端固定传 20,调大只方便爬全店)
|
||||
if pageSize < 1 || pageSize > 20 {
|
||||
pageSize = 20
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/wangjia/jiu/backend/testutil"
|
||||
)
|
||||
|
||||
// 反爬收紧回归(2026-07):page_size 上限夹到 20;公开接口响应绝不出现
|
||||
// 成本类敏感字段名(白名单 DTO 防未来 Preload 全量 struct 回归泄露)。
|
||||
|
||||
func setupPublicFullRouter(db *gorm.DB) *gin.Engine {
|
||||
h := NewPublicHandler(db)
|
||||
r := gin.New()
|
||||
r.Use(gin.Recovery())
|
||||
r.GET("/api/v1/public/shops/:shop_code/products", h.ListShopProducts)
|
||||
r.GET("/api/v1/public/products/:public_id", h.GetProduct)
|
||||
return r
|
||||
}
|
||||
|
||||
func TestPublicListPageSizeClampedTo20(t *testing.T) {
|
||||
db := testutil.SetupTestDB()
|
||||
shop := testutil.CreateTestShop(db, "PUBPS")
|
||||
wh := testutil.CreateTestWarehouse(db, shop.ID, "仓")
|
||||
r := setupPublicFullRouter(db)
|
||||
|
||||
for i := 0; i < 25; i++ {
|
||||
p := testutil.CreateTestProduct(db, shop.ID, fmt.Sprintf("酒%02d", i))
|
||||
setPublicID(db, p.ID, fmt.Sprintf("pub-ps-%02d", i))
|
||||
addInventory(db, shop.ID, wh.ID, p.ID, 1)
|
||||
}
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
req, _ := http.NewRequest("GET", "/api/v1/public/shops/PUBPS/products?page_size=50", nil)
|
||||
r.ServeHTTP(w, req)
|
||||
require.Equal(t, http.StatusOK, w.Code)
|
||||
|
||||
var resp map[string]interface{}
|
||||
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp))
|
||||
assert.Equal(t, float64(25), resp["total"])
|
||||
data := resp["data"].([]interface{})
|
||||
assert.Len(t, data, 20, "page_size=50 应被夹到上限 20")
|
||||
assert.Equal(t, float64(20), resp["page_size"])
|
||||
}
|
||||
|
||||
// 公开响应不得出现的敏感字段名(成本/利润口径只对管理员,公开面零暴露)。
|
||||
var sensitiveFieldNames = []string{"cost", "purchase_price", "profit"}
|
||||
|
||||
func assertNoSensitiveFields(t *testing.T, body string) {
|
||||
t.Helper()
|
||||
lower := strings.ToLower(body)
|
||||
for _, f := range sensitiveFieldNames {
|
||||
assert.NotContains(t, lower, f, "公开接口响应不应含敏感字段名 %q", f)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicResponsesExcludeCostFields(t *testing.T) {
|
||||
db := testutil.SetupTestDB()
|
||||
shop := testutil.CreateTestShop(db, "PUBSEC")
|
||||
wh := testutil.CreateTestWarehouse(db, shop.ID, "仓")
|
||||
r := setupPublicFullRouter(db)
|
||||
|
||||
p := testutil.CreateTestProduct(db, shop.ID, "敏感字段酒")
|
||||
setPublicID(db, p.ID, "pub-sec-1")
|
||||
addInventory(db, shop.ID, wh.ID, p.ID, 3)
|
||||
|
||||
// 列表
|
||||
w := httptest.NewRecorder()
|
||||
req, _ := http.NewRequest("GET", "/api/v1/public/shops/PUBSEC/products", nil)
|
||||
r.ServeHTTP(w, req)
|
||||
require.Equal(t, http.StatusOK, w.Code)
|
||||
assertNoSensitiveFields(t, w.Body.String())
|
||||
|
||||
// 单品
|
||||
w2 := httptest.NewRecorder()
|
||||
req2, _ := http.NewRequest("GET", "/api/v1/public/products/pub-sec-1", nil)
|
||||
r.ServeHTTP(w2, req2)
|
||||
require.Equal(t, http.StatusOK, w2.Code)
|
||||
assertNoSensitiveFields(t, w2.Body.String())
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package middleware
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"github.com/wangjia/jiu/backend/config"
|
||||
"github.com/wangjia/jiu/backend/internal/ratelimit"
|
||||
)
|
||||
|
||||
// DailyQuota 同一 IP 的自然日累计配额——分钟级限流之上的第二道反爬闸。
|
||||
// key = "dq:<scope>:<yyyymmdd>:<ip>",TTL 26h(跨午夜/时钟偏移裕量,键按日期
|
||||
// 自然分片,旧键过期自灭)。limit<=0 或总开关关闭时放行;计数器出错(redis
|
||||
// 降级窗口等)也放行——宁可漏限不误伤真实客户。
|
||||
func DailyQuota(scope string, limit func() int) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
lim := limit()
|
||||
if !config.C.RateLimit.Enabled || lim <= 0 {
|
||||
c.Next()
|
||||
return
|
||||
}
|
||||
ip := c.ClientIP()
|
||||
if ip == "" {
|
||||
c.Next()
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
key := fmt.Sprintf("dq:%s:%s:%s", scope, now.Format("20060102"), ip)
|
||||
n, err := ratelimit.Default().Counter().Incr(key, 26*time.Hour)
|
||||
if err == nil && n > int64(lim) {
|
||||
c.Header("Retry-After", strconv.Itoa(secondsToMidnight(now)))
|
||||
c.AbortWithStatusJSON(http.StatusTooManyRequests,
|
||||
gin.H{"error": "今日访问次数已达上限,请明日再试"})
|
||||
return
|
||||
}
|
||||
c.Next()
|
||||
}
|
||||
}
|
||||
|
||||
// secondsToMidnight 距下一个本地零点的秒数(日配额的自然重置时刻)。
|
||||
func secondsToMidnight(now time.Time) int {
|
||||
next := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()).AddDate(0, 0, 1)
|
||||
return int(next.Sub(now).Seconds())
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package middleware
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"github.com/wangjia/jiu/backend/config"
|
||||
)
|
||||
|
||||
func newQuotaRouter(scope string, limit int) *gin.Engine {
|
||||
r := gin.New()
|
||||
r.GET("/", DailyQuota(scope, func() int { return limit }), func(c *gin.Context) {
|
||||
c.String(http.StatusOK, "ok")
|
||||
})
|
||||
return r
|
||||
}
|
||||
|
||||
func TestDailyQuotaExceeded429(t *testing.T) {
|
||||
enableRateLimit(t)
|
||||
r := newQuotaRouter("tq1", 3)
|
||||
for i := 0; i < 3; i++ {
|
||||
if w := hit(r, "7.7.7.7"); w.Code != http.StatusOK {
|
||||
t.Fatalf("配额内第 %d 发应放行,得到 %d", i+1, w.Code)
|
||||
}
|
||||
}
|
||||
w := hit(r, "7.7.7.7")
|
||||
if w.Code != http.StatusTooManyRequests {
|
||||
t.Fatalf("超日配额应 429,得到 %d", w.Code)
|
||||
}
|
||||
if w.Header().Get("Retry-After") == "" {
|
||||
t.Fatal("429 应带 Retry-After(到午夜秒数)")
|
||||
}
|
||||
// 不同 IP 独立
|
||||
if w := hit(r, "8.8.8.8"); w.Code != http.StatusOK {
|
||||
t.Fatalf("其他 IP 应不受影响,得到 %d", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDailyQuotaScopesIndependent(t *testing.T) {
|
||||
enableRateLimit(t)
|
||||
a := newQuotaRouter("tq2a", 1)
|
||||
b := newQuotaRouter("tq2b", 1)
|
||||
if w := hit(a, "9.9.9.9"); w.Code != http.StatusOK {
|
||||
t.Fatalf("scope a 首发应放行,得到 %d", w.Code)
|
||||
}
|
||||
if w := hit(b, "9.9.9.9"); w.Code != http.StatusOK {
|
||||
t.Fatalf("scope b 有独立配额池,应放行,得到 %d", w.Code)
|
||||
}
|
||||
if w := hit(a, "9.9.9.9"); w.Code != http.StatusTooManyRequests {
|
||||
t.Fatalf("scope a 第二发应 429,得到 %d", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDailyQuotaDisabledOrZeroPassthrough(t *testing.T) {
|
||||
// limit=0 → 该闸关闭
|
||||
enableRateLimit(t)
|
||||
r := newQuotaRouter("tq3", 0)
|
||||
for i := 0; i < 5; i++ {
|
||||
if w := hit(r, "10.0.0.1"); w.Code != http.StatusOK {
|
||||
t.Fatalf("limit=0 第 %d 发应放行,得到 %d", i+1, w.Code)
|
||||
}
|
||||
}
|
||||
// 总开关关闭 → 放行
|
||||
prev := config.C.RateLimit.Enabled
|
||||
config.C.RateLimit.Enabled = false
|
||||
defer func() { config.C.RateLimit.Enabled = prev }()
|
||||
r2 := newQuotaRouter("tq4", 1)
|
||||
for i := 0; i < 5; i++ {
|
||||
if w := hit(r2, "10.0.0.2"); w.Code != http.StatusOK {
|
||||
t.Fatalf("总开关关闭第 %d 发应放行,得到 %d", i+1, w.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -59,9 +59,12 @@ func Setup(r *gin.Engine, db *gorm.DB) {
|
||||
refreshIP := middleware.RateLimitByIP(middleware.PerMinute(rl.RefreshPerMin), rl.RefreshPerMin)
|
||||
registerIP := middleware.RateLimitByIP(middleware.PerMinute(rl.RegisterPerMin), rl.RegisterPerMin)
|
||||
errorsIP := middleware.RateLimitByIP(middleware.PerMinute(rl.ErrorsPerMin), rl.ErrorsPerMin)
|
||||
// 日配额二级闸(同一 IP 自然日累计,反爬):单品 API 与 OG 页同一配额池
|
||||
dqProduct := middleware.DailyQuota("product", func() int { return config.C.RateLimit.DailyProductPerIP })
|
||||
dqShopList := middleware.DailyQuota("shoplist", func() int { return config.C.RateLimit.DailyShopListPerIP })
|
||||
|
||||
// 公开商品详情页(注入 OG 标签的 Flutter index.html,供社交分享爬虫读取)
|
||||
r.GET("/product/:public_id", publicReadIP, publicH.ProductPage)
|
||||
r.GET("/product/:public_id", publicReadIP, dqProduct, publicH.ProductPage)
|
||||
|
||||
v1 := r.Group("/api/v1")
|
||||
|
||||
@@ -77,8 +80,8 @@ func Setup(r *gin.Engine, db *gorm.DB) {
|
||||
// 公开接口(无需登录):读接口防爬、写接口防刷
|
||||
public := v1.Group("/public")
|
||||
{
|
||||
public.GET("/products/:public_id", publicReadIP, publicH.GetProduct)
|
||||
public.GET("/shops/:shop_code/products", shopListIP, publicH.ListShopProducts)
|
||||
public.GET("/products/:public_id", publicReadIP, dqProduct, publicH.GetProduct)
|
||||
public.GET("/shops/:shop_code/products", shopListIP, dqShopList, publicH.ListShopProducts)
|
||||
public.GET("/release", publicReadIP, publicH.GetRelease)
|
||||
public.POST("/errors", errorsIP, errorReportH.Submit)
|
||||
public.POST("/register", registerIP, authH.Register)
|
||||
|
||||
Reference in New Issue
Block a user