feat(backend): 公开接口反爬轻量版——per-IP 日配额 + 列表收紧(todo #3)
DailyQuota 二级闸(单品 1000/日、店铺列表 300/日,0=关;出错放行不误伤); 店铺列表 page_size 上限 50→20(客户端固定传 20 无破坏);公开响应敏感字段 (cost/purchase_price/profit)零暴露回归测试。 明确不做:签名链接(QR 已印刷+UUIDv4 不可枚举)、滑块(杀零门槛分享)、登录墙。 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -232,7 +232,8 @@ func (h *PublicHandler) ListShopProducts(c *gin.Context) {
|
|||||||
if page < 1 {
|
if page < 1 {
|
||||||
page = 1
|
page = 1
|
||||||
}
|
}
|
||||||
if pageSize < 1 || pageSize > 50 {
|
// 上限 20(2026-07 反爬收紧:客户端固定传 20,调大只方便爬全店)
|
||||||
|
if pageSize < 1 || pageSize > 20 {
|
||||||
pageSize = 20
|
pageSize = 20
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,90 @@
|
|||||||
|
package handler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/gin-gonic/gin"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
|
||||||
|
"github.com/wangjia/jiu/backend/testutil"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 反爬收紧回归(2026-07):page_size 上限夹到 20;公开接口响应绝不出现
|
||||||
|
// 成本类敏感字段名(白名单 DTO 防未来 Preload 全量 struct 回归泄露)。
|
||||||
|
|
||||||
|
func setupPublicFullRouter(db *gorm.DB) *gin.Engine {
|
||||||
|
h := NewPublicHandler(db)
|
||||||
|
r := gin.New()
|
||||||
|
r.Use(gin.Recovery())
|
||||||
|
r.GET("/api/v1/public/shops/:shop_code/products", h.ListShopProducts)
|
||||||
|
r.GET("/api/v1/public/products/:public_id", h.GetProduct)
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPublicListPageSizeClampedTo20(t *testing.T) {
|
||||||
|
db := testutil.SetupTestDB()
|
||||||
|
shop := testutil.CreateTestShop(db, "PUBPS")
|
||||||
|
wh := testutil.CreateTestWarehouse(db, shop.ID, "仓")
|
||||||
|
r := setupPublicFullRouter(db)
|
||||||
|
|
||||||
|
for i := 0; i < 25; i++ {
|
||||||
|
p := testutil.CreateTestProduct(db, shop.ID, fmt.Sprintf("酒%02d", i))
|
||||||
|
setPublicID(db, p.ID, fmt.Sprintf("pub-ps-%02d", i))
|
||||||
|
addInventory(db, shop.ID, wh.ID, p.ID, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
req, _ := http.NewRequest("GET", "/api/v1/public/shops/PUBPS/products?page_size=50", nil)
|
||||||
|
r.ServeHTTP(w, req)
|
||||||
|
require.Equal(t, http.StatusOK, w.Code)
|
||||||
|
|
||||||
|
var resp map[string]interface{}
|
||||||
|
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp))
|
||||||
|
assert.Equal(t, float64(25), resp["total"])
|
||||||
|
data := resp["data"].([]interface{})
|
||||||
|
assert.Len(t, data, 20, "page_size=50 应被夹到上限 20")
|
||||||
|
assert.Equal(t, float64(20), resp["page_size"])
|
||||||
|
}
|
||||||
|
|
||||||
|
// 公开响应不得出现的敏感字段名(成本/利润口径只对管理员,公开面零暴露)。
|
||||||
|
var sensitiveFieldNames = []string{"cost", "purchase_price", "profit"}
|
||||||
|
|
||||||
|
func assertNoSensitiveFields(t *testing.T, body string) {
|
||||||
|
t.Helper()
|
||||||
|
lower := strings.ToLower(body)
|
||||||
|
for _, f := range sensitiveFieldNames {
|
||||||
|
assert.NotContains(t, lower, f, "公开接口响应不应含敏感字段名 %q", f)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPublicResponsesExcludeCostFields(t *testing.T) {
|
||||||
|
db := testutil.SetupTestDB()
|
||||||
|
shop := testutil.CreateTestShop(db, "PUBSEC")
|
||||||
|
wh := testutil.CreateTestWarehouse(db, shop.ID, "仓")
|
||||||
|
r := setupPublicFullRouter(db)
|
||||||
|
|
||||||
|
p := testutil.CreateTestProduct(db, shop.ID, "敏感字段酒")
|
||||||
|
setPublicID(db, p.ID, "pub-sec-1")
|
||||||
|
addInventory(db, shop.ID, wh.ID, p.ID, 3)
|
||||||
|
|
||||||
|
// 列表
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
req, _ := http.NewRequest("GET", "/api/v1/public/shops/PUBSEC/products", nil)
|
||||||
|
r.ServeHTTP(w, req)
|
||||||
|
require.Equal(t, http.StatusOK, w.Code)
|
||||||
|
assertNoSensitiveFields(t, w.Body.String())
|
||||||
|
|
||||||
|
// 单品
|
||||||
|
w2 := httptest.NewRecorder()
|
||||||
|
req2, _ := http.NewRequest("GET", "/api/v1/public/products/pub-sec-1", nil)
|
||||||
|
r.ServeHTTP(w2, req2)
|
||||||
|
require.Equal(t, http.StatusOK, w2.Code)
|
||||||
|
assertNoSensitiveFields(t, w2.Body.String())
|
||||||
|
}
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
package middleware
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/gin-gonic/gin"
|
||||||
|
|
||||||
|
"github.com/wangjia/jiu/backend/config"
|
||||||
|
"github.com/wangjia/jiu/backend/internal/ratelimit"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DailyQuota 同一 IP 的自然日累计配额——分钟级限流之上的第二道反爬闸。
|
||||||
|
// key = "dq:<scope>:<yyyymmdd>:<ip>",TTL 26h(跨午夜/时钟偏移裕量,键按日期
|
||||||
|
// 自然分片,旧键过期自灭)。limit<=0 或总开关关闭时放行;计数器出错(redis
|
||||||
|
// 降级窗口等)也放行——宁可漏限不误伤真实客户。
|
||||||
|
func DailyQuota(scope string, limit func() int) gin.HandlerFunc {
|
||||||
|
return func(c *gin.Context) {
|
||||||
|
lim := limit()
|
||||||
|
if !config.C.RateLimit.Enabled || lim <= 0 {
|
||||||
|
c.Next()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ip := c.ClientIP()
|
||||||
|
if ip == "" {
|
||||||
|
c.Next()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
key := fmt.Sprintf("dq:%s:%s:%s", scope, now.Format("20060102"), ip)
|
||||||
|
n, err := ratelimit.Default().Counter().Incr(key, 26*time.Hour)
|
||||||
|
if err == nil && n > int64(lim) {
|
||||||
|
c.Header("Retry-After", strconv.Itoa(secondsToMidnight(now)))
|
||||||
|
c.AbortWithStatusJSON(http.StatusTooManyRequests,
|
||||||
|
gin.H{"error": "今日访问次数已达上限,请明日再试"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
c.Next()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// secondsToMidnight 距下一个本地零点的秒数(日配额的自然重置时刻)。
|
||||||
|
func secondsToMidnight(now time.Time) int {
|
||||||
|
next := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()).AddDate(0, 0, 1)
|
||||||
|
return int(next.Sub(now).Seconds())
|
||||||
|
}
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
package middleware
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/gin-gonic/gin"
|
||||||
|
|
||||||
|
"github.com/wangjia/jiu/backend/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newQuotaRouter(scope string, limit int) *gin.Engine {
|
||||||
|
r := gin.New()
|
||||||
|
r.GET("/", DailyQuota(scope, func() int { return limit }), func(c *gin.Context) {
|
||||||
|
c.String(http.StatusOK, "ok")
|
||||||
|
})
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDailyQuotaExceeded429(t *testing.T) {
|
||||||
|
enableRateLimit(t)
|
||||||
|
r := newQuotaRouter("tq1", 3)
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
if w := hit(r, "7.7.7.7"); w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("配额内第 %d 发应放行,得到 %d", i+1, w.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
w := hit(r, "7.7.7.7")
|
||||||
|
if w.Code != http.StatusTooManyRequests {
|
||||||
|
t.Fatalf("超日配额应 429,得到 %d", w.Code)
|
||||||
|
}
|
||||||
|
if w.Header().Get("Retry-After") == "" {
|
||||||
|
t.Fatal("429 应带 Retry-After(到午夜秒数)")
|
||||||
|
}
|
||||||
|
// 不同 IP 独立
|
||||||
|
if w := hit(r, "8.8.8.8"); w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("其他 IP 应不受影响,得到 %d", w.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDailyQuotaScopesIndependent(t *testing.T) {
|
||||||
|
enableRateLimit(t)
|
||||||
|
a := newQuotaRouter("tq2a", 1)
|
||||||
|
b := newQuotaRouter("tq2b", 1)
|
||||||
|
if w := hit(a, "9.9.9.9"); w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("scope a 首发应放行,得到 %d", w.Code)
|
||||||
|
}
|
||||||
|
if w := hit(b, "9.9.9.9"); w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("scope b 有独立配额池,应放行,得到 %d", w.Code)
|
||||||
|
}
|
||||||
|
if w := hit(a, "9.9.9.9"); w.Code != http.StatusTooManyRequests {
|
||||||
|
t.Fatalf("scope a 第二发应 429,得到 %d", w.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDailyQuotaDisabledOrZeroPassthrough(t *testing.T) {
|
||||||
|
// limit=0 → 该闸关闭
|
||||||
|
enableRateLimit(t)
|
||||||
|
r := newQuotaRouter("tq3", 0)
|
||||||
|
for i := 0; i < 5; i++ {
|
||||||
|
if w := hit(r, "10.0.0.1"); w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("limit=0 第 %d 发应放行,得到 %d", i+1, w.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 总开关关闭 → 放行
|
||||||
|
prev := config.C.RateLimit.Enabled
|
||||||
|
config.C.RateLimit.Enabled = false
|
||||||
|
defer func() { config.C.RateLimit.Enabled = prev }()
|
||||||
|
r2 := newQuotaRouter("tq4", 1)
|
||||||
|
for i := 0; i < 5; i++ {
|
||||||
|
if w := hit(r2, "10.0.0.2"); w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("总开关关闭第 %d 发应放行,得到 %d", i+1, w.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -59,9 +59,12 @@ func Setup(r *gin.Engine, db *gorm.DB) {
|
|||||||
refreshIP := middleware.RateLimitByIP(middleware.PerMinute(rl.RefreshPerMin), rl.RefreshPerMin)
|
refreshIP := middleware.RateLimitByIP(middleware.PerMinute(rl.RefreshPerMin), rl.RefreshPerMin)
|
||||||
registerIP := middleware.RateLimitByIP(middleware.PerMinute(rl.RegisterPerMin), rl.RegisterPerMin)
|
registerIP := middleware.RateLimitByIP(middleware.PerMinute(rl.RegisterPerMin), rl.RegisterPerMin)
|
||||||
errorsIP := middleware.RateLimitByIP(middleware.PerMinute(rl.ErrorsPerMin), rl.ErrorsPerMin)
|
errorsIP := middleware.RateLimitByIP(middleware.PerMinute(rl.ErrorsPerMin), rl.ErrorsPerMin)
|
||||||
|
// 日配额二级闸(同一 IP 自然日累计,反爬):单品 API 与 OG 页同一配额池
|
||||||
|
dqProduct := middleware.DailyQuota("product", func() int { return config.C.RateLimit.DailyProductPerIP })
|
||||||
|
dqShopList := middleware.DailyQuota("shoplist", func() int { return config.C.RateLimit.DailyShopListPerIP })
|
||||||
|
|
||||||
// 公开商品详情页(注入 OG 标签的 Flutter index.html,供社交分享爬虫读取)
|
// 公开商品详情页(注入 OG 标签的 Flutter index.html,供社交分享爬虫读取)
|
||||||
r.GET("/product/:public_id", publicReadIP, publicH.ProductPage)
|
r.GET("/product/:public_id", publicReadIP, dqProduct, publicH.ProductPage)
|
||||||
|
|
||||||
v1 := r.Group("/api/v1")
|
v1 := r.Group("/api/v1")
|
||||||
|
|
||||||
@@ -77,8 +80,8 @@ func Setup(r *gin.Engine, db *gorm.DB) {
|
|||||||
// 公开接口(无需登录):读接口防爬、写接口防刷
|
// 公开接口(无需登录):读接口防爬、写接口防刷
|
||||||
public := v1.Group("/public")
|
public := v1.Group("/public")
|
||||||
{
|
{
|
||||||
public.GET("/products/:public_id", publicReadIP, publicH.GetProduct)
|
public.GET("/products/:public_id", publicReadIP, dqProduct, publicH.GetProduct)
|
||||||
public.GET("/shops/:shop_code/products", shopListIP, publicH.ListShopProducts)
|
public.GET("/shops/:shop_code/products", shopListIP, dqShopList, publicH.ListShopProducts)
|
||||||
public.GET("/release", publicReadIP, publicH.GetRelease)
|
public.GET("/release", publicReadIP, publicH.GetRelease)
|
||||||
public.POST("/errors", errorsIP, errorReportH.Submit)
|
public.POST("/errors", errorsIP, errorReportH.Submit)
|
||||||
public.POST("/register", registerIP, authH.Register)
|
public.POST("/register", registerIP, authH.Register)
|
||||||
|
|||||||
Reference in New Issue
Block a user