fix(backend): 库存导入改用 shakinm/xlsReader，按唯一编号去重，入库日期取自 Excel
extrame/xls 误读真实 .xls（3264 行只读出 455 行、编号丢失、数字被当 日期序列），换用 shakinm/xlsReader 正确读取全部行。去重改为「有商品编号 时仅按编号匹配」，避免不同批次被 NSS 回退合并；Inventory.ProductCode 存行自身唯一编号。入库时间取 Excel 入库日期列写入 created_at。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
wangjia
@@ -1,71 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
xls "github.com/extrame/xls"
|
||||
)
|
||||
|
||||
func safeXlsRow(sheet *xls.WorkSheet, r int) (row *xls.Row) {
|
||||
defer func() { recover() }()
|
||||
return sheet.Row(r)
|
||||
}
|
||||
|
||||
func main() {
|
||||
wb, err := xls.Open("/Users/wangjia/.claude/jobs/4f07558d/tmp/test_150rows.xls", "utf-8")
|
||||
if err != nil {
|
||||
fmt.Println("open error:", err)
|
||||
return
|
||||
}
|
||||
sheet := wb.GetSheet(0)
|
||||
if sheet == nil {
|
||||
fmt.Println("no sheet")
|
||||
return
|
||||
}
|
||||
fmt.Printf("MaxRow: %d\n", sheet.MaxRow)
|
||||
|
||||
numCols := 0
|
||||
headerRow := sheet.Row(0)
|
||||
fmt.Printf("Header LastCol: %d\n", headerRow.LastCol())
|
||||
for c := 0; c < headerRow.LastCol(); c++ {
|
||||
v := strings.TrimSpace(headerRow.Col(c))
|
||||
if v != "" {
|
||||
numCols = c + 1
|
||||
}
|
||||
}
|
||||
if numCols == 0 {
|
||||
numCols = 20
|
||||
}
|
||||
fmt.Printf("numCols: %d\n", numCols)
|
||||
|
||||
const maxRows = 100000
|
||||
const maxEmpty = 5
|
||||
emptyStreak := 0
|
||||
rowCount := 0
|
||||
for r := 0; r < maxRows; r++ {
|
||||
row := safeXlsRow(sheet, r)
|
||||
cells := make([]string, numCols)
|
||||
isEmpty := true
|
||||
if row != nil {
|
||||
for c := 0; c < numCols; c++ {
|
||||
cells[c] = strings.TrimSpace(row.Col(c))
|
||||
if cells[c] != "" {
|
||||
isEmpty = false
|
||||
}
|
||||
}
|
||||
}
|
||||
if isEmpty {
|
||||
emptyStreak++
|
||||
if emptyStreak >= maxEmpty {
|
||||
fmt.Printf("Breaking at r=%d after %d empty streak\n", r, emptyStreak)
|
||||
break
|
||||
}
|
||||
rowCount++
|
||||
continue
|
||||
}
|
||||
emptyStreak = 0
|
||||
rowCount++
|
||||
}
|
||||
fmt.Printf("Total rows read: %d\n", rowCount)
|
||||
}
|
||||
+7
-7
@@ -3,12 +3,18 @@ module github.com/wangjia/jiu/backend
|
||||
go 1.26.1
|
||||
|
||||
require (
|
||||
github.com/disintegration/imaging v1.6.2
|
||||
github.com/gin-gonic/gin v1.12.0
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/mozillazg/go-pinyin v0.21.0
|
||||
github.com/shakinm/xlsReader v0.9.12
|
||||
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
|
||||
github.com/spf13/viper v1.21.0
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/xuri/excelize/v2 v2.10.1
|
||||
golang.org/x/crypto v0.49.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
gorm.io/driver/mysql v1.6.0
|
||||
gorm.io/driver/sqlite v1.6.0
|
||||
gorm.io/gorm v1.31.1
|
||||
@@ -21,9 +27,6 @@ require (
|
||||
github.com/bytedance/sonic/loader v0.5.0 // indirect
|
||||
github.com/cloudwego/base64x v0.1.6 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/disintegration/imaging v1.6.2 // indirect
|
||||
github.com/extrame/ole2 v0.0.0-20160812065207-d69429661ad7 // indirect
|
||||
github.com/extrame/xls v0.0.1 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.12 // indirect
|
||||
github.com/gin-contrib/sse v1.1.0 // indirect
|
||||
@@ -34,7 +37,6 @@ require (
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||
github.com/goccy/go-json v0.10.5 // indirect
|
||||
github.com/goccy/go-yaml v1.19.2 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/jinzhu/now v1.1.5 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
@@ -42,9 +44,9 @@ require (
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-sqlite3 v1.14.22 // indirect
|
||||
github.com/metakeule/fmtdate v1.1.2 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/mozillazg/go-pinyin v0.21.0 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/quic-go/qpack v0.6.0 // indirect
|
||||
@@ -52,7 +54,6 @@ require (
|
||||
github.com/richardlehane/mscfb v1.0.6 // indirect
|
||||
github.com/richardlehane/msoleps v1.0.6 // indirect
|
||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/spf13/cast v1.10.0 // indirect
|
||||
@@ -71,5 +72,4 @@ require (
|
||||
golang.org/x/sys v0.42.0 // indirect
|
||||
golang.org/x/text v0.35.0 // indirect
|
||||
google.golang.org/protobuf v1.36.10 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
+6
-4
@@ -13,10 +13,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
|
||||
github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
|
||||
github.com/extrame/ole2 v0.0.0-20160812065207-d69429661ad7 h1:n+nk0bNe2+gVbRI8WRbLFVwwcBQ0rr5p+gzkKb6ol8c=
|
||||
github.com/extrame/ole2 v0.0.0-20160812065207-d69429661ad7/go.mod h1:GPpMrAfHdb8IdQ1/R2uIRBsNfnPnwsYE9YYI5WyY1zw=
|
||||
github.com/extrame/xls v0.0.1 h1:jI7L/o3z73TyyENPopsLS/Jlekm3nF1a/kF5hKBvy/k=
|
||||
github.com/extrame/xls v0.0.1/go.mod h1:iACcgahst7BboCpIMSpnFs4SKyU9ZjsvZBfNbUxZOJI=
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
||||
@@ -68,6 +64,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
|
||||
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/metakeule/fmtdate v1.1.2 h1:n9M7H9HfAqp+6OA98wXGMdcAr6omshSNVct65Bks1lQ=
|
||||
github.com/metakeule/fmtdate v1.1.2/go.mod h1:2JyMFlKxeoGy1qS6obQukT0AL0Y4iNANQL8scbSdT4E=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
@@ -91,6 +89,8 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR
|
||||
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
|
||||
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
|
||||
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
|
||||
github.com/shakinm/xlsReader v0.9.12 h1:F6GWYtCzfzQqdIuqZJ0MU3YJ7uwH1ofJtmTKyWmANQk=
|
||||
github.com/shakinm/xlsReader v0.9.12/go.mod h1:ME9pqIGf+547L4aE4YTZzwmhsij+5K9dR+k84OO6WSs=
|
||||
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
|
||||
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
|
||||
@@ -147,8 +147,10 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
|
||||
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
|
||||
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
|
||||
google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
|
||||
@@ -2,15 +2,13 @@ package handler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/extrame/xls"
|
||||
"github.com/shakinm/xlsReader/xls"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/google/uuid"
|
||||
"github.com/xuri/excelize/v2"
|
||||
@@ -550,10 +548,11 @@ func (h *ImportHandler) ImportInventory(c *gin.Context) {
|
||||
invByNSS[nssKey] = inv
|
||||
}
|
||||
lookupInv := func(productCode, name, series, spec string, whID uint64) *model.Inventory {
|
||||
// 有商品编号时，仅按「编号|仓库」唯一匹配，绝不回退到名称匹配：
|
||||
// 同一款酒的不同批次/年份是不同的商品编号，名称相同但属于独立库存记录，
|
||||
// 回退名称匹配会把它们错误合并成一条（3264 行被压成 1134 行）。
|
||||
if productCode != "" {
|
||||
if inv, ok := invByCode[fmt.Sprintf("%s|%d", productCode, whID)]; ok {
|
||||
return inv
|
||||
}
|
||||
return invByCode[fmt.Sprintf("%s|%d", productCode, whID)]
|
||||
}
|
||||
return invByNSS[fmt.Sprintf("%s|%s|%s|%d", name, series, spec, whID)]
|
||||
}
|
||||
@@ -782,7 +781,7 @@ func (h *ImportHandler) ImportInventory(c *gin.Context) {
|
||||
ProductID: &productIDCopy,
|
||||
StockInItemID: nil,
|
||||
Quantity: qty,
|
||||
ProductCode: prod.Code,
|
||||
ProductCode: productCode, // 行自身的商品编号（每条库存记录唯一），而非商品目录编号
|
||||
ProductName: prod.Name,
|
||||
Series: prod.Series,
|
||||
Spec: prod.Spec,
|
||||
@@ -858,65 +857,38 @@ func parseUploadedExcel(c *gin.Context) ([][]string, error) {
|
||||
var rows [][]string
|
||||
|
||||
if isOLE {
|
||||
// 老格式 BIFF — extrame/xls 需要文件路径，写入临时文件
|
||||
tmp, tmpErr := os.CreateTemp("", "import_*.xls")
|
||||
if tmpErr != nil {
|
||||
return nil, fmt.Errorf("cannot create temp file: %s", tmpErr.Error())
|
||||
}
|
||||
defer os.Remove(tmp.Name())
|
||||
if _, cpErr := io.Copy(tmp, f); cpErr != nil {
|
||||
tmp.Close()
|
||||
return nil, fmt.Errorf("cannot write temp file: %s", cpErr.Error())
|
||||
}
|
||||
tmp.Close()
|
||||
|
||||
wb, xlErr := xls.Open(tmp.Name(), "utf-8")
|
||||
// 老格式 BIFF (.xls)。注意：extrame/xls 对部分真实导出文件会错读单元格
|
||||
// （字符串整列丢失、数字被当成日期序列号），导致 3000 行只解析出几百行。
|
||||
// 改用 shakinm/xlsReader，可正确读取共享字符串表与数值。
|
||||
wb, xlErr := xls.OpenReader(f)
|
||||
if xlErr != nil {
|
||||
return nil, fmt.Errorf("invalid xls file: %s", xlErr.Error())
|
||||
}
|
||||
sheet := wb.GetSheet(0)
|
||||
if sheet == nil {
|
||||
sheet, shErr := wb.GetSheet(0)
|
||||
if shErr != nil || sheet == nil {
|
||||
return nil, fmt.Errorf("no sheet found")
|
||||
}
|
||||
// LastCol() returns 0 for many data rows in extrame/xls; derive column
|
||||
// count from the header row instead.
|
||||
numCols := 0
|
||||
headerRow := sheet.Row(0)
|
||||
for c := 0; c < headerRow.LastCol(); c++ {
|
||||
if strings.TrimSpace(headerRow.Col(c)) != "" {
|
||||
numCols = c + 1
|
||||
}
|
||||
numRows := sheet.GetNumberRows()
|
||||
if numRows < 1 {
|
||||
return nil, fmt.Errorf("empty or invalid sheet")
|
||||
}
|
||||
// 列数以表头行为准
|
||||
header, _ := sheet.GetRow(0)
|
||||
numCols := len(header.GetCols())
|
||||
if numCols == 0 {
|
||||
numCols = 20
|
||||
}
|
||||
// sheet.MaxRow 依赖 DIMENSIONS 记录，旧软件导出的 XLS 该值可能偏小。
|
||||
// 改为读到连续 5 行全空为止，最多 100000 行防止死循环。
|
||||
const maxRows = 100000
|
||||
const maxEmpty = 5
|
||||
emptyStreak := 0
|
||||
for r := 0; r < maxRows; r++ {
|
||||
row := safeXlsRow(sheet, r)
|
||||
for r := 0; r < numRows; r++ {
|
||||
row, _ := sheet.GetRow(r)
|
||||
cells := make([]string, numCols)
|
||||
isEmpty := true
|
||||
if row != nil {
|
||||
for c := 0; c < numCols; c++ {
|
||||
cells[c] = strings.TrimSpace(row.Col(c))
|
||||
if cells[c] != "" {
|
||||
isEmpty = false
|
||||
for col := 0; col < numCols; col++ {
|
||||
if cd, cErr := row.GetCol(col); cErr == nil && cd != nil {
|
||||
cells[col] = strings.TrimSpace(cd.GetString())
|
||||
}
|
||||
}
|
||||
}
|
||||
if isEmpty {
|
||||
emptyStreak++
|
||||
if emptyStreak >= maxEmpty {
|
||||
break
|
||||
}
|
||||
// 保留空行，让调用方自行跳过
|
||||
rows = append(rows, cells)
|
||||
continue
|
||||
}
|
||||
emptyStreak = 0
|
||||
// 保留空行，让调用方自行跳过
|
||||
rows = append(rows, cells)
|
||||
}
|
||||
} else {
|
||||
@@ -1042,9 +1014,3 @@ func cell(row []string, idx int) string {
|
||||
}
|
||||
return strings.TrimSpace(row[idx])
|
||||
}
|
||||
|
||||
// safeXlsRow 安全读取 XLS 行，捕获 extrame/xls 在超出行数时的 panic。
|
||||
func safeXlsRow(sheet *xls.WorkSheet, r int) (row *xls.Row) {
|
||||
defer func() { recover() }() //nolint:errcheck
|
||||
return sheet.Row(r)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user