Files
jiu/backend/cmd/import-history/main.go
T
wangjia 90d2c8668b feat(backend): 历史进销存导入器 cmd/import-history + 单据明细快照列
一次性 CLI 导入旧系统 data/ 入库·出库 总表/详情表到当前库,历史只读(绝不
回放 inventories/inventory_logs)。明细忠实直存不做 SKU 归一:order/item 表
新增 creator_id(制单人)与 product_code/product_name/series/spec 快照列
(出库另补 batch_no/production_date),导入行 product_id 指向共享占位商品。
order_no 天然主键 (shop_id,order_no) 幂等去重,可重复跑。

testutil sqlite DDL 同步补列以保持 service 测试通过。

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-20 07:05:08 +08:00

591 lines
19 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// import-history —— 一次性把旧系统导出的历史进销存数据迁入当前库。
//
// 设计要点(已与用户确认):
// - 历史只读:in/out 单只作历史记录导入,绝不回放库存增减、绝不触碰
// inventories / inventory_logs。当前库存请另用现有 ImportInventory 从
// 库存.xls 导入。
// - 完全忠实、不做 SKU 归一:明细的 商品编号/商品名称/系列/规格 各存为
// 一个独立列(product_code/product_name/series/spec)。生产库 stock_*_items
// 有 product_id→products 外键且 NOT NULL,故明细 product_id 统一指向一个
// 占位商品(HIST-PLACEHOLDER),不按 SKU 建商品、不依赖商品主数据,展示层
// 优先读明细自身快照列。单位 不导入。
// - 单据编号 作单据天然主键,(shop_id, order_no) 幂等去重,可重复跑。
// - 明细按 单据编号 join 表头取 order_id 外键。
//
// 用法(在 backend/ 目录下执行):
//
// go run ./cmd/import-history --shop-code S001 --data-dir ../data --dry-run
// go run ./cmd/import-history --shop-code S001 --data-dir ../data
package main
import (
"crypto/rand"
"encoding/hex"
"flag"
"fmt"
"log"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/google/uuid"
"github.com/xuri/excelize/v2"
"golang.org/x/crypto/bcrypt"
"gorm.io/driver/mysql"
"gorm.io/gorm"
"gorm.io/gorm/logger"
"github.com/wangjia/jiu/backend/config"
"github.com/wangjia/jiu/backend/internal/model"
)
// 真实在岗人员(用户列损坏,需规整匹配)。前三位据旧系统核实,
// 汤振宇 由 dry-run 实测发现(出现 1375 次的完好姓名,非乱码)。
var realPeople = []struct {
prefix2 string // 前两个汉字,用于容损匹配(如 "丁国GSHD" → 丁国 → 丁国辉)
fullName string
username string
}{
{"丁国", "丁国辉", "hist_dingguohui"},
{"刘春", "刘春燕", "hist_liuchunyan"},
{"张志", "张志磊", "hist_zhangzhilei"},
{"汤振", "汤振宇", "hist_tangzhenyu"},
}
func main() {
var (
shopCode = flag.String("shop-code", "", "目标门店 code(与 shop-id 二选一)")
shopID = flag.Uint64("shop-id", 0, "目标门店 id(与 shop-code 二选一)")
dataDir = flag.String("data-dir", "../data", "data 目录路径")
dryRun = flag.Bool("dry-run", false, "只统计不写库")
envFile = flag.String("env-file", "", "可选:systemd 风格 KEY=VALUE 环境文件(用于远端读取 DATABASE_DSN,自行解析避免 shell 引号问题)")
)
flag.Parse()
if *envFile != "" {
loadEnvFile(*envFile)
}
config.Load()
db, err := gorm.Open(mysql.Open(config.C.Database.DSN), &gorm.Config{
Logger: logger.Default.LogMode(logger.Silent),
})
if err != nil {
log.Fatalf("连接数据库失败: %v", err)
}
// 确保历史导入用到的新列已就位(AutoMigrate 自动加可空列,不影响现有数据)。
// dry-run 必须只读:跳过 AutoMigrate,绝不改动目标库表结构。
if !*dryRun {
if err := db.AutoMigrate(
&model.StockInOrder{}, &model.StockInItem{},
&model.StockOutOrder{}, &model.StockOutItem{},
); err != nil {
log.Fatalf("AutoMigrate 失败: %v", err)
}
}
// ── 解析目标门店 ──────────────────────────────────────
var shop model.Shop
switch {
case *shopID > 0:
if err := db.First(&shop, *shopID).Error; err != nil {
log.Fatalf("门店 id=%d 不存在: %v", *shopID, err)
}
case *shopCode != "":
if err := db.Where("code = ?", *shopCode).First(&shop).Error; err != nil {
log.Fatalf("门店 code=%s 不存在: %v", *shopCode, err)
}
default:
log.Fatal("必须提供 --shop-code 或 --shop-id")
}
log.Printf("目标门店: %s (code=%s, id=%d) dry-run=%v", shop.Name, shop.Code, shop.ID, *dryRun)
// ── 默认仓库 ──────────────────────────────────────────
var wh model.Warehouse
if db.Where("shop_id = ? AND is_default = ? AND deleted_at IS NULL", shop.ID, true).First(&wh).Error != nil {
if db.Where("shop_id = ? AND deleted_at IS NULL", shop.ID).Order("id ASC").First(&wh).Error != nil {
log.Fatalf("门店 %d 无任何仓库,请先创建仓库", shop.ID)
}
}
log.Printf("默认仓库: %s (id=%d)", wh.Name, wh.ID)
ic := &importCtx{db: db, shopID: shop.ID, whID: wh.ID, dryRun: *dryRun}
ic.ensureUsers()
ic.ensurePlaceholderProduct()
// ── 入库 ──────────────────────────────────────────────
ic.importDoc(
filepath.Join(*dataDir, "入库单总表.xlsx"),
filepath.Join(*dataDir, "入库详情单总表.xlsx"),
true,
)
// ── 出库 ──────────────────────────────────────────────
ic.importDoc(
filepath.Join(*dataDir, "出库单总表.xlsx"),
filepath.Join(*dataDir, "出库详情单总表.xlsx"),
false,
)
ic.report()
}
// loadEnvFile 解析 systemd EnvironmentFile 风格的 KEY=VALUE 文件并写入进程环境。
// 自行按首个 '=' 切分,值原样保留(不做 shell 解引号),从而安全处理 DSN 中的
// & ( ) ? 等 shell 特殊字符。config.Load() 随后经 AutomaticEnv 读取 DATABASE_DSN。
func loadEnvFile(path string) {
data, err := os.ReadFile(path)
if err != nil {
log.Fatalf("读取 env 文件 %s 失败: %v", path, err)
}
for _, line := range strings.Split(string(data), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
eq := strings.IndexByte(line, '=')
if eq <= 0 {
continue
}
key := strings.TrimSpace(line[:eq])
val := strings.TrimSpace(line[eq+1:])
// 去掉成对包裹的引号(若有)
if len(val) >= 2 && (val[0] == '"' || val[0] == '\'') && val[len(val)-1] == val[0] {
val = val[1 : len(val)-1]
}
_ = os.Setenv(key, val)
}
}
// ── 导入上下文 ───────────────────────────────────────────
type importCtx struct {
db *gorm.DB
shopID uint64
whID uint64
dryRun bool
systemUserID uint64
placeholderProductID uint64 // 历史明细统一指向的占位商品(满足 product_id 外键)
personToUser map[string]uint64 // fullName → user id
partnerCache map[string]*uint64
// 统计
statOrders int
statItems int
statSkipped int // order_no 已存在跳过
statOrphanND int // 无明细单
statNoHeader int // 明细找不到表头(理论上为 0
newPartners int
unmatchedOps map[string]int // 未匹配到三人的人员原值 → 次数
}
func randPassword() string {
b := make([]byte, 16)
_, _ = rand.Read(b)
h, _ := bcrypt.GenerateFromPassword([]byte(hex.EncodeToString(b)), bcrypt.DefaultCost)
return string(h)
}
// ensureUsers 预置 system 兜底用户 + 三位真实人员(历史归属用,inactive)。
func (ic *importCtx) ensureUsers() {
ic.personToUser = map[string]uint64{}
ic.partnerCache = map[string]*uint64{}
ic.unmatchedOps = map[string]int{}
ensure := func(username, realName, role string) uint64 {
var u model.User
if ic.db.Where("shop_id = ? AND username = ?", ic.shopID, username).First(&u).Error == nil {
return u.ID
}
u = model.User{
ShopID: ic.shopID, Username: username, RealName: realName,
Role: role, IsActive: false, PasswordHash: randPassword(),
}
if ic.dryRun {
log.Printf("[dry-run] 将建用户: %s (%s)", username, realName)
return 0
}
if err := ic.db.Create(&u).Error; err != nil {
log.Fatalf("创建用户 %s 失败: %v", username, err)
}
log.Printf("建用户: %s (%s) id=%d", username, realName, u.ID)
return u.ID
}
ic.systemUserID = ensure("hist_system", "系统导入", "operator")
for _, p := range realPeople {
ic.personToUser[p.fullName] = ensure(p.username, p.fullName, "operator")
}
}
// 历史明细 product_id 占位商品的固定 code。
const placeholderProductCode = "HIST-PLACEHOLDER"
// ensurePlaceholderProduct 取/建一个占位商品,历史明细的 product_id 统一指向它,
// 用于满足 stock_*_items.product_id → products 的外键约束。展示层优先读明细
// 自身快照列(product_name/series/spec),故该占位商品仅作引用、不参与展示。
func (ic *importCtx) ensurePlaceholderProduct() {
var p model.Product
if ic.db.Where("shop_id = ? AND code = ? AND deleted_at IS NULL", ic.shopID, placeholderProductCode).First(&p).Error == nil {
ic.placeholderProductID = p.ID
return
}
if ic.dryRun {
log.Printf("[dry-run] 将建占位商品: %s", placeholderProductCode)
return
}
p = model.Product{
TenantBase: model.TenantBase{ShopID: ic.shopID},
PublicID: uuid.New().String(),
Code: placeholderProductCode,
Name: "历史导入占位",
Unit: "",
Remark: "历史进销存导入占位商品:明细真实商品信息见单据明细行自身列,请勿用于新业务",
}
if err := ic.db.Create(&p).Error; err != nil {
log.Fatalf("创建占位商品失败: %v", err)
}
ic.placeholderProductID = p.ID
log.Printf("建占位商品: %s id=%d", placeholderProductCode, p.ID)
}
// matchPerson 把损坏的人员原值规整到三位真人之一,返回 user id 指针。
// 匹配不到返回 nil(用于 creator/reviewer 等可空列;operator 另用 systemUserID 兜底)。
func (ic *importCtx) matchPerson(raw string) *uint64 {
s := strings.TrimSpace(raw)
if s == "" {
return nil
}
for _, p := range realPeople {
// 完整名出现 或 前两汉字前缀命中(容损)
if strings.Contains(s, p.fullName) || strings.HasPrefix(s, p.prefix2) {
id := ic.personToUser[p.fullName]
return &id
}
}
ic.unmatchedOps[s]++
return nil
}
// findOrCreatePartner 按名匹配往来单位,缺失自动建。
func (ic *importCtx) findOrCreatePartner(name, ptype string) *uint64 {
name = strings.TrimSpace(name)
if name == "" {
return nil
}
if cached, ok := ic.partnerCache[name]; ok {
return cached
}
var p model.Partner
if ic.db.Where("shop_id = ? AND name = ? AND deleted_at IS NULL", ic.shopID, name).First(&p).Error == nil {
id := p.ID
ic.partnerCache[name] = &id
return &id
}
ic.newPartners++
if ic.dryRun {
ic.partnerCache[name] = nil
return nil
}
p = model.Partner{
TenantBase: model.TenantBase{ShopID: ic.shopID},
Name: name, Type: ptype, Status: "enabled",
}
if err := ic.db.Create(&p).Error; err != nil {
log.Printf("创建往来单位 %s 失败: %v", name, err)
ic.partnerCache[name] = nil
return nil
}
id := p.ID
ic.partnerCache[name] = &id
return &id
}
// ── 解析与映射 ───────────────────────────────────────────
func mapStatus(raw string) string {
switch strings.TrimSpace(raw) {
case "已审核":
return "approved"
case "待审核":
return "pending"
case "被驳回", "作废":
return "rejected"
default:
return "draft"
}
}
func parseFloatLoose(s string) float64 {
s = strings.TrimSpace(strings.ReplaceAll(s, ",", ""))
if s == "" {
return 0
}
f, err := strconv.ParseFloat(s, 64)
if err != nil {
return 0
}
return f
}
var dateLayouts = []string{"2006-01-02", "2006/01/02", "2006-01-02 15:04:05", "2006/01/02 15:04:05"}
func parseDatePtr(s string) *model.Date {
s = strings.TrimSpace(s)
if s == "" {
return nil
}
for _, l := range dateLayouts {
if t, err := time.ParseInLocation(l, s, time.Local); err == nil {
return &model.Date{Time: t}
}
}
// Excel 序列号
if serial, err := strconv.ParseFloat(s, 64); err == nil && serial > 0 && serial < 100000 {
base := time.Date(1899, 12, 30, 0, 0, 0, 0, time.Local)
return &model.Date{Time: base.AddDate(0, 0, int(serial))}
}
return nil
}
func parseDateOrNow(s string) model.Date {
if d := parseDatePtr(s); d != nil {
return *d
}
return model.Date{Time: time.Now()}
}
// readSheet 读取首个工作表,返回表头列名→列索引映射 + 数据行。
func readSheet(path string) (map[string]int, [][]string) {
f, err := excelize.OpenFile(path)
if err != nil {
log.Fatalf("打开 %s 失败: %v", path, err)
}
defer f.Close()
rows, err := f.GetRows(f.GetSheetName(0))
if err != nil {
log.Fatalf("读取 %s 失败: %v", path, err)
}
if len(rows) < 1 {
log.Fatalf("%s 为空", path)
}
cols := map[string]int{}
for i, name := range rows[0] {
cols[strings.TrimSpace(name)] = i
}
return cols, rows[1:]
}
func get(row []string, cols map[string]int, name string) string {
idx, ok := cols[name]
if !ok || idx >= len(row) {
return ""
}
return strings.TrimSpace(row[idx])
}
// ── 核心导入 ─────────────────────────────────────────────
func (ic *importCtx) importDoc(headerPath, detailPath string, isIn bool) {
label := "出库"
orderType := "sale"
partnerType := "customer"
if isIn {
label, orderType, partnerType = "入库", "purchase", "supplier"
}
hCols, hRows := readSheet(headerPath)
dCols, dRows := readSheet(detailPath)
// 明细按单据编号分组
detailsByNo := map[string][][]string{}
for _, r := range dRows {
no := get(r, dCols, "单据编号")
if no == "" {
continue
}
detailsByNo[no] = append(detailsByNo[no], r)
}
log.Printf("[%s] 表头 %d 单 / 明细 %d 行(%d 单号)", label, len(hRows), len(dRows), len(detailsByNo))
seen := map[string]bool{}
for _, hr := range hRows {
orderNo := get(hr, hCols, "单据编号")
if orderNo == "" || seen[orderNo] {
continue
}
seen[orderNo] = true
details := detailsByNo[orderNo]
if len(details) == 0 {
ic.statOrphanND++
}
partnerName := get(hr, hCols, "单位名称")
partnerID := ic.findOrCreatePartner(partnerName, partnerType)
operatorID := ic.systemUserID
if op := ic.matchPerson(get(hr, hCols, "经手人")); op != nil {
operatorID = *op
}
creatorID := ic.matchPerson(get(hr, hCols, "制单人"))
reviewerID := ic.matchPerson(get(hr, hCols, "审核人"))
status := mapStatus(get(hr, hCols, "单据状态"))
orderDate := parseDateOrNow(get(hr, hCols, "单据日期"))
totalAmount := parseFloatLoose(get(hr, hCols, "单据金额"))
if ic.dryRun {
// 幂等:已存在则计入跳过
var cnt int64
ic.tableForOrder(isIn).Where("shop_id = ? AND order_no = ?", ic.shopID, orderNo).Count(&cnt)
if cnt > 0 {
ic.statSkipped++
continue
}
ic.statOrders++
ic.statItems += len(details)
continue
}
if ic.writeOrder(isIn, orderNo, orderType, status, orderDate, totalAmount,
partnerID, operatorID, creatorID, reviewerID, details, dCols) {
ic.statOrders++
ic.statItems += len(details)
} else {
ic.statSkipped++
}
}
// 明细单号在表头里找不到的(理论上为 0,留作校验)
headerNos := map[string]bool{}
for _, hr := range hRows {
if no := get(hr, hCols, "单据编号"); no != "" {
headerNos[no] = true
}
}
for no := range detailsByNo {
if !headerNos[no] {
ic.statNoHeader++
}
}
}
func (ic *importCtx) tableForOrder(isIn bool) *gorm.DB {
if isIn {
return ic.db.Model(&model.StockInOrder{})
}
return ic.db.Model(&model.StockOutOrder{})
}
// writeOrder 在事务内写一张单 + 其明细。返回 false 表示 order_no 已存在(跳过)。
func (ic *importCtx) writeOrder(
isIn bool, orderNo, orderType, status string, orderDate model.Date, totalAmount float64,
partnerID *uint64, operatorID uint64, creatorID, reviewerID *uint64,
details [][]string, dCols map[string]int,
) bool {
created := false
err := ic.db.Transaction(func(tx *gorm.DB) error {
if isIn {
var cnt int64
tx.Model(&model.StockInOrder{}).Where("shop_id = ? AND order_no = ?", ic.shopID, orderNo).Count(&cnt)
if cnt > 0 {
return nil
}
o := model.StockInOrder{
TenantBase: model.TenantBase{ShopID: ic.shopID},
OrderNo: orderNo, Type: orderType, WarehouseID: ic.whID,
PartnerID: partnerID, OperatorID: operatorID, CreatorID: creatorID,
ReviewerID: reviewerID, Status: status, OrderDate: orderDate, TotalAmount: totalAmount,
}
if err := tx.Create(&o).Error; err != nil {
return err
}
for _, dr := range details {
it := model.StockInItem{
OrderID: o.ID, ShopID: ic.shopID, ProductID: ic.placeholderProductID,
ProductCode: get(dr, dCols, "商品编号"),
ProductName: get(dr, dCols, "商品名称"),
Series: get(dr, dCols, "系列"),
Spec: get(dr, dCols, "规格"),
Quantity: parseFloatLoose(get(dr, dCols, "数量")),
UnitPrice: parseFloatLoose(get(dr, dCols, "单价")),
TotalPrice: parseFloatLoose(get(dr, dCols, "金额")),
BatchNo: get(dr, dCols, "批次号"),
}
it.ProductionDate = parseDatePtr(get(dr, dCols, "生产日期"))
if err := tx.Create(&it).Error; err != nil {
return err
}
}
} else {
var cnt int64
tx.Model(&model.StockOutOrder{}).Where("shop_id = ? AND order_no = ?", ic.shopID, orderNo).Count(&cnt)
if cnt > 0 {
return nil
}
o := model.StockOutOrder{
TenantBase: model.TenantBase{ShopID: ic.shopID},
OrderNo: orderNo, Type: orderType, WarehouseID: ic.whID,
PartnerID: partnerID, OperatorID: operatorID, CreatorID: creatorID,
ReviewerID: reviewerID, Status: status, OrderDate: orderDate, TotalAmount: totalAmount,
}
if err := tx.Create(&o).Error; err != nil {
return err
}
for _, dr := range details {
it := model.StockOutItem{
OrderID: o.ID, ShopID: ic.shopID, ProductID: ic.placeholderProductID,
ProductCode: get(dr, dCols, "商品编号"),
ProductName: get(dr, dCols, "商品名称"),
Series: get(dr, dCols, "系列"),
Spec: get(dr, dCols, "规格"),
Quantity: parseFloatLoose(get(dr, dCols, "数量")),
UnitPrice: parseFloatLoose(get(dr, dCols, "单价")),
TotalPrice: parseFloatLoose(get(dr, dCols, "金额")),
BatchNo: get(dr, dCols, "批次号"),
}
it.ProductionDate = parseDatePtr(get(dr, dCols, "生产日期"))
if err := tx.Create(&it).Error; err != nil {
return err
}
}
}
created = true
return nil
})
if err != nil {
log.Printf("写单 %s 失败: %v", orderNo, err)
return false
}
return created
}
func (ic *importCtx) report() {
fmt.Println("═══════════════════════════════════════════")
fmt.Println(" 历史数据导入汇总")
fmt.Println("═══════════════════════════════════════════")
if ic.dryRun {
fmt.Println(" *** DRY-RUN(未写库)***")
}
fmt.Printf(" 新建单据:%d\n", ic.statOrders)
fmt.Printf(" 新建明细:%d\n", ic.statItems)
fmt.Printf(" 跳过(已存在)%d\n", ic.statSkipped)
fmt.Printf(" 无明细单:%d\n", ic.statOrphanND)
fmt.Printf(" 明细缺表头(应为0)%d\n", ic.statNoHeader)
fmt.Printf(" 新建往来单位:%d\n", ic.newPartners)
if len(ic.unmatchedOps) > 0 {
fmt.Printf(" 未匹配人员(归 system)%d 种\n", len(ic.unmatchedOps))
shown := 0
for k, v := range ic.unmatchedOps {
fmt.Printf(" %q ×%d\n", k, v)
if shown++; shown >= 15 {
fmt.Println(" ...")
break
}
}
}
fmt.Println("═══════════════════════════════════════════")
}