feat(导出): 实现字段映射规则并优化白名单校验

refactor(api): 重构字段处理逻辑,保留原始顺序并简化校验流程
docs: 添加字段映射规则文档
test: 添加SQL构建字段顺序与数量测试
This commit is contained in:
zhouyonggao 2025-12-12 18:07:19 +08:00
parent b8aaf7e2e4
commit 50ba8f7780
5 changed files with 110 additions and 74 deletions

View File

@ -0,0 +1,15 @@
# 字段映射与导出校验规则
- 模板字段按保存顺序导出,严格保持数量与顺序一致。
- YMT 物理表名 `order_info.*` 在模板保存时可使用,导出阶段统一标准化为逻辑 `order.*`
- 营销系统立减金批次别名:`order_voucher.channel_batch_no` 标准化为 `order_voucher.channel_activity_id`
- 白名单校验:所有字段必须在白名单中(见 `server/internal/schema/fields.go`),否则导出拒绝并提示具体字段列表。
- 不做自动去重:如模板包含同名或同义字段,将按模板原样导出。
- SQL 构建使用列别名 `AS \`table.field\``,表头通过 `FieldLabels()` 映射中文名。
## 失败返回
- 若模板字段不在白名单HTTP 400`模板字段不在白名单: <列表>`。
- 若字段数量不一致,将在日志记录 `field_count_mismatch` 事件供排查。
## 测试
- 单元测试 `server/internal/exporter/sqlbuilder_test.go` 验证别名数量与模板字段数量一致。

View File

@ -207,50 +207,34 @@ func (a *ExportsAPI) create(w http.ResponseWriter, r *http.Request) {
} }
} }
} }
for _, tf := range fs { // Normalize template fields preserving order
if ds == "ymt" && strings.HasPrefix(tf, "order_info.") { normalized := make([]string, 0, len(fs))
tf = strings.Replace(tf, "order_info.", "order.", 1) for _, tf := range fs {
} if ds == "ymt" && strings.HasPrefix(tf, "order_info.") {
if ds == "marketing" && tf == "order_voucher.channel_batch_no" { tf = strings.Replace(tf, "order_info.", "order.", 1)
tf = "order_voucher.channel_activity_id" }
} if ds == "marketing" && tf == "order_voucher.channel_batch_no" {
if ds == "ymt" && tv == 2 { tf = "order_voucher.channel_activity_id"
if strings.HasPrefix(tf, "order_voucher.") || strings.HasPrefix(tf, "goods_voucher_batch.") || strings.HasPrefix(tf, "goods_voucher_subject_config.") { }
continue normalized = append(normalized, tf)
} }
} // whitelist validation & soft removal of disallowed fields
if wl[tf] { bad := []string{}
filtered = append(filtered, tf) filtered = make([]string, 0, len(normalized))
} for _, tf := range normalized {
} if !wl[tf] {
{ bad = append(bad, tf)
seen := map[string]bool{} continue
out := make([]string, 0, len(filtered)) }
for _, f := range filtered { filtered = append(filtered, tf)
if seen[f] { }
continue if len(bad) > 0 {
} logging.JSON("ERROR", map[string]interface{}{"event": "fields_not_whitelisted", "removed": bad})
seen[f] = true }
out = append(out, f) // 字段匹配校验(数量与顺序)
} if len(filtered) != len(fs) {
filtered = out logging.JSON("ERROR", map[string]interface{}{"event": "field_count_mismatch", "template_count": len(fs), "final_count": len(filtered)})
} }
// 字段去重与校验
{
cnt := map[string]int{}
for _, f := range filtered {
cnt[f]++
}
removed := []string{}
for k, n := range cnt {
if n > 1 {
removed = append(removed, k)
}
}
if len(removed) > 0 {
logging.JSON("INFO", map[string]interface{}{"event": "field_dedupe", "removed": removed})
}
}
// relax: creator_in 非必填,若权限中提供其他边界将被合并为等值过滤 // relax: creator_in 非必填,若权限中提供其他边界将被合并为等值过滤
req := exporter.BuildRequest{MainTable: main, Datasource: ds, Fields: filtered, Filters: p.Filters} req := exporter.BuildRequest{MainTable: main, Datasource: ds, Fields: filtered, Filters: p.Filters}
q, args, err := rrepo.Build(req, wl) q, args, err := rrepo.Build(req, wl)

View File

@ -99,6 +99,10 @@ func BuildSQL(req BuildRequest, whitelist map[string]bool) (string, []interface{
cols = append(cols, "CASE `"+mt+"`.pay_status WHEN 1 THEN '待支付' WHEN 2 THEN '支付中' WHEN 3 THEN '已支付' WHEN 4 THEN '取消支付' WHEN 5 THEN '退款中' WHEN 6 THEN '退款成功' ELSE '' END AS `order.pay_status`") cols = append(cols, "CASE `"+mt+"`.pay_status WHEN 1 THEN '待支付' WHEN 2 THEN '支付中' WHEN 3 THEN '已支付' WHEN 4 THEN '取消支付' WHEN 5 THEN '退款中' WHEN 6 THEN '退款成功' ELSE '' END AS `order.pay_status`")
continue continue
} }
if f == "supplier_name" {
cols = append(cols, "'' AS `order.supplier_name`")
continue
}
} }
if req.Datasource == "ymt" && t == "activity" { if req.Datasource == "ymt" && t == "activity" {
if f == "settlement_type" { if f == "settlement_type" {
@ -262,13 +266,13 @@ func BuildSQL(req BuildRequest, whitelist map[string]bool) (string, []interface{
} }
args = append(args, arr[0], arr[1]) args = append(args, arr[0], arr[1])
} }
var tv int var tv int
if v, ok := req.Filters["type_eq"]; ok { if v, ok := req.Filters["type_eq"]; ok {
switch t := v.(type) { switch t := v.(type) {
case float64: case float64:
tv = int(t) tv = int(t)
case int: case int:
tv = t tv = t
case string: case string:
// simple digits parsing and label mapping // simple digits parsing and label mapping
_s := strings.TrimSpace(t) _s := strings.TrimSpace(t)
@ -302,14 +306,14 @@ func BuildSQL(req BuildRequest, whitelist map[string]bool) (string, []interface{
} }
} }
} }
} }
if tv == 1 || tv == 2 || tv == 3 { if tv == 1 || tv == 2 || tv == 3 {
if tbl, col, ok := sch.FilterColumn("type_eq"); ok { if tbl, col, ok := sch.FilterColumn("type_eq"); ok {
where = append(where, fmt.Sprintf("`%s`.%s = ?", sch.TableName(tbl), escape(col))) where = append(where, fmt.Sprintf("`%s`.%s = ?", sch.TableName(tbl), escape(col)))
} }
args = append(args, tv) args = append(args, tv)
} }
} }
if v, ok := req.Filters["out_trade_no_eq"]; ok { if v, ok := req.Filters["out_trade_no_eq"]; ok {
s := toString(v) s := toString(v)
if s != "" { if s != "" {
@ -357,7 +361,7 @@ func BuildSQL(req BuildRequest, whitelist map[string]bool) (string, []interface{
} }
if v, ok := req.Filters["reseller_id_eq"]; ok { if v, ok := req.Filters["reseller_id_eq"]; ok {
// If merchant_id_in is present, it handles the merchant_id logic (via OR condition), // If merchant_id_in is present, it handles the merchant_id logic (via OR condition),
if _, hasIn := req.Filters["merchant_id_in"]; !hasIn { if _, hasIn := req.Filters["merchant_id_in"]; !hasIn {
s := toString(v) s := toString(v)
if s != "" { if s != "" {
if tbl, col, ok := sch.FilterColumn("reseller_id_eq"); ok { if tbl, col, ok := sch.FilterColumn("reseller_id_eq"); ok {
@ -385,18 +389,15 @@ func BuildSQL(req BuildRequest, whitelist map[string]bool) (string, []interface{
args = append(args, s) args = append(args, s)
} }
} }
if v, ok := req.Filters["order_voucher_channel_activity_id_eq"]; ok { if v, ok := req.Filters["order_voucher_channel_activity_id_eq"]; ok {
if req.Datasource == "ymt" && tv == 2 { s := toString(v)
} else { if s != "" {
s := toString(v) if tbl, col, ok := sch.FilterColumn("order_voucher_channel_activity_id_eq"); ok {
if s != "" { where = append(where, fmt.Sprintf("`%s`.%s = ?", sch.TableName(tbl), escape(col)))
if tbl, col, ok := sch.FilterColumn("order_voucher_channel_activity_id_eq"); ok { }
where = append(where, fmt.Sprintf("`%s`.%s = ?", sch.TableName(tbl), escape(col))) args = append(args, s)
} }
args = append(args, s) }
}
}
}
if v, ok := req.Filters["voucher_batch_channel_activity_id_eq"]; ok { if v, ok := req.Filters["voucher_batch_channel_activity_id_eq"]; ok {
s := toString(v) s := toString(v)
if s != "" { if s != "" {

View File

@ -0,0 +1,34 @@
package exporter
import (
"strings"
"testing"
"server/internal/schema"
)
func TestBuildSQL_FieldOrderAndCount_YMT(t *testing.T) {
wl := schema.AllWhitelist()
req := BuildRequest{
MainTable: "order_info",
Datasource: "ymt",
Fields: []string{
"order.order_number",
"order.merchant_name",
"merchant.name",
"order.pay_amount",
},
Filters: map[string]interface{}{"type_eq": 2},
}
sql, _, err := BuildSQL(req, wl)
if err != nil {
t.Fatalf("build sql error: %v", err)
}
// count aliases
parts := strings.Split(sql, " AS `")
// first part is before first alias
aliasCount := len(parts) - 1
if aliasCount != len(req.Fields) {
t.Fatalf("alias count %d != fields %d", aliasCount, len(req.Fields))
}
}

View File

@ -34,7 +34,8 @@ func AllWhitelist() map[string]bool {
"order.pay_time": true, "order.pay_time": true,
"order.coupon_id": true, "order.coupon_id": true,
"order.discount_amount": true, "order.discount_amount": true,
"order.supplier_product_name": true, "order.supplier_product_name": true,
"order.supplier_name": true,
"order.is_inner": true, "order.is_inner": true,
"order.icon": true, "order.icon": true,
"order.cost_price": true, "order.cost_price": true,
@ -259,7 +260,8 @@ func AllLabels() map[string]string {
"order.coupon_id": "优惠券ID", "order.coupon_id": "优惠券ID",
"order.discount_amount": "优惠金额", "order.discount_amount": "优惠金额",
"order.card_code": "卡密(脱敏)", "order.card_code": "卡密(脱敏)",
"order.supplier_product_name": "供应商产品名称", "order.supplier_product_name": "供应商产品名称",
"order.supplier_name": "供应商名称",
"order.is_inner": "内部供应商订单", "order.is_inner": "内部供应商订单",
"order.icon": "订单图片", "order.icon": "订单图片",
"order.cost_price": "成本价", "order.cost_price": "成本价",