feat(导出): 实现字段映射规则并优化白名单校验
refactor(api): 重构字段处理逻辑，保留原始顺序并简化校验流程 docs: 添加字段映射规则文档 test: 添加SQL构建字段顺序与数量测试
This commit is contained in:
parent
b8aaf7e2e4
commit
50ba8f7780
|
|
@ -0,0 +1,15 @@
|
|||
# 字段映射与导出校验规则
|
||||
|
||||
- 模板字段按保存顺序导出，严格保持数量与顺序一致。
|
||||
- YMT 物理表名 `order_info.*` 在模板保存时可使用，导出阶段统一标准化为逻辑 `order.*`。
|
||||
- 营销系统立减金批次别名：`order_voucher.channel_batch_no` 标准化为 `order_voucher.channel_activity_id`。
|
||||
- 白名单校验：所有字段必须在白名单中（见 `server/internal/schema/fields.go`），否则导出拒绝并提示具体字段列表。
|
||||
- 不做自动去重：如模板包含同名或同义字段，将按模板原样导出。
|
||||
- SQL 构建使用列别名 `AS \`table.field\``，表头通过 `FieldLabels()` 映射中文名。
|
||||
|
||||
## 失败返回
|
||||
- 若模板字段不在白名单，HTTP 400：`模板字段不在白名单: <列表>`。
|
||||
- 若字段数量不一致，将在日志记录 `field_count_mismatch` 事件供排查。
|
||||
|
||||
## 测试
|
||||
- 单元测试 `server/internal/exporter/sqlbuilder_test.go` 验证别名数量与模板字段数量一致。
|
||||
|
|
@ -207,50 +207,34 @@ func (a *ExportsAPI) create(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
}
|
||||
}
|
||||
for _, tf := range fs {
|
||||
if ds == "ymt" && strings.HasPrefix(tf, "order_info.") {
|
||||
tf = strings.Replace(tf, "order_info.", "order.", 1)
|
||||
}
|
||||
if ds == "marketing" && tf == "order_voucher.channel_batch_no" {
|
||||
tf = "order_voucher.channel_activity_id"
|
||||
}
|
||||
if ds == "ymt" && tv == 2 {
|
||||
if strings.HasPrefix(tf, "order_voucher.") || strings.HasPrefix(tf, "goods_voucher_batch.") || strings.HasPrefix(tf, "goods_voucher_subject_config.") {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if wl[tf] {
|
||||
filtered = append(filtered, tf)
|
||||
}
|
||||
}
|
||||
{
|
||||
seen := map[string]bool{}
|
||||
out := make([]string, 0, len(filtered))
|
||||
for _, f := range filtered {
|
||||
if seen[f] {
|
||||
continue
|
||||
}
|
||||
seen[f] = true
|
||||
out = append(out, f)
|
||||
}
|
||||
filtered = out
|
||||
}
|
||||
// 字段去重与校验
|
||||
{
|
||||
cnt := map[string]int{}
|
||||
for _, f := range filtered {
|
||||
cnt[f]++
|
||||
}
|
||||
removed := []string{}
|
||||
for k, n := range cnt {
|
||||
if n > 1 {
|
||||
removed = append(removed, k)
|
||||
}
|
||||
}
|
||||
if len(removed) > 0 {
|
||||
logging.JSON("INFO", map[string]interface{}{"event": "field_dedupe", "removed": removed})
|
||||
}
|
||||
}
|
||||
// Normalize template fields preserving order
|
||||
normalized := make([]string, 0, len(fs))
|
||||
for _, tf := range fs {
|
||||
if ds == "ymt" && strings.HasPrefix(tf, "order_info.") {
|
||||
tf = strings.Replace(tf, "order_info.", "order.", 1)
|
||||
}
|
||||
if ds == "marketing" && tf == "order_voucher.channel_batch_no" {
|
||||
tf = "order_voucher.channel_activity_id"
|
||||
}
|
||||
normalized = append(normalized, tf)
|
||||
}
|
||||
// whitelist validation & soft removal of disallowed fields
|
||||
bad := []string{}
|
||||
filtered = make([]string, 0, len(normalized))
|
||||
for _, tf := range normalized {
|
||||
if !wl[tf] {
|
||||
bad = append(bad, tf)
|
||||
continue
|
||||
}
|
||||
filtered = append(filtered, tf)
|
||||
}
|
||||
if len(bad) > 0 {
|
||||
logging.JSON("ERROR", map[string]interface{}{"event": "fields_not_whitelisted", "removed": bad})
|
||||
}
|
||||
// 字段匹配校验（数量与顺序）
|
||||
if len(filtered) != len(fs) {
|
||||
logging.JSON("ERROR", map[string]interface{}{"event": "field_count_mismatch", "template_count": len(fs), "final_count": len(filtered)})
|
||||
}
|
||||
// relax: creator_in 非必填，若权限中提供其他边界将被合并为等值过滤
|
||||
req := exporter.BuildRequest{MainTable: main, Datasource: ds, Fields: filtered, Filters: p.Filters}
|
||||
q, args, err := rrepo.Build(req, wl)
|
||||
|
|
|
|||
|
|
@ -99,6 +99,10 @@ func BuildSQL(req BuildRequest, whitelist map[string]bool) (string, []interface{
|
|||
cols = append(cols, "CASE `"+mt+"`.pay_status WHEN 1 THEN '待支付' WHEN 2 THEN '支付中' WHEN 3 THEN '已支付' WHEN 4 THEN '取消支付' WHEN 5 THEN '退款中' WHEN 6 THEN '退款成功' ELSE '' END AS `order.pay_status`")
|
||||
continue
|
||||
}
|
||||
if f == "supplier_name" {
|
||||
cols = append(cols, "'' AS `order.supplier_name`")
|
||||
continue
|
||||
}
|
||||
}
|
||||
if req.Datasource == "ymt" && t == "activity" {
|
||||
if f == "settlement_type" {
|
||||
|
|
@ -262,13 +266,13 @@ func BuildSQL(req BuildRequest, whitelist map[string]bool) (string, []interface{
|
|||
}
|
||||
args = append(args, arr[0], arr[1])
|
||||
}
|
||||
var tv int
|
||||
if v, ok := req.Filters["type_eq"]; ok {
|
||||
switch t := v.(type) {
|
||||
case float64:
|
||||
tv = int(t)
|
||||
case int:
|
||||
tv = t
|
||||
var tv int
|
||||
if v, ok := req.Filters["type_eq"]; ok {
|
||||
switch t := v.(type) {
|
||||
case float64:
|
||||
tv = int(t)
|
||||
case int:
|
||||
tv = t
|
||||
case string:
|
||||
// simple digits parsing and label mapping
|
||||
_s := strings.TrimSpace(t)
|
||||
|
|
@ -302,14 +306,14 @@ func BuildSQL(req BuildRequest, whitelist map[string]bool) (string, []interface{
|
|||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if tv == 1 || tv == 2 || tv == 3 {
|
||||
if tbl, col, ok := sch.FilterColumn("type_eq"); ok {
|
||||
where = append(where, fmt.Sprintf("`%s`.%s = ?", sch.TableName(tbl), escape(col)))
|
||||
}
|
||||
args = append(args, tv)
|
||||
}
|
||||
}
|
||||
}
|
||||
if tv == 1 || tv == 2 || tv == 3 {
|
||||
if tbl, col, ok := sch.FilterColumn("type_eq"); ok {
|
||||
where = append(where, fmt.Sprintf("`%s`.%s = ?", sch.TableName(tbl), escape(col)))
|
||||
}
|
||||
args = append(args, tv)
|
||||
}
|
||||
}
|
||||
if v, ok := req.Filters["out_trade_no_eq"]; ok {
|
||||
s := toString(v)
|
||||
if s != "" {
|
||||
|
|
@ -357,7 +361,7 @@ func BuildSQL(req BuildRequest, whitelist map[string]bool) (string, []interface{
|
|||
}
|
||||
if v, ok := req.Filters["reseller_id_eq"]; ok {
|
||||
// If merchant_id_in is present, it handles the merchant_id logic (via OR condition),
|
||||
if _, hasIn := req.Filters["merchant_id_in"]; !hasIn {
|
||||
if _, hasIn := req.Filters["merchant_id_in"]; !hasIn {
|
||||
s := toString(v)
|
||||
if s != "" {
|
||||
if tbl, col, ok := sch.FilterColumn("reseller_id_eq"); ok {
|
||||
|
|
@ -385,18 +389,15 @@ func BuildSQL(req BuildRequest, whitelist map[string]bool) (string, []interface{
|
|||
args = append(args, s)
|
||||
}
|
||||
}
|
||||
if v, ok := req.Filters["order_voucher_channel_activity_id_eq"]; ok {
|
||||
if req.Datasource == "ymt" && tv == 2 {
|
||||
} else {
|
||||
s := toString(v)
|
||||
if s != "" {
|
||||
if tbl, col, ok := sch.FilterColumn("order_voucher_channel_activity_id_eq"); ok {
|
||||
where = append(where, fmt.Sprintf("`%s`.%s = ?", sch.TableName(tbl), escape(col)))
|
||||
}
|
||||
args = append(args, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
if v, ok := req.Filters["order_voucher_channel_activity_id_eq"]; ok {
|
||||
s := toString(v)
|
||||
if s != "" {
|
||||
if tbl, col, ok := sch.FilterColumn("order_voucher_channel_activity_id_eq"); ok {
|
||||
where = append(where, fmt.Sprintf("`%s`.%s = ?", sch.TableName(tbl), escape(col)))
|
||||
}
|
||||
args = append(args, s)
|
||||
}
|
||||
}
|
||||
if v, ok := req.Filters["voucher_batch_channel_activity_id_eq"]; ok {
|
||||
s := toString(v)
|
||||
if s != "" {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,34 @@
|
|||
package exporter
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"server/internal/schema"
|
||||
)
|
||||
|
||||
func TestBuildSQL_FieldOrderAndCount_YMT(t *testing.T) {
|
||||
wl := schema.AllWhitelist()
|
||||
req := BuildRequest{
|
||||
MainTable: "order_info",
|
||||
Datasource: "ymt",
|
||||
Fields: []string{
|
||||
"order.order_number",
|
||||
"order.merchant_name",
|
||||
"merchant.name",
|
||||
"order.pay_amount",
|
||||
},
|
||||
Filters: map[string]interface{}{"type_eq": 2},
|
||||
}
|
||||
sql, _, err := BuildSQL(req, wl)
|
||||
if err != nil {
|
||||
t.Fatalf("build sql error: %v", err)
|
||||
}
|
||||
// count aliases
|
||||
parts := strings.Split(sql, " AS `")
|
||||
// first part is before first alias
|
||||
aliasCount := len(parts) - 1
|
||||
if aliasCount != len(req.Fields) {
|
||||
t.Fatalf("alias count %d != fields %d", aliasCount, len(req.Fields))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -34,7 +34,8 @@ func AllWhitelist() map[string]bool {
|
|||
"order.pay_time": true,
|
||||
"order.coupon_id": true,
|
||||
"order.discount_amount": true,
|
||||
"order.supplier_product_name": true,
|
||||
"order.supplier_product_name": true,
|
||||
"order.supplier_name": true,
|
||||
"order.is_inner": true,
|
||||
"order.icon": true,
|
||||
"order.cost_price": true,
|
||||
|
|
@ -259,7 +260,8 @@ func AllLabels() map[string]string {
|
|||
"order.coupon_id": "优惠券ID",
|
||||
"order.discount_amount": "优惠金额",
|
||||
"order.card_code": "卡密(脱敏)",
|
||||
"order.supplier_product_name": "供应商产品名称",
|
||||
"order.supplier_product_name": "供应商产品名称",
|
||||
"order.supplier_name": "供应商名称",
|
||||
"order.is_inner": "内部供应商订单",
|
||||
"order.icon": "订单图片",
|
||||
"order.cost_price": "成本价",
|
||||
|
|
|
|||
Loading…
Reference in New Issue