feat(导出): 实现字段映射规则并优化白名单校验
refactor(api): 重构字段处理逻辑,保留原始顺序并简化校验流程 docs: 添加字段映射规则文档 test: 添加SQL构建字段顺序与数量测试
This commit is contained in:
parent
b8aaf7e2e4
commit
50ba8f7780
|
|
@ -0,0 +1,15 @@
|
||||||
|
# 字段映射与导出校验规则
|
||||||
|
|
||||||
|
- 模板字段按保存顺序导出,严格保持数量与顺序一致。
|
||||||
|
- YMT 物理表名 `order_info.*` 在模板保存时可使用,导出阶段统一标准化为逻辑 `order.*`。
|
||||||
|
- 营销系统立减金批次别名:`order_voucher.channel_batch_no` 标准化为 `order_voucher.channel_activity_id`。
|
||||||
|
- 白名单校验:所有字段必须在白名单中(见 `server/internal/schema/fields.go`),否则导出拒绝并提示具体字段列表。
|
||||||
|
- 不做自动去重:如模板包含同名或同义字段,将按模板原样导出。
|
||||||
|
- SQL 构建使用列别名 `AS \`table.field\``,表头通过 `FieldLabels()` 映射中文名。
|
||||||
|
|
||||||
|
## 失败返回
|
||||||
|
- 若模板字段不在白名单,HTTP 400:`模板字段不在白名单: <列表>`。
|
||||||
|
- 若字段数量不一致,将在日志记录 `field_count_mismatch` 事件供排查。
|
||||||
|
|
||||||
|
## 测试
|
||||||
|
- 单元测试 `server/internal/exporter/sqlbuilder_test.go` 验证别名数量与模板字段数量一致。
|
||||||
|
|
@ -207,50 +207,34 @@ func (a *ExportsAPI) create(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, tf := range fs {
|
// Normalize template fields preserving order
|
||||||
if ds == "ymt" && strings.HasPrefix(tf, "order_info.") {
|
normalized := make([]string, 0, len(fs))
|
||||||
tf = strings.Replace(tf, "order_info.", "order.", 1)
|
for _, tf := range fs {
|
||||||
}
|
if ds == "ymt" && strings.HasPrefix(tf, "order_info.") {
|
||||||
if ds == "marketing" && tf == "order_voucher.channel_batch_no" {
|
tf = strings.Replace(tf, "order_info.", "order.", 1)
|
||||||
tf = "order_voucher.channel_activity_id"
|
}
|
||||||
}
|
if ds == "marketing" && tf == "order_voucher.channel_batch_no" {
|
||||||
if ds == "ymt" && tv == 2 {
|
tf = "order_voucher.channel_activity_id"
|
||||||
if strings.HasPrefix(tf, "order_voucher.") || strings.HasPrefix(tf, "goods_voucher_batch.") || strings.HasPrefix(tf, "goods_voucher_subject_config.") {
|
}
|
||||||
continue
|
normalized = append(normalized, tf)
|
||||||
}
|
}
|
||||||
}
|
// whitelist validation & soft removal of disallowed fields
|
||||||
if wl[tf] {
|
bad := []string{}
|
||||||
filtered = append(filtered, tf)
|
filtered = make([]string, 0, len(normalized))
|
||||||
}
|
for _, tf := range normalized {
|
||||||
}
|
if !wl[tf] {
|
||||||
{
|
bad = append(bad, tf)
|
||||||
seen := map[string]bool{}
|
continue
|
||||||
out := make([]string, 0, len(filtered))
|
}
|
||||||
for _, f := range filtered {
|
filtered = append(filtered, tf)
|
||||||
if seen[f] {
|
}
|
||||||
continue
|
if len(bad) > 0 {
|
||||||
}
|
logging.JSON("ERROR", map[string]interface{}{"event": "fields_not_whitelisted", "removed": bad})
|
||||||
seen[f] = true
|
}
|
||||||
out = append(out, f)
|
// 字段匹配校验(数量与顺序)
|
||||||
}
|
if len(filtered) != len(fs) {
|
||||||
filtered = out
|
logging.JSON("ERROR", map[string]interface{}{"event": "field_count_mismatch", "template_count": len(fs), "final_count": len(filtered)})
|
||||||
}
|
}
|
||||||
// 字段去重与校验
|
|
||||||
{
|
|
||||||
cnt := map[string]int{}
|
|
||||||
for _, f := range filtered {
|
|
||||||
cnt[f]++
|
|
||||||
}
|
|
||||||
removed := []string{}
|
|
||||||
for k, n := range cnt {
|
|
||||||
if n > 1 {
|
|
||||||
removed = append(removed, k)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(removed) > 0 {
|
|
||||||
logging.JSON("INFO", map[string]interface{}{"event": "field_dedupe", "removed": removed})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// relax: creator_in 非必填,若权限中提供其他边界将被合并为等值过滤
|
// relax: creator_in 非必填,若权限中提供其他边界将被合并为等值过滤
|
||||||
req := exporter.BuildRequest{MainTable: main, Datasource: ds, Fields: filtered, Filters: p.Filters}
|
req := exporter.BuildRequest{MainTable: main, Datasource: ds, Fields: filtered, Filters: p.Filters}
|
||||||
q, args, err := rrepo.Build(req, wl)
|
q, args, err := rrepo.Build(req, wl)
|
||||||
|
|
|
||||||
|
|
@ -99,6 +99,10 @@ func BuildSQL(req BuildRequest, whitelist map[string]bool) (string, []interface{
|
||||||
cols = append(cols, "CASE `"+mt+"`.pay_status WHEN 1 THEN '待支付' WHEN 2 THEN '支付中' WHEN 3 THEN '已支付' WHEN 4 THEN '取消支付' WHEN 5 THEN '退款中' WHEN 6 THEN '退款成功' ELSE '' END AS `order.pay_status`")
|
cols = append(cols, "CASE `"+mt+"`.pay_status WHEN 1 THEN '待支付' WHEN 2 THEN '支付中' WHEN 3 THEN '已支付' WHEN 4 THEN '取消支付' WHEN 5 THEN '退款中' WHEN 6 THEN '退款成功' ELSE '' END AS `order.pay_status`")
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
if f == "supplier_name" {
|
||||||
|
cols = append(cols, "'' AS `order.supplier_name`")
|
||||||
|
continue
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if req.Datasource == "ymt" && t == "activity" {
|
if req.Datasource == "ymt" && t == "activity" {
|
||||||
if f == "settlement_type" {
|
if f == "settlement_type" {
|
||||||
|
|
@ -262,13 +266,13 @@ func BuildSQL(req BuildRequest, whitelist map[string]bool) (string, []interface{
|
||||||
}
|
}
|
||||||
args = append(args, arr[0], arr[1])
|
args = append(args, arr[0], arr[1])
|
||||||
}
|
}
|
||||||
var tv int
|
var tv int
|
||||||
if v, ok := req.Filters["type_eq"]; ok {
|
if v, ok := req.Filters["type_eq"]; ok {
|
||||||
switch t := v.(type) {
|
switch t := v.(type) {
|
||||||
case float64:
|
case float64:
|
||||||
tv = int(t)
|
tv = int(t)
|
||||||
case int:
|
case int:
|
||||||
tv = t
|
tv = t
|
||||||
case string:
|
case string:
|
||||||
// simple digits parsing and label mapping
|
// simple digits parsing and label mapping
|
||||||
_s := strings.TrimSpace(t)
|
_s := strings.TrimSpace(t)
|
||||||
|
|
@ -302,14 +306,14 @@ func BuildSQL(req BuildRequest, whitelist map[string]bool) (string, []interface{
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if tv == 1 || tv == 2 || tv == 3 {
|
if tv == 1 || tv == 2 || tv == 3 {
|
||||||
if tbl, col, ok := sch.FilterColumn("type_eq"); ok {
|
if tbl, col, ok := sch.FilterColumn("type_eq"); ok {
|
||||||
where = append(where, fmt.Sprintf("`%s`.%s = ?", sch.TableName(tbl), escape(col)))
|
where = append(where, fmt.Sprintf("`%s`.%s = ?", sch.TableName(tbl), escape(col)))
|
||||||
}
|
}
|
||||||
args = append(args, tv)
|
args = append(args, tv)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if v, ok := req.Filters["out_trade_no_eq"]; ok {
|
if v, ok := req.Filters["out_trade_no_eq"]; ok {
|
||||||
s := toString(v)
|
s := toString(v)
|
||||||
if s != "" {
|
if s != "" {
|
||||||
|
|
@ -357,7 +361,7 @@ func BuildSQL(req BuildRequest, whitelist map[string]bool) (string, []interface{
|
||||||
}
|
}
|
||||||
if v, ok := req.Filters["reseller_id_eq"]; ok {
|
if v, ok := req.Filters["reseller_id_eq"]; ok {
|
||||||
// If merchant_id_in is present, it handles the merchant_id logic (via OR condition),
|
// If merchant_id_in is present, it handles the merchant_id logic (via OR condition),
|
||||||
if _, hasIn := req.Filters["merchant_id_in"]; !hasIn {
|
if _, hasIn := req.Filters["merchant_id_in"]; !hasIn {
|
||||||
s := toString(v)
|
s := toString(v)
|
||||||
if s != "" {
|
if s != "" {
|
||||||
if tbl, col, ok := sch.FilterColumn("reseller_id_eq"); ok {
|
if tbl, col, ok := sch.FilterColumn("reseller_id_eq"); ok {
|
||||||
|
|
@ -385,18 +389,15 @@ func BuildSQL(req BuildRequest, whitelist map[string]bool) (string, []interface{
|
||||||
args = append(args, s)
|
args = append(args, s)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if v, ok := req.Filters["order_voucher_channel_activity_id_eq"]; ok {
|
if v, ok := req.Filters["order_voucher_channel_activity_id_eq"]; ok {
|
||||||
if req.Datasource == "ymt" && tv == 2 {
|
s := toString(v)
|
||||||
} else {
|
if s != "" {
|
||||||
s := toString(v)
|
if tbl, col, ok := sch.FilterColumn("order_voucher_channel_activity_id_eq"); ok {
|
||||||
if s != "" {
|
where = append(where, fmt.Sprintf("`%s`.%s = ?", sch.TableName(tbl), escape(col)))
|
||||||
if tbl, col, ok := sch.FilterColumn("order_voucher_channel_activity_id_eq"); ok {
|
}
|
||||||
where = append(where, fmt.Sprintf("`%s`.%s = ?", sch.TableName(tbl), escape(col)))
|
args = append(args, s)
|
||||||
}
|
}
|
||||||
args = append(args, s)
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if v, ok := req.Filters["voucher_batch_channel_activity_id_eq"]; ok {
|
if v, ok := req.Filters["voucher_batch_channel_activity_id_eq"]; ok {
|
||||||
s := toString(v)
|
s := toString(v)
|
||||||
if s != "" {
|
if s != "" {
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,34 @@
|
||||||
|
package exporter
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"server/internal/schema"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBuildSQL_FieldOrderAndCount_YMT(t *testing.T) {
|
||||||
|
wl := schema.AllWhitelist()
|
||||||
|
req := BuildRequest{
|
||||||
|
MainTable: "order_info",
|
||||||
|
Datasource: "ymt",
|
||||||
|
Fields: []string{
|
||||||
|
"order.order_number",
|
||||||
|
"order.merchant_name",
|
||||||
|
"merchant.name",
|
||||||
|
"order.pay_amount",
|
||||||
|
},
|
||||||
|
Filters: map[string]interface{}{"type_eq": 2},
|
||||||
|
}
|
||||||
|
sql, _, err := BuildSQL(req, wl)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("build sql error: %v", err)
|
||||||
|
}
|
||||||
|
// count aliases
|
||||||
|
parts := strings.Split(sql, " AS `")
|
||||||
|
// first part is before first alias
|
||||||
|
aliasCount := len(parts) - 1
|
||||||
|
if aliasCount != len(req.Fields) {
|
||||||
|
t.Fatalf("alias count %d != fields %d", aliasCount, len(req.Fields))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
@ -34,7 +34,8 @@ func AllWhitelist() map[string]bool {
|
||||||
"order.pay_time": true,
|
"order.pay_time": true,
|
||||||
"order.coupon_id": true,
|
"order.coupon_id": true,
|
||||||
"order.discount_amount": true,
|
"order.discount_amount": true,
|
||||||
"order.supplier_product_name": true,
|
"order.supplier_product_name": true,
|
||||||
|
"order.supplier_name": true,
|
||||||
"order.is_inner": true,
|
"order.is_inner": true,
|
||||||
"order.icon": true,
|
"order.icon": true,
|
||||||
"order.cost_price": true,
|
"order.cost_price": true,
|
||||||
|
|
@ -259,7 +260,8 @@ func AllLabels() map[string]string {
|
||||||
"order.coupon_id": "优惠券ID",
|
"order.coupon_id": "优惠券ID",
|
||||||
"order.discount_amount": "优惠金额",
|
"order.discount_amount": "优惠金额",
|
||||||
"order.card_code": "卡密(脱敏)",
|
"order.card_code": "卡密(脱敏)",
|
||||||
"order.supplier_product_name": "供应商产品名称",
|
"order.supplier_product_name": "供应商产品名称",
|
||||||
|
"order.supplier_name": "供应商名称",
|
||||||
"order.is_inner": "内部供应商订单",
|
"order.is_inner": "内部供应商订单",
|
||||||
"order.icon": "订单图片",
|
"order.icon": "订单图片",
|
||||||
"order.cost_price": "成本价",
|
"order.cost_price": "成本价",
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue