MarketingSystemDataExportTool/server/internal/exporter/sqlbuilder.go

665 lines
19 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package exporter
import (
"encoding/json"
"errors"
"fmt"
"log"
"server/internal/schema"
"server/internal/utils"
"strings"
)
// isZeroID returns true when the value is effectively empty/zero for ID filters.
func isZeroID(v interface{}) bool {
switch t := v.(type) {
case nil:
return true
case string:
s := strings.TrimSpace(t)
return s == "" || s == "0"
case int, int32, int64:
return fmt.Sprint(t) == "0"
case float32, float64:
return fmt.Sprint(t) == "0" || fmt.Sprint(t) == "0.0"
}
return false
}
type BuildRequest struct {
MainTable string
Datasource string
Fields []string // table.field
Filters map[string]interface{}
}
func BuildSQL(req BuildRequest, whitelist map[string]bool) (string, []interface{}, error) {
sql, args, _, err := BuildSQLWithFields(req, whitelist)
return sql, args, err
}
// BuildSQLWithFields 构建SQL并返回实际使用的字段列表
func BuildSQLWithFields(req BuildRequest, whitelist map[string]bool) (string, []interface{}, []string, error) {
if req.MainTable != "order" && req.MainTable != "order_info" {
return "", nil, nil, errors.New("unsupported main table")
}
sch := schema.Get(req.Datasource, req.MainTable)
if req.Datasource == "marketing" && req.MainTable == "order" {
if v, ok := req.Filters["create_time_between"]; ok {
switch t := v.(type) {
case []interface{}:
if len(t) != 2 {
return "", nil, nil, errors.New("create_time_between 需要两个时间值")
}
case []string:
if len(t) != 2 {
return "", nil, nil, errors.New("create_time_between 需要两个时间值")
}
default:
return "", nil, nil, errors.New("create_time_between 格式错误")
}
} else {
return "", nil, nil, errors.New("缺少时间过滤:必须提供 create_time_between")
}
}
cols := []string{}
usedFields := []string{} // 记录实际使用的字段
skippedFields := []string{} // 记录被过滤的字段
need := map[string]bool{}
for _, tf := range req.Fields {
// normalize YMT physical names saved previously to logical names
if req.Datasource == "ymt" && strings.HasPrefix(tf, "order_info.") {
tf = strings.Replace(tf, "order_info.", "order.", 1)
}
if whitelist != nil && !whitelist[tf] {
skippedFields = append(skippedFields, tf)
continue
}
parts := strings.Split(tf, ".")
if len(parts) != 2 {
return "", nil, nil, errors.New("invalid field format")
}
t, f := parts[0], parts[1]
if req.Datasource == "marketing" && t == "order_voucher" && f == "channel_batch_no" {
f = "channel_activity_id"
}
if req.Datasource == "ymt" && t == "order_voucher" && f == "channel_activity_id" {
f = "channel_batch_no"
}
need[t] = true
mt := sch.TableName(t)
mf, _ := sch.MapField(t, f)
// 特殊处理supplier_name需要JOIN supplier表
if req.Datasource == "ymt" && t == "order" && f == "supplier_name" {
need["supplier"] = true
cols = append(cols, "`supplier`.name AS `order.supplier_name`")
usedFields = append(usedFields, tf)
continue
}
// 移除所有CASE WHEN枚举转换直接查询原始字段
// 以下字段不再做CASE WHEN在业务层转换
if req.Datasource == "ymt" && t == "activity" && f == "settlement_type" {
// 移除枚举转换
}
if t == "merchant" && f == "third_party" {
// 移除枚举转换
}
if t == "order" && f == "is_retry" {
// 移除枚举转换
}
if t == "order" && f == "is_inner" {
// 移除枚举转换
}
if req.Datasource == "ymt" && t == "order_digit" && f == "order_type" {
// 移除枚举转换
}
if t == "order_digit" && f == "sms_channel" {
// 移除枚举转换
}
if t == "order_cash" && f == "receive_status" {
// 移除枚举转换
}
if t == "order_cash" && f == "channel" {
// 移除枚举转换
}
if t == "order_voucher" && f == "channel" {
// 移除枚举转换
}
if t == "order_voucher" && f == "status" {
// 移除枚举转换
}
if t == "order_voucher" && f == "receive_mode" {
// 移除枚举转换
}
// Fallback for YMT tables that are not joined in schema: voucher, voucher_batch, merchant_key_send
if req.Datasource == "ymt" && (t == "voucher" || t == "voucher_batch" || t == "merchant_key_send") {
cols = append(cols, "'' AS `"+t+"."+f+"`")
usedFields = append(usedFields, tf)
continue
}
cols = append(cols, "`"+mt+"`."+escape(mf)+" AS `"+t+"."+f+"`")
usedFields = append(usedFields, tf)
}
if len(cols) == 0 {
return "", nil, nil, errors.New("no fields")
}
sb := strings.Builder{}
baseCols := strings.Join(cols, ",")
sb.WriteString("SELECT ")
sb.WriteString(baseCols)
sb.WriteString(" FROM `" + sch.TableName(req.MainTable) + "`")
args := []interface{}{}
where := []string{}
// collect need from filters referencing related tables
if _, ok := req.Filters["order_cash_cash_activity_id_eq"]; ok {
need["order_cash"] = true
}
if _, ok := req.Filters["order_voucher_channel_activity_id_eq"]; ok {
need["order_voucher"] = true
}
if _, ok := req.Filters["voucher_batch_channel_activity_id_eq"]; ok {
need["voucher_batch"] = true
need["voucher"] = true
need["order_voucher"] = true
}
if _, ok := req.Filters["merchant_out_biz_no_eq"]; ok {
need["merchant_key_send"] = true
}
// normalize merchant_id_eq: skip 0/空,非零转为 merchant_id_in 以便和 creator_in 做 OR
if v, ok := req.Filters["merchant_id_eq"]; ok {
if isZeroID(v) {
delete(req.Filters, "merchant_id_eq")
} else {
if _, has := req.Filters["merchant_id_in"]; !has {
req.Filters["merchant_id_in"] = []interface{}{v}
}
delete(req.Filters, "merchant_id_eq")
}
}
// normalize reseller_id_eq (YMT 直连 merchant_id)
if req.Datasource == "ymt" && (req.MainTable == "order" || req.MainTable == "order_info") {
if v, ok := req.Filters["reseller_id_eq"]; ok {
if isZeroID(v) {
delete(req.Filters, "reseller_id_eq")
} else {
if _, has := req.Filters["merchant_id_in"]; !has {
req.Filters["merchant_id_in"] = []interface{}{v}
}
delete(req.Filters, "reseller_id_eq")
}
}
}
// helper: treat "0"/0/空 as未提供的 ID
isZeroID := func(v interface{}) bool {
switch t := v.(type) {
case nil:
return true
case string:
s := strings.TrimSpace(t)
return s == "" || s == "0"
case int, int32, int64:
return fmt.Sprint(t) == "0"
case float32, float64:
return fmt.Sprint(t) == "0" || fmt.Sprint(t) == "0.0"
}
return false
}
// Handle creator_in and merchant_id_in with OR logic if both exist
var creatorArgs []interface{}
hasCreator := false
if v, ok := req.Filters["creator_in"]; ok {
switch t := v.(type) {
case []interface{}:
for _, x := range t {
if !isZeroID(x) {
creatorArgs = append(creatorArgs, x)
}
}
case []int:
for _, x := range t {
if !isZeroID(x) {
creatorArgs = append(creatorArgs, x)
}
}
case []string:
for _, x := range t {
if !isZeroID(x) {
creatorArgs = append(creatorArgs, x)
}
}
}
if len(creatorArgs) > 0 {
hasCreator = true
}
}
var merchantArgs []interface{}
hasMerchant := false
if v, ok := req.Filters["merchant_id_in"]; ok {
switch t := v.(type) {
case []interface{}:
for _, x := range t {
if !isZeroID(x) {
merchantArgs = append(merchantArgs, x)
}
}
case []int:
for _, x := range t {
if !isZeroID(x) {
merchantArgs = append(merchantArgs, x)
}
}
case []string:
for _, x := range t {
if !isZeroID(x) {
merchantArgs = append(merchantArgs, x)
}
}
}
if len(merchantArgs) > 0 {
hasMerchant = true
}
}
// Apply the logic: if both present, use OR. Else use individual.
if hasCreator && hasMerchant {
cTbl, cCol, cOk := sch.FilterColumn("creator_in")
mTbl, mCol, mOk := sch.FilterColumn("merchant_id_in")
if cOk && mOk {
cPh := strings.Repeat("?,", len(creatorArgs))
cPh = strings.TrimSuffix(cPh, ",")
mPh := strings.Repeat("?,", len(merchantArgs))
mPh = strings.TrimSuffix(mPh, ",")
where = append(where, fmt.Sprintf("(`%s`.%s IN (%s) OR `%s`.%s IN (%s))",
sch.TableName(cTbl), escape(cCol), cPh,
sch.TableName(mTbl), escape(mCol), mPh))
args = append(args, creatorArgs...)
args = append(args, merchantArgs...)
} else if cOk {
// Fallback: only creator valid (e.g. marketing system)
ph := strings.Repeat("?,", len(creatorArgs))
ph = strings.TrimSuffix(ph, ",")
where = append(where, fmt.Sprintf("`%s`.%s IN (%s)", sch.TableName(cTbl), escape(cCol), ph))
args = append(args, creatorArgs...)
}
} else if hasCreator {
if tbl, col, ok := sch.FilterColumn("creator_in"); ok {
ph := strings.Repeat("?,", len(creatorArgs))
ph = strings.TrimSuffix(ph, ",")
where = append(where, fmt.Sprintf("`%s`.%s IN (%s)", sch.TableName(tbl), escape(col), ph))
args = append(args, creatorArgs...)
}
}
if v, ok := req.Filters["create_time_between"]; ok {
var arr []interface{}
b, _ := json.Marshal(v)
json.Unmarshal(b, &arr)
if len(arr) != 2 {
return "", nil, nil, errors.New("create_time_between requires 2 values")
}
if tbl, col, ok := sch.FilterColumn("create_time_between"); ok {
where = append(where, fmt.Sprintf("`%s`.%s BETWEEN ? AND ?", sch.TableName(tbl), escape(col)))
}
args = append(args, arr[0], arr[1])
}
var tv int
if v, ok := req.Filters["type_eq"]; ok {
switch t := v.(type) {
case float64:
tv = int(t)
case int:
tv = t
case string:
// simple digits parsing and label mapping
_s := strings.TrimSpace(t)
for i := 0; i < len(_s); i++ {
c := _s[i]
if c < '0' || c > '9' {
continue
}
tv = tv*10 + int(c-'0')
}
if tv == 0 {
if req.Datasource == "ymt" {
if _s == "红包订单" {
tv = 1
}
if _s == "直充卡密订单" {
tv = 2
}
if _s == "立减金订单" {
tv = 3
}
} else {
if _s == "直充卡密" {
tv = 1
}
if _s == "立减金" {
tv = 2
}
if _s == "红包" {
tv = 3
}
}
}
}
if tv == 1 || tv == 2 || tv == 3 {
if tbl, col, ok := sch.FilterColumn("type_eq"); ok {
where = append(where, fmt.Sprintf("`%s`.%s = ?", sch.TableName(tbl), escape(col)))
}
args = append(args, tv)
}
}
if v, ok := req.Filters["out_trade_no_eq"]; ok {
s := utils.ToString(v)
if s != "" {
if tbl, col, ok := sch.FilterColumn("out_trade_no_eq"); ok {
where = append(where, fmt.Sprintf("`%s`.%s = ?", sch.TableName(tbl), escape(col)))
}
args = append(args, s)
}
}
if v, ok := req.Filters["account_eq"]; ok {
s := utils.ToString(v)
if s != "" {
if tbl, col, ok := sch.FilterColumn("account_eq"); ok {
where = append(where, fmt.Sprintf("`%s`.%s = ?", sch.TableName(tbl), escape(col)))
}
args = append(args, s)
}
}
if v, ok := req.Filters["plan_id_eq"]; ok {
s := utils.ToString(v)
if s != "" && s != "0" {
if tbl, col, ok := sch.FilterColumn("plan_id_eq"); ok {
where = append(where, fmt.Sprintf("`%s`.%s = ?", sch.TableName(tbl), escape(col)))
}
args = append(args, s)
}
}
if v, ok := req.Filters["key_batch_id_eq"]; ok {
s := utils.ToString(v)
if s != "" {
if tbl, col, ok := sch.FilterColumn("key_batch_id_eq"); ok {
where = append(where, fmt.Sprintf("`%s`.%s = ?", sch.TableName(tbl), escape(col)))
}
args = append(args, s)
}
}
if v, ok := req.Filters["product_id_eq"]; ok {
s := utils.ToString(v)
if s != "" {
if tbl, col, ok := sch.FilterColumn("product_id_eq"); ok {
where = append(where, fmt.Sprintf("`%s`.%s = ?", sch.TableName(tbl), escape(col)))
}
args = append(args, s)
}
}
if v, ok := req.Filters["reseller_id_eq"]; ok {
// If merchant_id_in is present, it handles the merchant_id logic (via OR condition),
if _, hasIn := req.Filters["merchant_id_in"]; !hasIn {
s := utils.ToString(v)
if s != "" {
if tbl, col, ok := sch.FilterColumn("reseller_id_eq"); ok {
where = append(where, fmt.Sprintf("`%s`.%s = ?", sch.TableName(tbl), escape(col)))
}
args = append(args, s)
}
}
}
if v, ok := req.Filters["code_batch_id_eq"]; ok {
s := utils.ToString(v)
if s != "" {
if tbl, col, ok := sch.FilterColumn("code_batch_id_eq"); ok {
where = append(where, fmt.Sprintf("`%s`.%s = ?", sch.TableName(tbl), escape(col)))
}
args = append(args, s)
}
}
if v, ok := req.Filters["order_cash_cash_activity_id_eq"]; ok {
s := utils.ToString(v)
if s != "" {
if tbl, col, ok := sch.FilterColumn("order_cash_cash_activity_id_eq"); ok {
where = append(where, fmt.Sprintf("`%s`.%s = ?", sch.TableName(tbl), escape(col)))
}
args = append(args, s)
}
}
if v, ok := req.Filters["order_voucher_channel_activity_id_eq"]; ok {
s := utils.ToString(v)
if s != "" {
if tbl, col, ok := sch.FilterColumn("order_voucher_channel_activity_id_eq"); ok {
where = append(where, fmt.Sprintf("`%s`.%s = ?", sch.TableName(tbl), escape(col)))
}
args = append(args, s)
}
}
if v, ok := req.Filters["voucher_batch_channel_activity_id_eq"]; ok {
s := utils.ToString(v)
if s != "" {
if tbl, col, ok := sch.FilterColumn("voucher_batch_channel_activity_id_eq"); ok {
where = append(where, fmt.Sprintf("`%s`.%s = ?", sch.TableName(tbl), escape(col)))
args = append(args, s)
}
}
}
if v, ok := req.Filters["merchant_out_biz_no_eq"]; ok {
s := utils.ToString(v)
if s != "" {
if tbl, col, ok := sch.FilterColumn("merchant_out_biz_no_eq"); ok {
where = append(where, fmt.Sprintf("`%s`.%s = ?", sch.TableName(tbl), escape(col)))
}
args = append(args, s)
}
}
// append necessary joins after collecting filter-driven needs
joins := sch.BuildJoins(need, req.MainTable)
for _, j := range joins {
sb.WriteString(j)
}
if len(where) > 0 {
sb.WriteString(" WHERE ")
sb.WriteString(strings.Join(where, " AND "))
}
// 记录被白名单过滤的字段(方便调试)
if len(skippedFields) > 0 {
log.Printf("[BUILD_SQL] skipped_fields (not in whitelist): %v, used_fields_count: %d, skipped_count: %d",
skippedFields, len(usedFields), len(skippedFields))
}
return sb.String(), args, usedFields, nil
}
func escape(s string) string {
if s == "key" {
return "`key`"
}
if s == "index" {
return "`index`"
}
return s
}
// BuildCountSQL: minimal COUNT for filters-only joins, counting distinct main PK to avoid 1:N duplication
func BuildCountSQL(req BuildRequest, whitelist map[string]bool) (string, []interface{}, error) {
if req.MainTable != "order" && req.MainTable != "order_info" {
return "", nil, errors.New("unsupported main table")
}
sch := schema.Get(req.Datasource, req.MainTable)
mt := sch.TableName(req.MainTable)
pkCol, _ := sch.MapField(req.MainTable, "order_number")
args := []interface{}{}
where := []string{}
need := map[string]bool{}
// mark joins only needed by filters
for k, _ := range req.Filters {
if tbl, _, ok := sch.FilterColumn(k); ok {
need[tbl] = true
}
}
//补齐链式关联过滤所需的 JOIN只用于 COUNT
if _, ok := req.Filters["order_cash_cash_activity_id_eq"]; ok {
need["order_cash"] = true
}
if _, ok := req.Filters["order_voucher_channel_activity_id_eq"]; ok {
need["order_voucher"] = true
}
if _, ok := req.Filters["voucher_batch_channel_activity_id_eq"]; ok {
need["voucher_batch"] = true
need["voucher"] = true
need["order_voucher"] = true
}
if _, ok := req.Filters["merchant_out_biz_no_eq"]; ok {
need["merchant_key_send"] = true
}
// normalize merchant_id_eq / reseller_id_eq for YMT: skip 0/空, 非零转为 merchant_id_in 以便与 creator_in 做 OR
if v, ok := req.Filters["merchant_id_eq"]; ok {
if isZeroID(v) {
delete(req.Filters, "merchant_id_eq")
} else {
if _, has := req.Filters["merchant_id_in"]; !has {
req.Filters["merchant_id_in"] = []interface{}{v}
}
delete(req.Filters, "merchant_id_eq")
}
}
// 易码通订单: reseller_id_eq 不再转为 merchant_id_in 与 creator_in 做 OR而是作为单独的 WHERE 条件
if req.Datasource == "ymt" && (req.MainTable == "order" || req.MainTable == "order_info") {
if v, ok := req.Filters["reseller_id_eq"]; ok {
if isZeroID(v) {
delete(req.Filters, "reseller_id_eq")
}
// 不再转换为 merchant_id_in保留 reseller_id_eq 作为单独 WHERE 条件
}
}
// Handle creator_in and merchant_id_in with OR logic if both exist
var creatorArgs []interface{}
hasCreator := false
if v, ok := req.Filters["creator_in"]; ok {
switch t := v.(type) {
case []interface{}:
for _, x := range t {
if !isZeroID(x) {
creatorArgs = append(creatorArgs, x)
}
}
case []int:
for _, x := range t {
if !isZeroID(x) {
creatorArgs = append(creatorArgs, x)
}
}
case []string:
for _, x := range t {
if !isZeroID(x) {
creatorArgs = append(creatorArgs, x)
}
}
}
if len(creatorArgs) > 0 {
hasCreator = true
}
}
var merchantArgs []interface{}
hasMerchant := false
if v, ok := req.Filters["merchant_id_in"]; ok {
switch t := v.(type) {
case []interface{}:
for _, x := range t {
if !isZeroID(x) {
merchantArgs = append(merchantArgs, x)
}
}
case []int:
for _, x := range t {
if !isZeroID(x) {
merchantArgs = append(merchantArgs, x)
}
}
case []string:
for _, x := range t {
if !isZeroID(x) {
merchantArgs = append(merchantArgs, x)
}
}
}
if len(merchantArgs) > 0 {
hasMerchant = true
}
}
if hasCreator && hasMerchant {
cTbl, cCol, cOk := sch.FilterColumn("creator_in")
mTbl, mCol, mOk := sch.FilterColumn("merchant_id_in")
if cOk && mOk {
cPh := strings.Repeat("?,", len(creatorArgs))
cPh = strings.TrimSuffix(cPh, ",")
mPh := strings.Repeat("?,", len(merchantArgs))
mPh = strings.TrimSuffix(mPh, ",")
where = append(where, fmt.Sprintf("(`%s`.%s IN (%s) OR `%s`.%s IN (%s))",
sch.TableName(cTbl), escape(cCol), cPh,
sch.TableName(mTbl), escape(mCol), mPh))
args = append(args, creatorArgs...)
args = append(args, merchantArgs...)
} else if cOk {
ph := strings.Repeat("?,", len(creatorArgs))
ph = strings.TrimSuffix(ph, ",")
where = append(where, fmt.Sprintf("`%s`.%s IN (%s)", sch.TableName(cTbl), escape(cCol), ph))
args = append(args, creatorArgs...)
}
} else if hasCreator {
if tbl, col, ok := sch.FilterColumn("creator_in"); ok {
ph := strings.Repeat("?,", len(creatorArgs))
ph = strings.TrimSuffix(ph, ",")
where = append(where, fmt.Sprintf("`%s`.%s IN (%s)", sch.TableName(tbl), escape(col), ph))
args = append(args, creatorArgs...)
}
}
// build WHERE from other filters
for k, v := range req.Filters {
if k == "creator_in" || k == "merchant_id_in" {
continue
}
if k == "plan_id_eq" && utils.ToString(v) == "0" {
continue
}
if k == "merchant_id_eq" && isZeroID(v) {
continue
}
if tbl, col, ok := sch.FilterColumn(k); ok {
switch k {
case "create_time_between":
var arr []interface{}
b, _ := json.Marshal(v)
_ = json.Unmarshal(b, &arr)
if len(arr) == 2 {
where = append(where, fmt.Sprintf("`%s`.%s BETWEEN ? AND ?", sch.TableName(tbl), escape(col)))
args = append(args, arr[0], arr[1])
}
default:
s := utils.ToString(v)
if s != "" {
where = append(where, fmt.Sprintf("`%s`.%s = ?", sch.TableName(tbl), escape(col)))
args = append(args, s)
}
}
}
}
sb := strings.Builder{}
sb.WriteString("SELECT COUNT(DISTINCT `" + mt + "`." + pkCol + ") FROM `" + mt + "`")
for _, j := range sch.BuildJoins(need, req.MainTable) {
sb.WriteString(j)
}
if len(where) > 0 {
sb.WriteString(" WHERE ")
sb.WriteString(strings.Join(where, " AND "))
}
return sb.String(), args, nil
}