feat: 实现营销系统数据导出工具的核心功能
新增数据导出工具的核心功能模块，包括： - 数据库连接与模型定义 - 路由与API处理逻辑 - SQL构建与执行 - 数据导出为CSV/XLSX格式 - 前端界面与交互 实现模板管理、任务队列、权限控制等完整业务流程
This commit is contained in:
parent
9ab4f8c9d2
commit
e7eff92b02
|
|
@ -0,0 +1,16 @@
|
|||
{
|
||||
"tables": ["order"],
|
||||
"fields": [
|
||||
"order.order_number",
|
||||
"order.creator",
|
||||
"order.out_trade_no",
|
||||
"order.type",
|
||||
"order.status",
|
||||
"order.contract_price",
|
||||
"order.num",
|
||||
"order.total",
|
||||
"order.pay_amount",
|
||||
"order.create_time",
|
||||
"order.update_time"
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
#!/usr/bin/env bash
|
||||
set -e
|
||||
cd "$(dirname "$0")/.."/server
|
||||
go mod tidy
|
||||
go run ./cmd/server
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
"marketing-system-data-tool/server/internal/api"
|
||||
"marketing-system-data-tool/server/internal/db"
|
||||
"marketing-system-data-tool/server/internal/migrate"
|
||||
)
|
||||
|
||||
func main() {
|
||||
ymtDSN := buildDSN(
|
||||
os.Getenv("YMT_DB_USER"),
|
||||
os.Getenv("YMT_DB_PASSWORD"),
|
||||
os.Getenv("YMT_DB_HOST"),
|
||||
os.Getenv("YMT_DB_PORT"),
|
||||
os.Getenv("YMT_DB_NAME"),
|
||||
)
|
||||
marketingDSN := buildDSN(
|
||||
os.Getenv("MARKETING_DB_USER"),
|
||||
os.Getenv("MARKETING_DB_PASSWORD"),
|
||||
os.Getenv("MARKETING_DB_HOST"),
|
||||
os.Getenv("MARKETING_DB_PORT"),
|
||||
os.Getenv("MARKETING_DB_NAME"),
|
||||
)
|
||||
ymt, err := db.ConnectMySQL(ymtDSN)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
marketing, err := db.ConnectMySQL(marketingDSN)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := migrate.Apply(ymt); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
r := api.NewRouter(ymt, marketing)
|
||||
srv := &http.Server{Addr: ":8080", Handler: r, ReadTimeout: 15 * time.Second, WriteTimeout: 60 * time.Second}
|
||||
log.Println("server listening on :8080")
|
||||
log.Fatal(srv.ListenAndServe())
|
||||
}
|
||||
|
||||
func buildDSN(user, pwd, host, port, dbname string) string {
|
||||
if user == "" || host == "" || port == "" || dbname == "" {
|
||||
return ""
|
||||
}
|
||||
return user + ":" + pwd + "@tcp(" + host + ":" + port + ")/" + dbname + "?parseTime=True&loc=Local&charset=utf8mb4"
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
module marketing-system-data-tool/server
|
||||
|
||||
go 1.21
|
||||
|
||||
require (
|
||||
github.com/go-sql-driver/mysql v1.7.1
|
||||
github.com/xuri/excelize/v2 v2.8.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
|
||||
github.com/richardlehane/mscfb v1.0.4 // indirect
|
||||
github.com/richardlehane/msoleps v1.0.3 // indirect
|
||||
github.com/xuri/efp v0.0.0-20231025114914-d1ff6096ae53 // indirect
|
||||
github.com/xuri/nfp v0.0.0-20230919160717-d98342af3f05 // indirect
|
||||
golang.org/x/crypto v0.19.0 // indirect
|
||||
golang.org/x/net v0.21.0 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
)
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
|
||||
github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
|
||||
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
|
||||
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM=
|
||||
github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk=
|
||||
github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
|
||||
github.com/richardlehane/msoleps v1.0.3 h1:aznSZzrwYRl3rLKRT3gUk9am7T/mLNSnJINvN0AQoVM=
|
||||
github.com/richardlehane/msoleps v1.0.3/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
|
||||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/xuri/efp v0.0.0-20231025114914-d1ff6096ae53 h1:Chd9DkqERQQuHpXjR/HSV1jLZA6uaoiwwH3vSuF3IW0=
|
||||
github.com/xuri/efp v0.0.0-20231025114914-d1ff6096ae53/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI=
|
||||
github.com/xuri/excelize/v2 v2.8.1 h1:pZLMEwK8ep+CLIUWpWmvW8IWE/yxqG0I1xcN6cVMGuQ=
|
||||
github.com/xuri/excelize/v2 v2.8.1/go.mod h1:oli1E4C3Pa5RXg1TBXn4ENCXDV5JUMlBluUhG7c+CEE=
|
||||
github.com/xuri/nfp v0.0.0-20230919160717-d98342af3f05 h1:qhbILQo1K3mphbwKh1vNm4oGezE1eF9fQWmNiIpSfI4=
|
||||
github.com/xuri/nfp v0.0.0-20230919160717-d98342af3f05/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ=
|
||||
golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4=
|
||||
golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE=
|
||||
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
|
@ -0,0 +1,262 @@
|
|||
package api
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"marketing-system-data-tool/server/internal/exporter"
|
||||
)
|
||||
|
||||
type ExportsAPI struct{
|
||||
meta *sql.DB
|
||||
marketing *sql.DB
|
||||
}
|
||||
|
||||
func ExportsHandler(meta, marketing *sql.DB) http.Handler {
|
||||
api := &ExportsAPI{meta: meta, marketing: marketing}
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
p := strings.TrimPrefix(r.URL.Path, "/api/exports")
|
||||
if r.Method == http.MethodPost && p == "" {
|
||||
api.create(w, r)
|
||||
return
|
||||
}
|
||||
if strings.HasPrefix(p, "/") {
|
||||
id := strings.TrimPrefix(p, "/")
|
||||
if r.Method == http.MethodGet && !strings.HasSuffix(p, "/download") {
|
||||
api.get(w, r, id)
|
||||
return
|
||||
}
|
||||
if r.Method == http.MethodGet && strings.HasSuffix(p, "/download") {
|
||||
id = strings.TrimSuffix(id, "/download")
|
||||
api.download(w, r, id)
|
||||
return
|
||||
}
|
||||
if r.Method == http.MethodPost && strings.HasSuffix(p, "/cancel") {
|
||||
id = strings.TrimSuffix(id, "/cancel")
|
||||
api.cancel(w, r, id)
|
||||
return
|
||||
}
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
})
|
||||
}
|
||||
|
||||
type ExportPayload struct{
|
||||
TemplateID uint64 `json:"template_id"`
|
||||
RequestedBy uint64 `json:"requested_by"`
|
||||
Permission map[string]interface{} `json:"permission"`
|
||||
Options map[string]interface{} `json:"options"`
|
||||
FileFormat string `json:"file_format"`
|
||||
}
|
||||
|
||||
func (a *ExportsAPI) create(w http.ResponseWriter, r *http.Request) {
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
var p ExportPayload
|
||||
json.Unmarshal(b, &p)
|
||||
var main string
|
||||
var fields, filters []byte
|
||||
row := a.meta.QueryRow("SELECT main_table, fields_json, filters_json FROM export_templates WHERE id=?", p.TemplateID)
|
||||
err := row.Scan(&main, &fields, &filters)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
w.Write([]byte("invalid template"))
|
||||
return
|
||||
}
|
||||
var fs []string
|
||||
var fl map[string]interface{}
|
||||
json.Unmarshal(fields, &fs)
|
||||
json.Unmarshal(filters, &fl)
|
||||
wl := map[string]bool{
|
||||
"order.order_number": true,
|
||||
"order.creator": true,
|
||||
"order.out_trade_no": true,
|
||||
"order.type": true,
|
||||
"order.status": true,
|
||||
"order.contract_price": true,
|
||||
"order.num": true,
|
||||
"order.total": true,
|
||||
"order.pay_amount": true,
|
||||
"order.create_time": true,
|
||||
"order.update_time": true,
|
||||
}
|
||||
req := exporter.BuildRequest{MainTable: main, Fields: fs, Filters: fl}
|
||||
q, args, err := exporter.BuildSQL(req, wl)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
w.Write([]byte(err.Error()))
|
||||
return
|
||||
}
|
||||
_, _ = exporter.RunExplain(a.marketing, q, args)
|
||||
res, err := a.meta.Exec("INSERT INTO export_jobs (template_id, status, requested_by, permission_scope_json, options_json, file_format, created_at) VALUES (?,?,?,?,?,?,?)", p.TemplateID, "queued", p.RequestedBy, toJSON(p.Permission), toJSON(p.Options), p.FileFormat, time.Now())
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
w.Write([]byte(err.Error()))
|
||||
return
|
||||
}
|
||||
id, _ := res.LastInsertId()
|
||||
go a.runJob(uint64(id), q, args, fs, p.FileFormat)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte("{\"id\":"+strconv.FormatInt(id,10)+"}"))
|
||||
}
|
||||
|
||||
func (a *ExportsAPI) runJob(id uint64, q string, args []interface{}, cols []string, fmt string) {
|
||||
a.meta.Exec("UPDATE export_jobs SET status=?, started_at=? WHERE id=?", "running", time.Now(), id)
|
||||
if fmt == "csv" {
|
||||
w, err := exporter.NewCSVWriter("storage", "export")
|
||||
if err != nil {
|
||||
a.meta.Exec("UPDATE export_jobs SET status=?, finished_at=? WHERE id=?", "failed", time.Now(), id)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(cols)
|
||||
rows, err := a.marketing.Query(q, args...)
|
||||
if err != nil {
|
||||
a.meta.Exec("UPDATE export_jobs SET status=?, finished_at=? WHERE id=?", "failed", time.Now(), id)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make([]interface{}, len(cols))
|
||||
dest := make([]interface{}, len(cols))
|
||||
for i := range out {
|
||||
dest[i] = &out[i]
|
||||
}
|
||||
var count int64
|
||||
for rows.Next() {
|
||||
if err := rows.Scan(dest...); err != nil {
|
||||
a.meta.Exec("UPDATE export_jobs SET status=?, finished_at=? WHERE id=?", "failed", time.Now(), id)
|
||||
return
|
||||
}
|
||||
vals := make([]string, len(cols))
|
||||
for i := range out {
|
||||
if b, ok := out[i].([]byte); ok {
|
||||
vals[i] = string(b)
|
||||
} else if out[i] == nil {
|
||||
vals[i] = ""
|
||||
} else {
|
||||
vals[i] = toString(out[i])
|
||||
}
|
||||
}
|
||||
w.WriteRow(vals)
|
||||
count++
|
||||
}
|
||||
path, size, _ := w.Close()
|
||||
a.meta.Exec("INSERT INTO export_job_files (job_id, storage_uri, row_count, size_bytes, created_at) VALUES (?,?,?,?,?)", id, path, count, size, time.Now())
|
||||
a.meta.Exec("UPDATE export_jobs SET status=?, finished_at=?, total_rows=? WHERE id=?", "completed", time.Now(), count, id)
|
||||
return
|
||||
}
|
||||
if fmt == "xlsx" {
|
||||
x, path, err := exporter.NewXLSXWriter("storage", "export", "Sheet1")
|
||||
if err != nil {
|
||||
a.meta.Exec("UPDATE export_jobs SET status=?, finished_at=? WHERE id=?", "failed", time.Now(), id)
|
||||
return
|
||||
}
|
||||
x.WriteHeader(cols)
|
||||
rows, err := a.marketing.Query(q, args...)
|
||||
if err != nil {
|
||||
a.meta.Exec("UPDATE export_jobs SET status=?, finished_at=? WHERE id=?", "failed", time.Now(), id)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make([]interface{}, len(cols))
|
||||
dest := make([]interface{}, len(cols))
|
||||
for i := range out {
|
||||
dest[i] = &out[i]
|
||||
}
|
||||
var count int64
|
||||
for rows.Next() {
|
||||
if err := rows.Scan(dest...); err != nil {
|
||||
a.meta.Exec("UPDATE export_jobs SET status=?, finished_at=? WHERE id=?", "failed", time.Now(), id)
|
||||
return
|
||||
}
|
||||
vals := make([]string, len(cols))
|
||||
for i := range out {
|
||||
if b, ok := out[i].([]byte); ok {
|
||||
vals[i] = string(b)
|
||||
} else if out[i] == nil {
|
||||
vals[i] = ""
|
||||
} else {
|
||||
vals[i] = toString(out[i])
|
||||
}
|
||||
}
|
||||
x.WriteRow(vals)
|
||||
count++
|
||||
}
|
||||
p, size, _ := x.Close(path)
|
||||
a.meta.Exec("INSERT INTO export_job_files (job_id, storage_uri, row_count, size_bytes, created_at) VALUES (?,?,?,?,?)", id, p, count, size, time.Now())
|
||||
a.meta.Exec("UPDATE export_jobs SET status=?, finished_at=?, total_rows=? WHERE id=?", "completed", time.Now(), count, id)
|
||||
return
|
||||
}
|
||||
a.meta.Exec("UPDATE export_jobs SET status=?, finished_at=? WHERE id=?", "failed", time.Now(), id)
|
||||
}
|
||||
|
||||
func (a *ExportsAPI) get(w http.ResponseWriter, r *http.Request, id string) {
|
||||
row := a.meta.QueryRow("SELECT id, template_id, status, requested_by, total_rows, file_format, started_at, finished_at, created_at, updated_at FROM export_jobs WHERE id=?", id)
|
||||
var m = map[string]interface{}{}
|
||||
var totalRows sql.NullInt64
|
||||
var startedAt, finishedAt sql.NullTime
|
||||
var createdAt, updatedAt time.Time
|
||||
err := row.Scan(&m["id"], &m["template_id"], &m["status"], &m["requested_by"], &totalRows, &m["file_format"], &startedAt, &finishedAt, &createdAt, &updatedAt)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
w.Write([]byte("not found"))
|
||||
return
|
||||
}
|
||||
m["total_rows"] = totalRows.Int64
|
||||
m["started_at"] = startedAt.Time
|
||||
m["finished_at"] = finishedAt.Time
|
||||
m["created_at"] = createdAt
|
||||
m["updated_at"] = updatedAt
|
||||
rows, _ := a.meta.Query("SELECT storage_uri, sheet_name, row_count, size_bytes FROM export_job_files WHERE job_id=?", id)
|
||||
files := []map[string]interface{}{}
|
||||
for rows.Next() {
|
||||
var uri, sheet sql.NullString
|
||||
var rc, sz sql.NullInt64
|
||||
rows.Scan(&uri, &sheet, &rc, &sz)
|
||||
files = append(files, map[string]interface{}{"storage_uri": uri.String, "sheet_name": sheet.String, "row_count": rc.Int64, "size_bytes": sz.Int64})
|
||||
}
|
||||
rows.Close()
|
||||
m["files"] = files
|
||||
b, _ := json.Marshal(m)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write(b)
|
||||
}
|
||||
|
||||
func (a *ExportsAPI) download(w http.ResponseWriter, r *http.Request, id string) {
|
||||
row := a.meta.QueryRow("SELECT storage_uri FROM export_job_files WHERE job_id=? ORDER BY id DESC LIMIT 1", id)
|
||||
var uri string
|
||||
err := row.Scan(&uri)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
w.Write([]byte("not found"))
|
||||
return
|
||||
}
|
||||
http.ServeFile(w, r, uri)
|
||||
}
|
||||
|
||||
func (a *ExportsAPI) cancel(w http.ResponseWriter, r *http.Request, id string) {
|
||||
a.meta.Exec("UPDATE export_jobs SET status=? WHERE id=? AND status IN ('queued','running')", "canceled", id)
|
||||
w.Write([]byte("ok"))
|
||||
}
|
||||
|
||||
func toString(v interface{}) string {
|
||||
switch t := v.(type) {
|
||||
case []byte:
|
||||
return string(t)
|
||||
case string:
|
||||
return t
|
||||
case int64:
|
||||
return strconv.FormatInt(t, 10)
|
||||
case int:
|
||||
return strconv.Itoa(t)
|
||||
case float64:
|
||||
return strconv.FormatFloat(t, 'f', -1, 64)
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
package api
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
func NewRouter(metaDB *sql.DB, marketingDB *sql.DB) http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("/api/templates", TemplatesHandler(metaDB, marketingDB))
|
||||
mux.Handle("/api/templates/", TemplatesHandler(metaDB, marketingDB))
|
||||
mux.Handle("/api/exports", ExportsHandler(metaDB, marketingDB))
|
||||
mux.Handle("/api/exports/", ExportsHandler(metaDB, marketingDB))
|
||||
mux.Handle("/", http.FileServer(http.Dir("web")))
|
||||
return mux
|
||||
}
|
||||
|
|
@ -0,0 +1,256 @@
|
|||
package api
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
"marketing-system-data-tool/server/internal/exporter"
|
||||
)
|
||||
|
||||
type TemplatesAPI struct{
|
||||
meta *sql.DB
|
||||
marketing *sql.DB
|
||||
}
|
||||
|
||||
func TemplatesHandler(meta, marketing *sql.DB) http.Handler {
|
||||
api := &TemplatesAPI{meta: meta, marketing: marketing}
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
p := strings.TrimPrefix(r.URL.Path, "/api/templates")
|
||||
if r.Method == http.MethodPost && p == "" {
|
||||
api.createTemplate(w, r)
|
||||
return
|
||||
}
|
||||
if r.Method == http.MethodGet && p == "" {
|
||||
api.listTemplates(w, r)
|
||||
return
|
||||
}
|
||||
if strings.HasPrefix(p, "/") {
|
||||
id := strings.TrimPrefix(p, "/")
|
||||
if r.Method == http.MethodGet {
|
||||
api.getTemplate(w, r, id)
|
||||
return
|
||||
}
|
||||
if r.Method == http.MethodPatch {
|
||||
api.patchTemplate(w, r, id)
|
||||
return
|
||||
}
|
||||
if r.Method == http.MethodPost && strings.HasSuffix(p, "/validate") {
|
||||
id = strings.TrimSuffix(id, "/validate")
|
||||
api.validateTemplate(w, r, id)
|
||||
return
|
||||
}
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
})
|
||||
}
|
||||
|
||||
type TemplatePayload struct{
|
||||
Name string `json:"name"`
|
||||
Datasource string `json:"datasource"`
|
||||
MainTable string `json:"main_table"`
|
||||
Fields []string `json:"fields"`
|
||||
Filters map[string]interface{} `json:"filters"`
|
||||
FileFormat string `json:"file_format"`
|
||||
OwnerID uint64 `json:"owner_id"`
|
||||
Visibility string `json:"visibility"`
|
||||
}
|
||||
|
||||
func (a *TemplatesAPI) createTemplate(w http.ResponseWriter, r *http.Request) {
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
var p TemplatePayload
|
||||
json.Unmarshal(b, &p)
|
||||
wl := map[string]bool{
|
||||
"order.order_number": true,
|
||||
"order.creator": true,
|
||||
"order.out_trade_no": true,
|
||||
"order.type": true,
|
||||
"order.status": true,
|
||||
"order.contract_price": true,
|
||||
"order.num": true,
|
||||
"order.total": true,
|
||||
"order.pay_amount": true,
|
||||
"order.create_time": true,
|
||||
"order.update_time": true,
|
||||
}
|
||||
req := exporter.BuildRequest{MainTable: p.MainTable, Fields: p.Fields, Filters: p.Filters}
|
||||
q, args, err := exporter.BuildSQL(req, wl)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
w.Write([]byte(err.Error()))
|
||||
return
|
||||
}
|
||||
_, score, err := exporter.RunExplain(a.marketing, q, args)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
w.Write([]byte(err.Error()))
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
_, err = a.meta.Exec(
|
||||
"INSERT INTO export_templates (name, datasource, main_table, fields_json, filters_json, file_format, visibility, owner_id, enabled, explain_score, last_validated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?)",
|
||||
p.Name, p.Datasource, p.MainTable, toJSON(p.Fields), toJSON(p.Filters), p.FileFormat, p.Visibility, p.OwnerID, 1, score, now,
|
||||
)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
w.Write([]byte(err.Error()))
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
w.Write([]byte("ok"))
|
||||
}
|
||||
|
||||
func (a *TemplatesAPI) listTemplates(w http.ResponseWriter, r *http.Request) {
|
||||
rows, err := a.meta.Query("SELECT id,name,datasource,main_table,file_format,visibility,owner_id,enabled,explain_score,last_validated_at,created_at,updated_at FROM export_templates ORDER BY updated_at DESC LIMIT 200")
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
w.Write([]byte(err.Error()))
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []map[string]interface{}{}
|
||||
for rows.Next() {
|
||||
var id uint64
|
||||
var name, datasource, mainTable, fileFormat, visibility string
|
||||
var ownerID uint64
|
||||
var enabled int
|
||||
var explainScore sql.NullInt64
|
||||
var lastValidatedAt sql.NullTime
|
||||
var createdAt, updatedAt time.Time
|
||||
err := rows.Scan(&id, &name, &datasource, &mainTable, &fileFormat, &visibility, &ownerID, &enabled, &explainScore, &lastValidatedAt, &createdAt, &updatedAt)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
w.Write([]byte(err.Error()))
|
||||
return
|
||||
}
|
||||
m := map[string]interface{}{"id": id, "name": name, "datasource": datasource, "main_table": mainTable, "file_format": fileFormat, "visibility": visibility, "owner_id": ownerID, "enabled": enabled == 1, "explain_score": explainScore.Int64, "last_validated_at": lastValidatedAt.Time, "created_at": createdAt, "updated_at": updatedAt}
|
||||
out = append(out, m)
|
||||
}
|
||||
b, _ := json.Marshal(out)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write(b)
|
||||
}
|
||||
|
||||
func (a *TemplatesAPI) getTemplate(w http.ResponseWriter, r *http.Request, id string) {
|
||||
row := a.meta.QueryRow("SELECT id,name,datasource,main_table,fields_json,filters_json,file_format,visibility,owner_id,enabled,explain_score,last_validated_at,created_at,updated_at FROM export_templates WHERE id=?", id)
|
||||
var m = map[string]interface{}{}
|
||||
var enabled int
|
||||
var explainScore sql.NullInt64
|
||||
var lastValidatedAt sql.NullTime
|
||||
var createdAt, updatedAt time.Time
|
||||
var fields, filters []byte
|
||||
err := row.Scan(&m["id"], &m["name"], &m["datasource"], &m["main_table"], &fields, &filters, &m["file_format"], &m["visibility"], &m["owner_id"], &enabled, &explainScore, &lastValidatedAt, &createdAt, &updatedAt)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
w.Write([]byte("not found"))
|
||||
return
|
||||
}
|
||||
m["enabled"] = enabled == 1
|
||||
m["explain_score"] = explainScore.Int64
|
||||
m["last_validated_at"] = lastValidatedAt.Time
|
||||
m["created_at"] = createdAt
|
||||
m["updated_at"] = updatedAt
|
||||
m["fields"] = fromJSON(fields)
|
||||
m["filters"] = fromJSON(filters)
|
||||
b, _ := json.Marshal(m)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write(b)
|
||||
}
|
||||
|
||||
func (a *TemplatesAPI) patchTemplate(w http.ResponseWriter, r *http.Request, id string) {
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
var p map[string]interface{}
|
||||
json.Unmarshal(b, &p)
|
||||
set := []string{}
|
||||
args := []interface{}{}
|
||||
for k, v := range p {
|
||||
switch k {
|
||||
case "name", "visibility", "file_format":
|
||||
set = append(set, k+"=?")
|
||||
args = append(args, v)
|
||||
case "enabled":
|
||||
set = append(set, "enabled=?")
|
||||
if v.(bool) {
|
||||
args = append(args, 1)
|
||||
} else {
|
||||
args = append(args, 0)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(set) == 0 {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
w.Write([]byte("no patch"))
|
||||
return
|
||||
}
|
||||
args = append(args, id)
|
||||
_, err := a.meta.Exec("UPDATE export_templates SET "+strings.Join(set, ",")+" WHERE id=?", args...)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
w.Write([]byte(err.Error()))
|
||||
return
|
||||
}
|
||||
w.Write([]byte("ok"))
|
||||
}
|
||||
|
||||
func (a *TemplatesAPI) validateTemplate(w http.ResponseWriter, r *http.Request, id string) {
|
||||
row := a.meta.QueryRow("SELECT main_table, fields_json, filters_json FROM export_templates WHERE id=?", id)
|
||||
var main string
|
||||
var fields, filters []byte
|
||||
err := row.Scan(&main, &fields, &filters)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
w.Write([]byte("not found"))
|
||||
return
|
||||
}
|
||||
var fs []string
|
||||
var fl map[string]interface{}
|
||||
json.Unmarshal(fields, &fs)
|
||||
json.Unmarshal(filters, &fl)
|
||||
wl := map[string]bool{
|
||||
"order.order_number": true,
|
||||
"order.creator": true,
|
||||
"order.out_trade_no": true,
|
||||
"order.type": true,
|
||||
"order.status": true,
|
||||
"order.contract_price": true,
|
||||
"order.num": true,
|
||||
"order.total": true,
|
||||
"order.pay_amount": true,
|
||||
"order.create_time": true,
|
||||
"order.update_time": true,
|
||||
}
|
||||
req := exporter.BuildRequest{MainTable: main, Fields: fs, Filters: fl}
|
||||
q, args, err := exporter.BuildSQL(req, wl)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
w.Write([]byte(err.Error()))
|
||||
return
|
||||
}
|
||||
_, score, err := exporter.RunExplain(a.marketing, q, args)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
w.Write([]byte(err.Error()))
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
_, err = a.meta.Exec("UPDATE export_templates SET explain_score=?, last_validated_at=? WHERE id=?", score, now, id)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
w.Write([]byte(err.Error()))
|
||||
return
|
||||
}
|
||||
w.Write([]byte("ok"))
|
||||
}
|
||||
|
||||
func toJSON(v interface{}) []byte {
|
||||
b, _ := json.Marshal(v)
|
||||
return b
|
||||
}
|
||||
|
||||
func fromJSON(b []byte) interface{} {
|
||||
var v interface{}
|
||||
json.Unmarshal(b, &v)
|
||||
return v
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
package db
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
func ConnectMySQL(dsn string) (*sql.DB, error) {
|
||||
if dsn == "" {
|
||||
return sql.Open("mysql", "invalid:invalid@tcp(localhost:3306)/invalid")
|
||||
}
|
||||
db, err := sql.Open("mysql", dsn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return db, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,156 @@
|
|||
package exporter
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
)
|
||||
|
||||
type ExplainRow struct {
|
||||
ID sql.NullInt64
|
||||
SelectType sql.NullString
|
||||
Table sql.NullString
|
||||
Type sql.NullString
|
||||
PossibleKeys sql.NullString
|
||||
Key sql.NullString
|
||||
KeyLen sql.NullString
|
||||
Ref sql.NullString
|
||||
Rows sql.NullInt64
|
||||
Filtered sql.NullFloat64
|
||||
Extra sql.NullString
|
||||
}
|
||||
|
||||
func RunExplain(db *sql.DB, q string, args []interface{}) ([]ExplainRow, int, error) {
|
||||
rows, err := db.Query("EXPLAIN "+q, args...)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
res := []ExplainRow{}
|
||||
cols, _ := rows.Columns()
|
||||
for rows.Next() {
|
||||
vals := make([]interface{}, len(cols))
|
||||
dest := make([]interface{}, len(cols))
|
||||
for i := range vals {
|
||||
dest[i] = &vals[i]
|
||||
}
|
||||
if err := rows.Scan(dest...); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
r := ExplainRow{}
|
||||
if len(cols) >= 10 {
|
||||
toRow(vals, &r)
|
||||
}
|
||||
res = append(res, r)
|
||||
}
|
||||
score := 100
|
||||
for _, r := range res {
|
||||
if r.Type.String == "ALL" {
|
||||
score -= 50
|
||||
}
|
||||
if r.Rows.Int64 > 1000000 {
|
||||
score -= 30
|
||||
}
|
||||
if r.Extra.Valid {
|
||||
if contains(r.Extra.String, "Using temporary") || contains(r.Extra.String, "Using filesort") {
|
||||
score -= 20
|
||||
}
|
||||
}
|
||||
}
|
||||
if score < 0 {
|
||||
score = 0
|
||||
}
|
||||
return res, score, nil
|
||||
}
|
||||
|
||||
func toRow(vals []interface{}, r *ExplainRow) {
|
||||
if s, ok := vals[0].([]byte); ok {
|
||||
r.ID.Int64 = toInt64(string(s))
|
||||
r.ID.Valid = true
|
||||
}
|
||||
if s, ok := vals[1].([]byte); ok {
|
||||
r.SelectType.String = string(s)
|
||||
r.SelectType.Valid = true
|
||||
}
|
||||
if s, ok := vals[2].([]byte); ok {
|
||||
r.Table.String = string(s)
|
||||
r.Table.Valid = true
|
||||
}
|
||||
if s, ok := vals[3].([]byte); ok {
|
||||
r.Type.String = string(s)
|
||||
r.Type.Valid = true
|
||||
}
|
||||
if s, ok := vals[4].([]byte); ok {
|
||||
r.PossibleKeys.String = string(s)
|
||||
r.PossibleKeys.Valid = true
|
||||
}
|
||||
if s, ok := vals[5].([]byte); ok {
|
||||
r.Key.String = string(s)
|
||||
r.Key.Valid = true
|
||||
}
|
||||
if s, ok := vals[6].([]byte); ok {
|
||||
r.KeyLen.String = string(s)
|
||||
r.KeyLen.Valid = true
|
||||
}
|
||||
if s, ok := vals[7].([]byte); ok {
|
||||
r.Ref.String = string(s)
|
||||
r.Ref.Valid = true
|
||||
}
|
||||
if s, ok := vals[8].([]byte); ok {
|
||||
r.Rows.Int64 = toInt64(string(s))
|
||||
r.Rows.Valid = true
|
||||
}
|
||||
if s, ok := vals[9].([]byte); ok {
|
||||
r.Filtered.Float64 = toFloat64(string(s))
|
||||
r.Filtered.Valid = true
|
||||
}
|
||||
if len(vals) > 10 {
|
||||
if s, ok := vals[10].([]byte); ok {
|
||||
r.Extra.String = string(s)
|
||||
r.Extra.Valid = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func contains(s, sub string) bool {
|
||||
for i := 0; i+len(sub) <= len(s); i++ {
|
||||
if s[i:i+len(sub)] == sub {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func toInt64(s string) int64 {
|
||||
var n int64
|
||||
for i := 0; i < len(s); i++ {
|
||||
c := s[i]
|
||||
if c < '0' || c > '9' {
|
||||
continue
|
||||
}
|
||||
n = n*10 + int64(c-'0')
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func toFloat64(s string) float64 {
|
||||
var n float64
|
||||
var d float64
|
||||
var seen bool
|
||||
d = 1
|
||||
for i := 0; i < len(s); i++ {
|
||||
c := s[i]
|
||||
if c == '.' {
|
||||
seen = true
|
||||
continue
|
||||
}
|
||||
if c < '0' || c > '9' {
|
||||
continue
|
||||
}
|
||||
if !seen {
|
||||
n = n*10 + float64(c-'0')
|
||||
} else {
|
||||
d *= 10
|
||||
n = n + float64(c-'0')/d
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
package exporter
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type BuildRequest struct {
|
||||
MainTable string
|
||||
Fields []string
|
||||
Filters map[string]interface{}
|
||||
}
|
||||
|
||||
func BuildSQL(req BuildRequest, whitelist map[string]bool) (string, []interface{}, error) {
|
||||
if req.MainTable != "order" {
|
||||
return "", nil, errors.New("unsupported main table")
|
||||
}
|
||||
cols := []string{}
|
||||
for _, f := range req.Fields {
|
||||
if !whitelist["order."+f] {
|
||||
return "", nil, errors.New("field not allowed")
|
||||
}
|
||||
if f == "key" || req.MainTable == "order" {
|
||||
cols = append(cols, "`order`."+escape(f))
|
||||
}
|
||||
}
|
||||
if len(cols) == 0 {
|
||||
return "", nil, errors.New("no fields")
|
||||
}
|
||||
sb := strings.Builder{}
|
||||
sb.WriteString("SELECT ")
|
||||
sb.WriteString(strings.Join(cols, ","))
|
||||
sb.WriteString(" FROM `order`")
|
||||
args := []interface{}{}
|
||||
where := []string{}
|
||||
if v, ok := req.Filters["creator_in"]; ok {
|
||||
ids := []interface{}{}
|
||||
switch t := v.(type) {
|
||||
case []interface{}:
|
||||
ids = t
|
||||
case []int:
|
||||
for _, x := range t {
|
||||
ids = append(ids, x)
|
||||
}
|
||||
case []string:
|
||||
for _, x := range t {
|
||||
ids = append(ids, x)
|
||||
}
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return "", nil, errors.New("creator_in required")
|
||||
}
|
||||
ph := strings.Repeat("?,", len(ids))
|
||||
ph = strings.TrimSuffix(ph, ",")
|
||||
where = append(where, "`order`.creator IN ("+ph+")")
|
||||
args = append(args, ids...)
|
||||
} else {
|
||||
return "", nil, errors.New("creator_in required")
|
||||
}
|
||||
if v, ok := req.Filters["create_time_between"]; ok {
|
||||
var arr []interface{}
|
||||
b, _ := json.Marshal(v)
|
||||
json.Unmarshal(b, &arr)
|
||||
if len(arr) != 2 {
|
||||
return "", nil, errors.New("create_time_between requires 2 values")
|
||||
}
|
||||
where = append(where, "`order`.create_time BETWEEN ? AND ?")
|
||||
args = append(args, arr[0], arr[1])
|
||||
}
|
||||
if len(where) > 0 {
|
||||
sb.WriteString(" WHERE ")
|
||||
sb.WriteString(strings.Join(where, " AND "))
|
||||
}
|
||||
return sb.String(), args, nil
|
||||
}
|
||||
|
||||
func escape(s string) string {
|
||||
if s == "key" {
|
||||
return "`key`"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
|
@ -0,0 +1,131 @@
|
|||
package exporter
|
||||
|
||||
import (
|
||||
"encoding/csv"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
"github.com/xuri/excelize/v2"
|
||||
)
|
||||
|
||||
type RowWriter interface {
|
||||
WriteHeader(cols []string) error
|
||||
WriteRow(vals []string) error
|
||||
Close() (string, int64, error)
|
||||
}
|
||||
|
||||
type CSVWriter struct {
|
||||
f *os.File
|
||||
w *csv.Writer
|
||||
count int64
|
||||
}
|
||||
|
||||
func NewCSVWriter(dir, name string) (*CSVWriter, error) {
|
||||
os.MkdirAll(dir, 0755)
|
||||
p := filepath.Join(dir, name+"_"+time.Now().Format("20060102150405")+".csv")
|
||||
f, err := os.Create(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &CSVWriter{f: f, w: csv.NewWriter(f)}, nil
|
||||
}
|
||||
|
||||
func (c *CSVWriter) WriteHeader(cols []string) error {
|
||||
if err := c.w.Write(cols); err != nil {
|
||||
return err
|
||||
}
|
||||
c.count++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *CSVWriter) WriteRow(vals []string) error {
|
||||
if err := c.w.Write(vals); err != nil {
|
||||
return err
|
||||
}
|
||||
c.count++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *CSVWriter) Close() (string, int64, error) {
|
||||
c.w.Flush()
|
||||
p := c.f.Name()
|
||||
info, _ := c.f.Stat()
|
||||
c.f.Close()
|
||||
return p, info.Size(), nil
|
||||
}
|
||||
|
||||
type XLSXWriter struct {
|
||||
f *excelize.File
|
||||
sheet string
|
||||
row int
|
||||
}
|
||||
|
||||
func NewXLSXWriter(dir, name, sheet string) (*XLSXWriter, string, error) {
|
||||
os.MkdirAll(dir, 0755)
|
||||
p := filepath.Join(dir, name+"_"+time.Now().Format("20060102150405")+".xlsx")
|
||||
f := excelize.NewFile()
|
||||
f.NewSheet(sheet)
|
||||
idx, _ := f.GetSheetIndex(sheet)
|
||||
f.SetActiveSheet(idx)
|
||||
return &XLSXWriter{f: f, sheet: sheet, row: 1}, p, nil
|
||||
}
|
||||
|
||||
func (x *XLSXWriter) WriteHeader(cols []string) error {
|
||||
for i, c := range cols {
|
||||
cell := col(i+1) + "1"
|
||||
if err := x.f.SetCellValue(x.sheet, cell, c); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
x.row = 2
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *XLSXWriter) WriteRow(vals []string) error {
|
||||
r := x.row
|
||||
for i, v := range vals {
|
||||
cell := col(i+1) + itoa(r)
|
||||
if err := x.f.SetCellValue(x.sheet, cell, v); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
x.row++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *XLSXWriter) Close(path string) (string, int64, error) {
|
||||
if err := x.f.SaveAs(path); err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return path, 0, nil
|
||||
}
|
||||
return path, info.Size(), nil
|
||||
}
|
||||
|
||||
func col(n int) string {
|
||||
s := ""
|
||||
for n > 0 {
|
||||
n--
|
||||
s = string('A'+(n%26)) + s
|
||||
n /= 26
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func itoa(n int) string {
|
||||
if n == 0 {
|
||||
return "0"
|
||||
}
|
||||
b := make([]byte, 0, 10)
|
||||
m := n
|
||||
for m > 0 {
|
||||
b = append(b, byte('0'+(m%10)))
|
||||
m /= 10
|
||||
}
|
||||
for i, j := 0, len(b)-1; i < j; i, j = i+1, j-1 {
|
||||
b[i], b[j] = b[j], b[i]
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
package migrate
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
)
|
||||
|
||||
func Apply(db *sql.DB) error {
|
||||
stmts := []string{
|
||||
"CREATE TABLE IF NOT EXISTS export_templates (id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, name VARCHAR(255) NOT NULL, datasource VARCHAR(32) NOT NULL, main_table VARCHAR(64) NOT NULL, joins_json JSON, fields_json JSON, filters_json JSON, file_format VARCHAR(16) NOT NULL, stats_enabled TINYINT(1) NOT NULL DEFAULT 0, sheet_split_by VARCHAR(64), visibility VARCHAR(16) NOT NULL DEFAULT 'private', owner_id BIGINT UNSIGNED NOT NULL, enabled TINYINT(1) NOT NULL DEFAULT 1, explain_json JSON, explain_score INT, last_validated_at DATETIME, created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP)",
|
||||
"CREATE TABLE IF NOT EXISTS export_jobs (id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, template_id BIGINT UNSIGNED NOT NULL, status VARCHAR(16) NOT NULL, requested_by BIGINT UNSIGNED NOT NULL, permission_scope_json JSON, options_json JSON, row_estimate BIGINT, total_rows BIGINT, file_format VARCHAR(16) NOT NULL, started_at DATETIME, finished_at DATETIME, created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, INDEX idx_template_id (template_id), INDEX idx_status (status), INDEX idx_requested_by (requested_by))",
|
||||
"CREATE TABLE IF NOT EXISTS export_job_files (id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, job_id BIGINT UNSIGNED NOT NULL, storage_uri VARCHAR(1024) NOT NULL, sheet_name VARCHAR(255), row_count BIGINT, size_bytes BIGINT, checksum VARCHAR(128), created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, INDEX idx_job_id (job_id))",
|
||||
"CREATE TABLE IF NOT EXISTS export_audits (id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, actor_id BIGINT UNSIGNED NOT NULL, action VARCHAR(64) NOT NULL, entity_type VARCHAR(64) NOT NULL, entity_id BIGINT UNSIGNED NOT NULL, detail_json JSON, created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, INDEX idx_entity (entity_type, entity_id), INDEX idx_actor (actor_id))",
|
||||
}
|
||||
for _, s := range stmts {
|
||||
if _, err := db.Exec(s); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
package models
|
||||
|
||||
import "time"
|
||||
|
||||
type Job struct {
|
||||
ID uint64
|
||||
TemplateID uint64
|
||||
Status string
|
||||
RequestedBy uint64
|
||||
PermissionScopeJSON []byte
|
||||
OptionsJSON []byte
|
||||
RowEstimate *int64
|
||||
TotalRows *int64
|
||||
FileFormat string
|
||||
StartedAt *time.Time
|
||||
FinishedAt *time.Time
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
package models
|
||||
|
||||
type Template struct {
|
||||
ID uint64
|
||||
Name string
|
||||
Datasource string
|
||||
MainTable string
|
||||
JoinsJSON []byte
|
||||
FieldsJSON []byte
|
||||
FiltersJSON []byte
|
||||
FileFormat string
|
||||
StatsEnabled bool
|
||||
SheetSplitBy *string
|
||||
Visibility string
|
||||
OwnerID uint64
|
||||
Enabled bool
|
||||
ExplainJSON []byte
|
||||
ExplainScore *int
|
||||
}
|
||||
|
|
@ -0,0 +1,90 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>MarketingSystemDataTool</title>
|
||||
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css" rel="stylesheet">
|
||||
<link rel="stylesheet" href="/styles.css">
|
||||
</head>
|
||||
<body>
|
||||
<header class="navbar navbar-dark bg-dark">
|
||||
<div class="container-fluid">
|
||||
<span class="navbar-brand mb-0 h1">导出工具</span>
|
||||
</div>
|
||||
</header>
|
||||
<main class="container py-3">
|
||||
<div class="row g-3">
|
||||
<div class="col-12 col-lg-7">
|
||||
<div class="card">
|
||||
<div class="card-header">模板列表</div>
|
||||
<div class="card-body" id="templates"></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-12 col-lg-5">
|
||||
<div class="card">
|
||||
<div class="card-header">新增模板</div>
|
||||
<div class="card-body">
|
||||
<form id="tpl-form" class="row g-3">
|
||||
<div class="col-12">
|
||||
<label class="form-label">模板名称</label>
|
||||
<input class="form-control" name="name" placeholder="模板名称" required>
|
||||
</div>
|
||||
<div class="col-6">
|
||||
<label class="form-label">数据源</label>
|
||||
<select class="form-select" name="datasource">
|
||||
<option value="marketing">营销系统</option>
|
||||
<option value="ymt">易码通</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="col-6">
|
||||
<label class="form-label">主表</label>
|
||||
<input class="form-control" name="main_table" value="order" required>
|
||||
</div>
|
||||
<div class="col-12">
|
||||
<label class="form-label">字段（逗号分隔）</label>
|
||||
<input class="form-control" name="fields" value="order_number,creator,out_trade_no,type,status,contract_price,num,total,pay_amount,create_time">
|
||||
</div>
|
||||
<div class="col-12">
|
||||
<label class="form-label">权限范围：creator 列表（逗号分隔）</label>
|
||||
<input class="form-control" name="creator_in" placeholder="如：1,2,3">
|
||||
</div>
|
||||
<div class="col-6">
|
||||
<label class="form-label">开始时间</label>
|
||||
<input class="form-control" type="datetime-local" name="time_begin">
|
||||
</div>
|
||||
<div class="col-6">
|
||||
<label class="form-label">结束时间</label>
|
||||
<input class="form-control" type="datetime-local" name="time_end">
|
||||
</div>
|
||||
<div class="col-6">
|
||||
<label class="form-label">输出格式</label>
|
||||
<select class="form-select" name="file_format"><option value="csv">CSV</option><option value="xlsx">XLSX</option></select>
|
||||
</div>
|
||||
<div class="col-6">
|
||||
<label class="form-label">可见性</label>
|
||||
<select class="form-select" name="visibility"><option value="private">个人</option><option value="public">公共</option></select>
|
||||
</div>
|
||||
<div class="col-6">
|
||||
<label class="form-label">所有者ID</label>
|
||||
<input class="form-control" name="owner_id" value="1">
|
||||
</div>
|
||||
<div class="col-12 d-grid">
|
||||
<button class="btn btn-primary" type="submit">创建并校验</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-12">
|
||||
<div class="card">
|
||||
<div class="card-header">导出任务</div>
|
||||
<div class="card-body" id="jobs"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/js/bootstrap.bundle.min.js"></script>
|
||||
<script src="/main.js"></script>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
async function loadTemplates(){
|
||||
const res=await fetch('/api/templates');
|
||||
const data=await res.json();
|
||||
const el=document.getElementById('templates');
|
||||
const rows=data.map(t=>`<tr><td>${t.id}</td><td>${t.name}</td><td>${t.datasource}</td><td>${t.file_format}</td><td>${t.explain_score||''}</td><td><button data-id="${t.id}" class="btn btn-primary btn-sm export">执行导出</button></td></tr>`).join('');
|
||||
el.innerHTML=`<table class="table table-striped table-sm"><thead><tr><th>ID</th><th>名称</th><th>数据源</th><th>格式</th><th>EXPLAIN评分</th><th>操作</th></tr></thead><tbody>${rows}</tbody></table>`;
|
||||
document.querySelectorAll('button.export').forEach(b=>b.onclick=async()=>{
|
||||
const id=b.getAttribute('data-id');
|
||||
const payload={template_id:Number(id),requested_by:1,permission:{},options:{},file_format:'csv'};
|
||||
const r=await fetch('/api/exports',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(payload)});
|
||||
const j=await r.json();
|
||||
loadJob(j.id);
|
||||
});
|
||||
}
|
||||
|
||||
async function loadJob(id){
|
||||
const res=await fetch('/api/exports/'+id);
|
||||
const j=await res.json();
|
||||
const el=document.getElementById('jobs');
|
||||
const link=j.files&&j.files.length?`<a class="btn btn-success btn-sm" href="/api/exports/${id}/download" target="_blank">下载</a>`:'';
|
||||
el.innerHTML=`<div class="alert alert-info">任务 ${id} 状态：<strong>${j.status}</strong> 行数：${j.total_rows||''} ${link}</div>`;
|
||||
}
|
||||
|
||||
document.getElementById('tpl-form').onsubmit=async(e)=>{
|
||||
e.preventDefault();
|
||||
const fd=new FormData(e.target);
|
||||
const fields=fd.get('fields').split(',').map(s=>s.trim());
|
||||
const filters={};
|
||||
const creators=fd.get('creator_in');
|
||||
if(creators){filters.creator_in=creators.split(',').map(s=>Number(s.trim()))}
|
||||
const tb=fd.get('time_begin');
|
||||
const te=fd.get('time_end');
|
||||
if(tb&&te){
|
||||
const fmt=(s)=>s.replace('T',' ')+':00';
|
||||
filters.create_time_between=[fmt(tb),fmt(te)]
|
||||
}
|
||||
const payload={
|
||||
name:fd.get('name'),
|
||||
datasource:fd.get('datasource'),
|
||||
main_table:fd.get('main_table'),
|
||||
fields,
|
||||
filters,
|
||||
file_format:fd.get('file_format'),
|
||||
owner_id:Number(fd.get('owner_id')),
|
||||
visibility:fd.get('visibility')
|
||||
};
|
||||
const res=await fetch('/api/templates',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(payload)});
|
||||
if(res.ok){
|
||||
loadTemplates();
|
||||
} else {
|
||||
const t=await res.text();
|
||||
const el=document.getElementById('templates');
|
||||
el.insertAdjacentHTML('afterbegin',`<div class="alert alert-danger">${t}</div>`);
|
||||
}
|
||||
};
|
||||
|
||||
loadTemplates();
|
||||
|
|
@ -0,0 +1 @@
|
|||
body{font-family:system-ui,Arial}
|
||||
Loading…
Reference in New Issue