Compare commits
No commits in common. "bcc66bad0a3d561d8ae9fe339118416cdf44bbde" and "913442aa7f8bab8f9766876f3fd802daff14f44b" have entirely different histories.
bcc66bad0a
...
913442aa7f
|
|
@ -7,6 +7,7 @@ import (
|
|||
"fmt"
|
||||
"gorm.io/driver/mysql"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
"runtime/trace"
|
||||
"strconv"
|
||||
"time"
|
||||
|
|
@ -22,13 +23,13 @@ func NewDb(str string) (*Db, error) {
|
|||
db, err := gorm.Open(
|
||||
mysql.Open(str+""),
|
||||
&gorm.Config{
|
||||
//Logger: logger.Discard,
|
||||
Logger: logger.Discard,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
db = db.Debug()
|
||||
//db = db.Debug()
|
||||
return &Db{
|
||||
db: db,
|
||||
}, nil
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ type CsvExporter struct {
|
|||
mFetcher DataFetcher
|
||||
file FileAdapter
|
||||
count int
|
||||
last interface{}
|
||||
|
||||
wg *sync.WaitGroup
|
||||
}
|
||||
|
|
@ -49,8 +50,8 @@ func (ee *CsvExporter) Export(sql, pk string) error {
|
|||
if ee.count > 0 {
|
||||
//异步导出数据到csv文件中
|
||||
go ee.exportToCsv(data)
|
||||
} else {
|
||||
ee.wg.Done()
|
||||
last := data.Data[ee.count-1]
|
||||
ee.last = last[ee.getPkIndex(data.Title, pk)]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -72,8 +73,8 @@ func (ee *CsvExporter) exportToCsv(data *Data) {
|
|||
}
|
||||
}
|
||||
|
||||
func (ee *CsvExporter) Count() int {
|
||||
return ee.count
|
||||
func (ee *CsvExporter) Last() (int, interface{}) {
|
||||
return ee.count, ee.last
|
||||
}
|
||||
|
||||
func (ee *CsvExporter) getPkIndex(titles []string, pk string) int {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,115 @@
|
|||
package export
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"github.com/tealeg/xlsx/v3"
|
||||
"reflect"
|
||||
)
|
||||
|
||||
const Sheet_Name = "sheet_1"
|
||||
|
||||
type Excel struct {
|
||||
f *File
|
||||
count int //总数
|
||||
isNew bool
|
||||
titles []string
|
||||
file *xlsx.File
|
||||
sheet *xlsx.Sheet
|
||||
}
|
||||
|
||||
func NewExcel(fileName string, limit int, param map[string]string) *Excel {
|
||||
return &Excel{
|
||||
f: NewFile(fileName, limit, param),
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Excel) slice() {
|
||||
if e.f.slice() {
|
||||
e.reset()
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Excel) reset() {
|
||||
e.save()
|
||||
e.f.NextFile()
|
||||
e.Open()
|
||||
e.WriteTitle(nil)
|
||||
e.slice()
|
||||
}
|
||||
|
||||
func (e *Excel) Open() error {
|
||||
var err error
|
||||
if e.f.IsFileExist() {
|
||||
e.file, err = xlsx.OpenFile(e.f.FileName())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
e.isNew = true
|
||||
e.file = xlsx.NewFile()
|
||||
}
|
||||
|
||||
var ok bool
|
||||
e.sheet, ok = e.file.Sheet[Sheet_Name]
|
||||
if !ok {
|
||||
e.sheet, err = e.file.AddSheet(Sheet_Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
//需要处理偏移数据
|
||||
e.f.SetRow(e.sheet.MaxRow)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *Excel) save() error {
|
||||
return e.file.Save(e.f.FileName())
|
||||
}
|
||||
|
||||
func (e *Excel) WriteTitle(titles []string) error {
|
||||
if e.file == nil || e.sheet == nil {
|
||||
return errors.New("没有执行open方法")
|
||||
}
|
||||
|
||||
if titles != nil {
|
||||
e.titles = titles
|
||||
}
|
||||
|
||||
if e.titles != nil && e.isNew {
|
||||
e.Write(e.titles)
|
||||
e.isNew = false
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *Excel) Write(data interface{}) error {
|
||||
if e.f.slice() {
|
||||
e.reset()
|
||||
}
|
||||
row := e.sheet.AddRow()
|
||||
|
||||
v := reflect.ValueOf(data)
|
||||
if v.Kind() == reflect.Ptr {
|
||||
v = v.Elem()
|
||||
}
|
||||
if v.Kind() != reflect.Slice {
|
||||
return errors.New("数据无效，不是切片类型")
|
||||
}
|
||||
|
||||
switch val := data.(type) {
|
||||
case []string:
|
||||
row.WriteSlice(val, -1)
|
||||
case []interface{}:
|
||||
for _, i := range data.([]interface{}) {
|
||||
cell := row.AddCell()
|
||||
cell.SetValue(i)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *Excel) Close() error {
|
||||
return e.save()
|
||||
}
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
package export
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
)
|
||||
|
||||
type ExcelExporter struct {
|
||||
mFetcher DataFetcher
|
||||
file FileAdapter
|
||||
count int
|
||||
last interface{}
|
||||
}
|
||||
|
||||
func NewExcelExporter(fetcher DataFetcher, file FileAdapter) DataExporter {
|
||||
return &ExcelExporter{
|
||||
mFetcher: fetcher,
|
||||
file: file,
|
||||
}
|
||||
}
|
||||
|
||||
func (ee *ExcelExporter) Fetcher(fetcher DataFetcher) {
|
||||
ee.mFetcher = fetcher
|
||||
}
|
||||
|
||||
func (ee *ExcelExporter) File(file FileAdapter) {
|
||||
ee.file = file
|
||||
}
|
||||
|
||||
func (ee *ExcelExporter) Export(sql, pk string) error {
|
||||
data, err := ee.mFetcher.Fetch(sql)
|
||||
if err != nil {
|
||||
return fmt.Errorf("数据获取错误:%w", err)
|
||||
}
|
||||
ee.count = len(data.Data)
|
||||
//fmt.Printf("Excel Exporter.Excel, got %v data\n", len(data))
|
||||
//ee.file.Open()
|
||||
ee.file.WriteTitle(data.Title)
|
||||
var last []string
|
||||
for _, val := range data.Data {
|
||||
last = val
|
||||
ee.file.Write(last)
|
||||
}
|
||||
|
||||
if len(last) > 0 {
|
||||
ee.last = last[getPkIndex(data.Title, pk)]
|
||||
}
|
||||
//ee.file.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ee *ExcelExporter) Last() (int, interface{}) {
|
||||
return ee.count, ee.last
|
||||
}
|
||||
|
||||
func getPkIndex(titles []string, pk string) int {
|
||||
if pk == "" {
|
||||
log.Println("pk is not empty")
|
||||
return -1
|
||||
}
|
||||
for i, title := range titles {
|
||||
if title == pk {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
package export
|
||||
|
||||
import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestExcelExporter_Export(t *testing.T) {
|
||||
data := NewMysqlDataFetcher("aaa")
|
||||
pwd, _ := os.Getwd()
|
||||
file := NewExcel(pwd+"/aa-{begin}.xlsx", 5, map[string]string{"begin": "202301"})
|
||||
e := NewExcelExporter(data, file)
|
||||
file.Open()
|
||||
err := e.Export("aa", "字段1")
|
||||
file.Close()
|
||||
assert.Nil(t, err)
|
||||
|
||||
assert.FileExists(t, pwd+"/aa-202301_0.xlsx")
|
||||
assert.FileExists(t, pwd+"/aa-202301_1.xlsx")
|
||||
assert.NoFileExists(t, pwd+"/aa-202301_2.xlsx")
|
||||
|
||||
_ = os.Remove(pwd + "/aa-202301_0.xlsx")
|
||||
_ = os.Remove(pwd + "/aa-202301_1.xlsx")
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
package export
|
||||
|
||||
import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestExcel_Write(t *testing.T) {
|
||||
pwd, _ := os.Getwd()
|
||||
e := NewExcel(pwd+"/aa-{begin}.xlsx", 5, map[string]string{"begin": "202301"})
|
||||
|
||||
e.Open()
|
||||
|
||||
e.WriteTitle([]string{"姓名", "年龄"})
|
||||
data := make([]interface{}, 2)
|
||||
data[0] = "张三"
|
||||
for i := 0; i < 9; i++ {
|
||||
data[1] = 10 + i
|
||||
e.Write(data)
|
||||
}
|
||||
e.Close()
|
||||
|
||||
assert.FileExists(t, pwd+"/aa-202301_0.xlsx")
|
||||
assert.FileExists(t, pwd+"/aa-202301_1.xlsx")
|
||||
assert.NoFileExists(t, pwd+"/aa-202301_2.xlsx")
|
||||
|
||||
_ = os.Remove(pwd + "/aa-202301_0.xlsx")
|
||||
_ = os.Remove(pwd + "/aa-202301_1.xlsx")
|
||||
|
||||
}
|
||||
|
|
@ -5,7 +5,7 @@ type (
|
|||
Fetcher(fetcher DataFetcher)
|
||||
File(file FileAdapter)
|
||||
Export(sql, pk string) error
|
||||
Count() int
|
||||
Last() (int, interface{})
|
||||
}
|
||||
|
||||
Data struct {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,35 @@
|
|||
package export
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
type MysqlDataFetcher struct {
|
||||
Config string
|
||||
}
|
||||
|
||||
func (mf *MysqlDataFetcher) Fetch(sql string) (*Data, error) {
|
||||
rows := make([][]string, 0, 6)
|
||||
// 插入6个随机数组成的切片，模拟查询要返回的数据集
|
||||
rows = append(rows, row(), row(), row(), row(), row(), row())
|
||||
return &Data{
|
||||
Title: []string{"字段1", "字段2", "字段3", "字段4", "字段5"},
|
||||
Data: rows,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func NewMysqlDataFetcher(configStr string) DataFetcher {
|
||||
return &MysqlDataFetcher{
|
||||
Config: configStr,
|
||||
}
|
||||
}
|
||||
|
||||
func row() []string {
|
||||
strs := make([]string, 5)
|
||||
nums := rand.Perm(5)
|
||||
for i, num := range nums {
|
||||
strs[i] = strconv.Itoa(num)
|
||||
}
|
||||
return strs
|
||||
}
|
||||
|
|
@ -5,7 +5,6 @@ import (
|
|||
"excel_export/biz/db"
|
||||
"excel_export/biz/export"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
|
@ -37,6 +36,7 @@ func (e *Csv) Export(sysName, jobName string, begin, end time.Time, batch int) e
|
|||
return e.JobHandler(job, d, map[string]interface{}{
|
||||
"begin": begin,
|
||||
"end": end,
|
||||
"last": 0,
|
||||
}, batch)
|
||||
}
|
||||
|
||||
|
|
@ -55,55 +55,40 @@ func (e *Csv) JobHandler(job config.Job, d export.DataFetcher, params map[string
|
|||
func (e *Csv) TaskExport(d export.DataFetcher, t config.Task, params map[string]interface{}, batch int, fileName string) error {
|
||||
var i int
|
||||
var wg sync.WaitGroup
|
||||
var total int
|
||||
|
||||
beginTime := params["begin"].(time.Time)
|
||||
lastTime := params["end"].(time.Time)
|
||||
over := false
|
||||
for i = 0; i < 10000; i++ {
|
||||
endTime := beginTime.Add(2 * time.Hour)
|
||||
//结束时间大于最后时间
|
||||
if endTime.After(lastTime) {
|
||||
endTime = lastTime
|
||||
over = true
|
||||
}
|
||||
for i = 0; i < 1000; i++ {
|
||||
|
||||
f, err := e.getCsvFile(i)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
params["begin"] = beginTime
|
||||
params["end"] = endTime
|
||||
|
||||
sql := t.GetSql(params)
|
||||
|
||||
e := export.NewCsvExporter(d, f)
|
||||
e.(*export.CsvExporter).WaitGroup(&wg)
|
||||
wg.Add(1)
|
||||
e.Export(sql, t.PK)
|
||||
e.Export(sql+" limit "+strconv.Itoa(batch), t.PK)
|
||||
|
||||
count := e.Count()
|
||||
count, last := e.Last()
|
||||
fmt.Printf("已导出 %d 条数据\n", batch*i+count)
|
||||
|
||||
total = total + count
|
||||
if count == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if over {
|
||||
if count == 0 {
|
||||
i = i - 1
|
||||
}
|
||||
if count < batch {
|
||||
break
|
||||
}
|
||||
beginTime = endTime
|
||||
|
||||
params["last"] = last
|
||||
time.Sleep(time.Microsecond * 30)
|
||||
}
|
||||
wg.Wait()
|
||||
//fmt.Println("tempDir", e.dirTemp)
|
||||
if total > 0 { //查询到数据
|
||||
//合并csv文件，并删除 临时目录
|
||||
if err := e.mergeCsvToExcel(e.dirTemp, i, fileName); err != nil {
|
||||
log.Printf("合并csv文件异常：%s", err.Error())
|
||||
return err
|
||||
}
|
||||
}
|
||||
//todo 合并csv文件，并删除 临时目录
|
||||
err := e.mergeCsvToExcel(e.dirTemp, i, fileName)
|
||||
fmt.Println(err)
|
||||
|
||||
//重置临时路径
|
||||
e.dirTemp = ""
|
||||
|
||||
|
|
@ -138,6 +123,5 @@ func (e *Csv) mergeCsvToExcel(path string, max int, out string) error {
|
|||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
//return m.Clear()
|
||||
return m.Clear()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,83 @@
|
|||
package cmd
|
||||
|
||||
import (
|
||||
"excel_export/biz/config"
|
||||
"excel_export/biz/db"
|
||||
"excel_export/biz/export"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Excel struct {
|
||||
conf *config.Config
|
||||
}
|
||||
|
||||
func NewExcel(conf *config.Config) *Excel {
|
||||
return &Excel{
|
||||
conf: conf,
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Excel) Export(sysName, jobName string, begin, end time.Time, batch int) error {
|
||||
job, dbStr, err := config.GetJob(e.conf, sysName, jobName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
d, err := db.NewDb(dbStr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return e.JobHandler(job, d, map[string]interface{}{
|
||||
"begin": begin,
|
||||
"end": end,
|
||||
"last": 0,
|
||||
}, batch)
|
||||
}
|
||||
|
||||
func (e *Excel) JobHandler(job config.Job, d export.DataFetcher, params map[string]interface{}, batch int) error {
|
||||
|
||||
f := export.NewExcel(job.File, job.Size, map[string]string{
|
||||
"begin": params["begin"].(time.Time).Format("20060102"),
|
||||
"end": params["end"].(time.Time).Format("20060102"),
|
||||
})
|
||||
|
||||
for i, task := range job.Tasks {
|
||||
fmt.Printf("执行导出任务：%d\n", i+1)
|
||||
if err := e.TaskExport(d, task, params, f, batch); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *Excel) TaskExport(d export.DataFetcher, t config.Task, params map[string]interface{}, f export.FileAdapter, batch int) error {
|
||||
//todo 最多分1000个批次进行处理
|
||||
|
||||
f.Open()
|
||||
defer f.Close()
|
||||
for i := 0; i < 1000; i++ {
|
||||
sql := t.GetSql(params)
|
||||
|
||||
e := export.NewExcelExporter(d, f)
|
||||
|
||||
e.Export(sql+" limit "+strconv.Itoa(batch), t.PK)
|
||||
|
||||
count, last := e.Last()
|
||||
fmt.Printf("已导出 %d 条数据\n", batch*i+count)
|
||||
|
||||
if count == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if count < batch {
|
||||
break
|
||||
}
|
||||
|
||||
params["last"] = last
|
||||
time.Sleep(time.Microsecond * 30)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -2,8 +2,8 @@ package cmd
|
|||
|
||||
import (
|
||||
"excel_export/biz/config"
|
||||
"fmt"
|
||||
"github.com/spf13/cobra"
|
||||
"log"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
|
@ -44,17 +44,13 @@ func exportAllJobRun(cmd *cobra.Command, args []string) {
|
|||
CmdError(cmd, "无效的参数：%s", err.Error())
|
||||
}
|
||||
|
||||
allBegin := time.Now()
|
||||
|
||||
for _, job := range sys.Jobs {
|
||||
log.Printf("执行【%s】【%s】导出\n", sName, job.Name)
|
||||
fmt.Printf("[%s]执行【%s】【%s】导出\n", time.Now().Format("2006-01-02 15:04:05"), sName, job.Name)
|
||||
b := time.Now()
|
||||
ee := NewCsv(config.DefaultConfig)
|
||||
if err := ee.Export(sName, job.Name, begin, end, batch); err != nil {
|
||||
CmdError(cmd, "【%s】【%s】导出错误：%s", sName, job.Name, err.Error())
|
||||
}
|
||||
log.Println("导出耗时：" + time.Now().Sub(b).String())
|
||||
ee.Export(sName, job.Name, begin, end, batch)
|
||||
e := time.Now()
|
||||
fmt.Println("导出耗时：" + e.Sub(b).String())
|
||||
}
|
||||
|
||||
log.Println("总耗时：" + time.Now().Sub(allBegin).String())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@ package cmd
|
|||
|
||||
import (
|
||||
"excel_export/biz/config"
|
||||
"fmt"
|
||||
"github.com/spf13/cobra"
|
||||
"log"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
|
@ -35,18 +35,15 @@ func exportAllRun(cmd *cobra.Command, args []string) {
|
|||
if err != nil {
|
||||
CmdError(cmd, "无效的参数：%s", err.Error())
|
||||
}
|
||||
allBegin := time.Now()
|
||||
for _, sys := range c.Systems {
|
||||
for _, job := range sys.Jobs {
|
||||
log.Printf("执行【%s】【%s】导出\n", sys.Name, job.Name)
|
||||
fmt.Printf("[%s]执行【%s】【%s】导出\n", time.Now().Format("2006-01-02 15:04:05"), sys.Name, job.Name)
|
||||
b := time.Now()
|
||||
ee := NewCsv(config.DefaultConfig)
|
||||
if err := ee.Export(sys.Name, job.Name, begin, end, batch); err != nil {
|
||||
CmdError(cmd, "【%s】【%s】导出错误：%s", sys.Name, job.Name, err.Error())
|
||||
}
|
||||
log.Println("导出耗时：" + time.Now().Sub(b).String())
|
||||
ee.Export(sys.Name, job.Name, begin, end, batch)
|
||||
e := time.Now()
|
||||
fmt.Println("导出耗时：" + e.Sub(b).String())
|
||||
}
|
||||
}
|
||||
log.Println("总耗时：" + time.Now().Sub(allBegin).String())
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@ package cmd
|
|||
|
||||
import (
|
||||
"excel_export/biz/config"
|
||||
"fmt"
|
||||
"github.com/spf13/cobra"
|
||||
"log"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
|
@ -43,7 +43,7 @@ func exportRun(cmd *cobra.Command, args []string) {
|
|||
CmdError(cmd, "%s", err.Error())
|
||||
}
|
||||
|
||||
log.Printf("执行【%s】【%s】导出\n", sName, jName)
|
||||
fmt.Printf("[%s]执行【%s】【%s】导出\n", time.Now().Format("2006-01-02 15:04:05"), sName, jName)
|
||||
|
||||
begin := MustFlagsDateTime(cmd, "begin")
|
||||
end := MustFlagsDateTime(cmd, "end")
|
||||
|
|
@ -55,8 +55,7 @@ func exportRun(cmd *cobra.Command, args []string) {
|
|||
}
|
||||
b := time.Now()
|
||||
ee := NewCsv(config.DefaultConfig)
|
||||
if err := ee.Export(sName, jName, begin, end, batch); err != nil {
|
||||
CmdError(cmd, "【%s】【%s】导出错误：%s", sName, jName, err.Error())
|
||||
}
|
||||
log.Println("耗时：" + time.Now().Sub(b).String())
|
||||
ee.Export(sName, jName, begin, end, batch)
|
||||
e := time.Now()
|
||||
fmt.Println("耗时：" + e.Sub(b).String())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
package cmd
|
||||
|
||||
import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
|
|
@ -19,10 +17,5 @@ func TestMerge_Write(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestMerge_Save(t *testing.T) {
|
||||
m := NewMerge(
|
||||
Reader{Path: os.TempDir() + "/3299772411", Index: 102},
|
||||
Writer{File: "sss.xlsx", Limit: 1000000},
|
||||
)
|
||||
err := m.Merge()
|
||||
assert.NoError(t, err)
|
||||
// m := NewMerge(os.TempDir()+"/3299772411",500000)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ func ParseFlagsDateTime(set *flag.FlagSet, key string) (*time.Time, error) {
|
|||
if err != nil {
|
||||
return nil, fmt.Errorf("获取参数异常：%w", err)
|
||||
}
|
||||
beginTime, err := time.ParseInLocation("2006-01-02 15:04:05", val, time.Local)
|
||||
beginTime, err := time.Parse("2006-01-02 15:04:05", val)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("不是有效的时间格式：%w", err)
|
||||
}
|
||||
|
|
|
|||
17
cmd/main.go
17
cmd/main.go
|
|
@ -2,17 +2,26 @@ package main
|
|||
|
||||
import (
|
||||
"excel_export/biz/config"
|
||||
"excel_export/biz/util"
|
||||
"excel_export/cmd/cmd"
|
||||
"github.com/tealeg/xlsx/v3"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
var Config *config.Config
|
||||
|
||||
func main() {
|
||||
//p := util.NewProf()
|
||||
//defer func() {
|
||||
// p.Close()
|
||||
//}()
|
||||
p := util.NewProf()
|
||||
defer func() {
|
||||
p.Close()
|
||||
}()
|
||||
|
||||
//设置默认格式
|
||||
xlsx.DefaultDateTimeOptions = xlsx.DateTimeOptions{
|
||||
Location: time.UTC,
|
||||
ExcelTimeFormat: "yyyy-m-d h:mm:ss",
|
||||
}
|
||||
|
||||
path, _ := os.Getwd()
|
||||
Config = config.LoadConfig(path + "/config")
|
||||
|
|
|
|||
|
|
@ -223,7 +223,7 @@ system:
|
|||
size: 1000000
|
||||
- name: "余额信息"
|
||||
tasks:
|
||||
- pk: "分销商编号"
|
||||
- pk:
|
||||
sql: >-
|
||||
SELECT
|
||||
b.reseller_id AS `分销商编号`,
|
||||
|
|
@ -237,7 +237,7 @@ system:
|
|||
LEFT JOIN reseller r ON r.id = b.reseller_id
|
||||
left join reseller_balance_day rbd on rbd.reseller_id = b.reseller_id and rbd.`day` = DATE_FORMAT(date_add('{end}', interval -1 day),"%Y-%m-%d")
|
||||
elt: "r.`status` = 1"
|
||||
timestamp: false
|
||||
timestamp:
|
||||
order: ""
|
||||
file: "直连天下-余额信息-{end}.xlsx"
|
||||
size: 1000000
|
||||
|
|
@ -510,113 +510,6 @@ system:
|
|||
order: "o.create_time,o.order_number"
|
||||
file: "雅兰芳-下游订单-{begin}-{end}-{task}.xlsx"
|
||||
size: 1000000
|
||||
- name: "余额信息"
|
||||
tasks:
|
||||
- pk: "分销商编号"
|
||||
sql: >-
|
||||
SELECT
|
||||
b.reseller_id AS `分销商编号`,
|
||||
r.`name` AS `简称`,
|
||||
r.full_name AS `全称`,
|
||||
balance AS `余额`,
|
||||
rbd.day_balance as `昨日余额`,
|
||||
extension AS `授信`
|
||||
FROM
|
||||
reseller_balance b
|
||||
LEFT JOIN reseller r ON r.id = b.reseller_id
|
||||
left join reseller_balance_day rbd on rbd.reseller_id = b.reseller_id and rbd.`day` = DATE_FORMAT(date_add('{end}', interval -1 day),"%Y-%m-%d")
|
||||
elt: "r.`status` = 1"
|
||||
timestamp: false
|
||||
order: ""
|
||||
file: "雅兰芳-余额信息-{end}.xlsx"
|
||||
size: 1000000
|
||||
- name: "批量充值"
|
||||
tasks:
|
||||
- pk: "流水号"
|
||||
sql: >-
|
||||
SELECT
|
||||
bd.customer as `分销商`,
|
||||
r.`name`,
|
||||
bd.price as `商品价格`,
|
||||
od.ours_product_id as `我们的商品id`,
|
||||
op.NAME as `商品名称`,
|
||||
p.name as `接口平台`,
|
||||
CASE od.`status`
|
||||
WHEN -6 THEN '手动失败'
|
||||
WHEN -5 THEN '手动重试'
|
||||
WHEN -3 THEN '卡单'
|
||||
WHEN -2 THEN '失败重试'
|
||||
WHEN -1 THEN '充值失败'
|
||||
WHEN 0 THEN '待充值'
|
||||
WHEN 1 THEN '充值成功'
|
||||
WHEN 2 THEN '充值中'
|
||||
else od.`status`
|
||||
END
|
||||
AS 充值状态,
|
||||
pp.`code` as `接口平台产品编码`,
|
||||
od.order_order_number as `系统订单号`,
|
||||
od.serial_number as `流水号`,
|
||||
od.terminal_account as `充值账号`,
|
||||
od.trade_price as `成交价格`,
|
||||
od.platform_price as `接口平台价格`,
|
||||
od.create_time as `创建时间`,
|
||||
od.execute_time as `执行时间`,
|
||||
bd.ding_talk_sn as `钉钉审批序号`,
|
||||
bd.remark as `批量备注`
|
||||
FROM
|
||||
order_direct od -- FORCE INDEX(idx_direct_create_time)
|
||||
left JOIN ours_product op ON od.ours_product_id = op.id
|
||||
left join platform_product pp on pp.id = od.platform_product_id
|
||||
left join platform p on pp.platform_id = p.id
|
||||
RIGHT JOIN batch_direct bd on bd.batch_id = od.order_order_number
|
||||
LEFT JOIN reseller r on r.id = bd.reseller_id
|
||||
elt: "bd.create_time BETWEEN {begin} AND {end} and od.serial_number > '{last}'"
|
||||
timestamp: true
|
||||
order: "od.create_time,od.serial_number"
|
||||
- pk: "流水号"
|
||||
sql: >-
|
||||
SELECT
|
||||
bd.customer as `分销商`,
|
||||
r.`name`,
|
||||
bd.price as `商品价格`,
|
||||
od.ours_product_id as `我们的商品id`,
|
||||
op.NAME as `商品名称`,
|
||||
p.name as `接口平台`,
|
||||
CASE od.`status`
|
||||
WHEN -6 THEN '手动失败'
|
||||
WHEN -5 THEN '手动重试'
|
||||
WHEN -3 THEN '卡单'
|
||||
WHEN -2 THEN '失败重试'
|
||||
WHEN -1 THEN '充值失败'
|
||||
WHEN 0 THEN '待充值'
|
||||
WHEN 1 THEN '充值成功'
|
||||
WHEN 2 THEN '充值中'
|
||||
else od.`status`
|
||||
END
|
||||
AS 充值状态,
|
||||
pp.`code` as `接口平台产品编码`,
|
||||
od.order_order_number as `系统订单号`,
|
||||
od.serial_number as `流水号`,
|
||||
od.terminal_account as `充值账号`,
|
||||
od.trade_price as `成交价格`,
|
||||
od.platform_price as `接口平台价格`,
|
||||
od.create_time as `创建时间`,
|
||||
od.execute_time as `执行时间`,
|
||||
bd.ding_talk_sn as `钉钉审批序号`,
|
||||
bd.remark as `批量备注`
|
||||
|
||||
FROM
|
||||
history_order_direct od
|
||||
left JOIN ours_product op ON od.ours_product_id = op.id
|
||||
left join platform_product pp on pp.id = od.platform_product_id
|
||||
left join platform p on pp.platform_id = p.id
|
||||
RIGHT JOIN batch_direct bd on bd.batch_id = od.order_order_number
|
||||
LEFT JOIN reseller r on r.id = bd.reseller_id
|
||||
elt: "bd.create_time BETWEEN {begin} AND {end} and od.serial_number > '{last}'"
|
||||
timestamp: true
|
||||
order: "od.create_time,od.serial_number"
|
||||
file: "雅兰芳-批量充值-{begin}-{end}-{task}.xlsx"
|
||||
size: 1000000
|
||||
|
||||
- name: "创意择优"
|
||||
db: "root:lhb767@tcp(120.79.35.82:3307)/new_sys?charset=utf8mb4&parseTime=True"
|
||||
|
|
@ -750,110 +643,3 @@ system:
|
|||
order: "o.create_time,o.order_number"
|
||||
file: "创意择优-下游订单-{begin}-{end}-{task}.xlsx"
|
||||
size: 1000000
|
||||
- name: "余额信息"
|
||||
tasks:
|
||||
- pk: "分销商编号"
|
||||
sql: >-
|
||||
SELECT
|
||||
b.reseller_id AS `分销商编号`,
|
||||
r.`name` AS `简称`,
|
||||
r.full_name AS `全称`,
|
||||
balance AS `余额`,
|
||||
rbd.day_balance as `昨日余额`,
|
||||
extension AS `授信`
|
||||
FROM
|
||||
reseller_balance b
|
||||
LEFT JOIN reseller r ON r.id = b.reseller_id
|
||||
left join reseller_balance_day rbd on rbd.reseller_id = b.reseller_id and rbd.`day` = DATE_FORMAT(date_add('{end}', interval -1 day),"%Y-%m-%d")
|
||||
elt: "r.`status` = 1"
|
||||
timestamp: false
|
||||
order: ""
|
||||
file: "创意择优-余额信息-{end}.xlsx"
|
||||
size: 1000000
|
||||
- name: "批量充值"
|
||||
tasks:
|
||||
- pk: "流水号"
|
||||
sql: >-
|
||||
SELECT
|
||||
bd.customer as `分销商`,
|
||||
r.`name`,
|
||||
bd.price as `商品价格`,
|
||||
od.ours_product_id as `我们的商品id`,
|
||||
op.NAME as `商品名称`,
|
||||
p.name as `接口平台`,
|
||||
CASE od.`status`
|
||||
WHEN -6 THEN '手动失败'
|
||||
WHEN -5 THEN '手动重试'
|
||||
WHEN -3 THEN '卡单'
|
||||
WHEN -2 THEN '失败重试'
|
||||
WHEN -1 THEN '充值失败'
|
||||
WHEN 0 THEN '待充值'
|
||||
WHEN 1 THEN '充值成功'
|
||||
WHEN 2 THEN '充值中'
|
||||
else od.`status`
|
||||
END
|
||||
AS 充值状态,
|
||||
pp.`code` as `接口平台产品编码`,
|
||||
od.order_order_number as `系统订单号`,
|
||||
od.serial_number as `流水号`,
|
||||
od.terminal_account as `充值账号`,
|
||||
od.trade_price as `成交价格`,
|
||||
od.platform_price as `接口平台价格`,
|
||||
od.create_time as `创建时间`,
|
||||
od.execute_time as `执行时间`,
|
||||
bd.ding_talk_sn as `钉钉审批序号`,
|
||||
bd.remark as `批量备注`
|
||||
FROM
|
||||
order_direct od -- FORCE INDEX(idx_direct_create_time)
|
||||
left JOIN ours_product op ON od.ours_product_id = op.id
|
||||
left join platform_product pp on pp.id = od.platform_product_id
|
||||
left join platform p on pp.platform_id = p.id
|
||||
RIGHT JOIN batch_direct bd on bd.batch_id = od.order_order_number
|
||||
LEFT JOIN reseller r on r.id = bd.reseller_id
|
||||
elt: "bd.create_time BETWEEN {begin} AND {end} and od.serial_number > '{last}'"
|
||||
timestamp: true
|
||||
order: "od.create_time,od.serial_number"
|
||||
- pk: "流水号"
|
||||
sql: >-
|
||||
SELECT
|
||||
bd.customer as `分销商`,
|
||||
r.`name`,
|
||||
bd.price as `商品价格`,
|
||||
od.ours_product_id as `我们的商品id`,
|
||||
op.NAME as `商品名称`,
|
||||
p.name as `接口平台`,
|
||||
CASE od.`status`
|
||||
WHEN -6 THEN '手动失败'
|
||||
WHEN -5 THEN '手动重试'
|
||||
WHEN -3 THEN '卡单'
|
||||
WHEN -2 THEN '失败重试'
|
||||
WHEN -1 THEN '充值失败'
|
||||
WHEN 0 THEN '待充值'
|
||||
WHEN 1 THEN '充值成功'
|
||||
WHEN 2 THEN '充值中'
|
||||
else od.`status`
|
||||
END
|
||||
AS 充值状态,
|
||||
pp.`code` as `接口平台产品编码`,
|
||||
od.order_order_number as `系统订单号`,
|
||||
od.serial_number as `流水号`,
|
||||
od.terminal_account as `充值账号`,
|
||||
od.trade_price as `成交价格`,
|
||||
od.platform_price as `接口平台价格`,
|
||||
od.create_time as `创建时间`,
|
||||
od.execute_time as `执行时间`,
|
||||
bd.ding_talk_sn as `钉钉审批序号`,
|
||||
bd.remark as `批量备注`
|
||||
|
||||
FROM
|
||||
history_order_direct od
|
||||
left JOIN ours_product op ON od.ours_product_id = op.id
|
||||
left join platform_product pp on pp.id = od.platform_product_id
|
||||
left join platform p on pp.platform_id = p.id
|
||||
RIGHT JOIN batch_direct bd on bd.batch_id = od.order_order_number
|
||||
LEFT JOIN reseller r on r.id = bd.reseller_id
|
||||
elt: "bd.create_time BETWEEN {begin} AND {end} and od.serial_number > '{last}'"
|
||||
timestamp: true
|
||||
order: "od.create_time,od.serial_number"
|
||||
file: "创意择优-批量充值-{begin}-{end}-{task}.xlsx"
|
||||
size: 1000000
|
||||
Loading…
Reference in New Issue