128 lines
2.4 KiB
Go
128 lines
2.4 KiB
Go
package cmd
|
||
|
||
import (
|
||
"excel_export/biz/config"
|
||
"excel_export/biz/db"
|
||
"excel_export/biz/export"
|
||
"fmt"
|
||
"os"
|
||
"strconv"
|
||
"sync"
|
||
"time"
|
||
)
|
||
|
||
type Csv struct {
|
||
conf *config.Config
|
||
dirTemp string
|
||
}
|
||
|
||
func NewCsv(conf *config.Config) *Csv {
|
||
return &Csv{
|
||
conf: conf,
|
||
}
|
||
}
|
||
|
||
func (e *Csv) Export(sysName, jobName string, begin, end time.Time, batch int) error {
|
||
job, dbStr, err := config.GetJob(e.conf, sysName, jobName)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
d, err := db.NewDb(dbStr)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
return e.JobHandler(job, d, map[string]interface{}{
|
||
"begin": begin,
|
||
"end": end,
|
||
"last": 0,
|
||
}, batch)
|
||
}
|
||
|
||
func (e *Csv) JobHandler(job config.Job, d export.DataFetcher, params map[string]interface{}, batch int) error {
|
||
|
||
for i, task := range job.Tasks {
|
||
fmt.Printf("执行导出任务：%d\n", i+1)
|
||
params["task"] = i
|
||
if err := e.TaskExport(d, task, params, batch, job.GetFileName(params)); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (e *Csv) TaskExport(d export.DataFetcher, t config.Task, params map[string]interface{}, batch int, fileName string) error {
|
||
var i int
|
||
var wg sync.WaitGroup
|
||
for i = 0; i < 1000; i++ {
|
||
|
||
f, err := e.getCsvFile(i)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
sql := t.GetSql(params)
|
||
|
||
e := export.NewCsvExporter(d, f)
|
||
e.(*export.CsvExporter).WaitGroup(&wg)
|
||
wg.Add(1)
|
||
e.Export(sql+" limit "+strconv.Itoa(batch), t.PK)
|
||
|
||
count, last := e.Last()
|
||
fmt.Printf("已导出 %d 条数据\n", batch*i+count)
|
||
|
||
if count == 0 {
|
||
return nil
|
||
}
|
||
|
||
if count < batch {
|
||
break
|
||
}
|
||
|
||
params["last"] = last
|
||
time.Sleep(time.Microsecond * 30)
|
||
}
|
||
wg.Wait()
|
||
//fmt.Println("tempDir", e.dirTemp)
|
||
//todo 合并csv文件，并删除 临时目录
|
||
err := e.mergeCsvToExcel(e.dirTemp, i, fileName)
|
||
fmt.Println(err)
|
||
|
||
//重置临时路径
|
||
e.dirTemp = ""
|
||
|
||
return nil
|
||
}
|
||
|
||
func (e *Csv) getCsvFile(index int) (*export.Csv, error) {
|
||
|
||
if e.dirTemp == "" {
|
||
path, err := os.MkdirTemp(os.TempDir(), "")
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
e.dirTemp = path
|
||
}
|
||
|
||
filename := e.dirTemp + "/data_{index}.csv"
|
||
|
||
f := export.NewCsv(filename, map[string]string{
|
||
"index": strconv.Itoa(index),
|
||
})
|
||
|
||
return f, nil
|
||
}
|
||
|
||
func (e *Csv) mergeCsvToExcel(path string, max int, out string) error {
|
||
m := NewMerge(
|
||
Reader{Path: path, Index: max},
|
||
Writer{File: out, Limit: 1000000},
|
||
)
|
||
if err := m.Merge(); err != nil {
|
||
return err
|
||
}
|
||
|
||
return m.Clear()
|
||
}
|