package cmd import ( "excel_export/biz/config" "excel_export/biz/db" "excel_export/biz/export" "fmt" "os" "strconv" "sync" "time" ) type Csv struct { conf *config.Config dirTemp string } func NewCsv(conf *config.Config) *Csv { return &Csv{ conf: conf, } } func (e *Csv) Export(sysName, jobName string, begin, end time.Time, batch int) error { job, dbStr, err := config.GetJob(e.conf, sysName, jobName) if err != nil { return err } d, err := db.NewDb(dbStr) if err != nil { return err } return e.JobHandler(job, d, map[string]interface{}{ "begin": begin, "end": end, "last": 0, }, batch) } func (e *Csv) JobHandler(job config.Job, d export.DataFetcher, params map[string]interface{}, batch int) error { for i, task := range job.Tasks { fmt.Printf("执行导出任务:%d\n", i+1) params["task"] = i if err := e.TaskExport(d, task, params, batch, job.GetFileName(params)); err != nil { return err } } return nil } func (e *Csv) TaskExport(d export.DataFetcher, t config.Task, params map[string]interface{}, batch int, fileName string) error { var i int var wg sync.WaitGroup for i = 0; i < 1000; i++ { f, err := e.getCsvFile(i) if err != nil { return err } sql := t.GetSql(params) e := export.NewCsvExporter(d, f) e.(*export.CsvExporter).WaitGroup(&wg) wg.Add(1) e.Export(sql+" limit "+strconv.Itoa(batch), t.PK) count, last := e.Last() fmt.Printf("已导出 %d 条数据\n", batch*i+count) if count == 0 { return nil } if count < batch { break } params["last"] = last time.Sleep(time.Microsecond * 30) } wg.Wait() //fmt.Println("tempDir", e.dirTemp) //todo 合并csv文件,并删除 临时目录 err := e.mergeCsvToExcel(e.dirTemp, i, fileName) fmt.Println(err) //重置临时路径 e.dirTemp = "" return nil } func (e *Csv) getCsvFile(index int) (*export.Csv, error) { if e.dirTemp == "" { path, err := os.MkdirTemp(os.TempDir(), "") if err != nil { return nil, err } e.dirTemp = path } filename := e.dirTemp + "/data_{index}.csv" f := export.NewCsv(filename, map[string]string{ "index": strconv.Itoa(index), }) return f, nil } func (e *Csv) mergeCsvToExcel(path string, max int, out string) error { m := NewMerge( Reader{Path: path, Index: max}, Writer{File: out, Limit: 1000000}, ) if err := m.Merge(); err != nil { return err } return m.Clear() }