From bcc66bad0a3d561d8ae9fe339118416cdf44bbde Mon Sep 17 00:00:00 2001 From: "Mr.Li" Date: Fri, 5 May 2023 22:05:00 +0800 Subject: [PATCH] =?UTF-8?q?fix:=E8=B0=83=E6=95=B4=E5=AF=BC=E5=87=BA?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E6=8B=86=E5=88=86=E6=A8=A1=E5=BC=8F=EF=BC=8C?= =?UTF-8?q?=E6=8C=89=E7=85=A7=E6=97=B6=E9=97=B4=E8=BF=9B=E8=A1=8C=E5=88=86?= =?UTF-8?q?=E7=89=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- biz/db/db.go | 5 ++-- biz/export/csv_exporter.go | 9 ++++---- biz/export/export.go | 2 +- cmd/cmd/csv.go | 47 ++++++++++++++++++++++++-------------- cmd/cmd/export-all-job.go | 4 ++++ cmd/cmd/export-all.go | 2 ++ cmd/cmd/merge_test.go | 9 +++++++- 7 files changed, 51 insertions(+), 27 deletions(-) diff --git a/biz/db/db.go b/biz/db/db.go index 5a3313c..ec414fe 100644 --- a/biz/db/db.go +++ b/biz/db/db.go @@ -7,7 +7,6 @@ import ( "fmt" "gorm.io/driver/mysql" "gorm.io/gorm" - "gorm.io/gorm/logger" "runtime/trace" "strconv" "time" @@ -23,13 +22,13 @@ func NewDb(str string) (*Db, error) { db, err := gorm.Open( mysql.Open(str+""), &gorm.Config{ - Logger: logger.Discard, + //Logger: logger.Discard, }, ) if err != nil { return nil, err } - //db = db.Debug() + db = db.Debug() return &Db{ db: db, }, nil diff --git a/biz/export/csv_exporter.go b/biz/export/csv_exporter.go index 03745d7..2c683fc 100644 --- a/biz/export/csv_exporter.go +++ b/biz/export/csv_exporter.go @@ -11,7 +11,6 @@ type CsvExporter struct { mFetcher DataFetcher file FileAdapter count int - last interface{} wg *sync.WaitGroup } @@ -50,8 +49,8 @@ func (ee *CsvExporter) Export(sql, pk string) error { if ee.count > 0 { //异步导出数据到csv文件中 go ee.exportToCsv(data) - last := data.Data[ee.count-1] - ee.last = last[ee.getPkIndex(data.Title, pk)] + } else { + ee.wg.Done() } return nil } @@ -73,8 +72,8 @@ func (ee *CsvExporter) exportToCsv(data *Data) { } } -func (ee *CsvExporter) Last() (int, interface{}) { - return ee.count, ee.last +func (ee *CsvExporter) Count() int { + return ee.count } func (ee *CsvExporter) getPkIndex(titles []string, pk string) int { diff --git a/biz/export/export.go b/biz/export/export.go index ee971d7..2ccb730 100644 --- a/biz/export/export.go +++ b/biz/export/export.go @@ -5,7 +5,7 @@ type ( Fetcher(fetcher DataFetcher) File(file FileAdapter) Export(sql, pk string) error - Last() (int, interface{}) + Count() int } Data struct { diff --git a/cmd/cmd/csv.go b/cmd/cmd/csv.go index 07affe6..c6209a9 100644 --- a/cmd/cmd/csv.go +++ b/cmd/cmd/csv.go @@ -37,7 +37,6 @@ func (e *Csv) Export(sysName, jobName string, begin, end time.Time, batch int) e return e.JobHandler(job, d, map[string]interface{}{ "begin": begin, "end": end, - "last": 0, }, batch) } @@ -56,42 +55,55 @@ func (e *Csv) JobHandler(job config.Job, d export.DataFetcher, params map[string func (e *Csv) TaskExport(d export.DataFetcher, t config.Task, params map[string]interface{}, batch int, fileName string) error { var i int var wg sync.WaitGroup - for i = 0; i < 1000; i++ { + var total int + + beginTime := params["begin"].(time.Time) + lastTime := params["end"].(time.Time) + over := false + for i = 0; i < 10000; i++ { + endTime := beginTime.Add(2 * time.Hour) + //结束时间大于最后时间 + if endTime.After(lastTime) { + endTime = lastTime + over = true + } f, err := e.getCsvFile(i) if err != nil { return err } - + params["begin"] = beginTime + params["end"] = endTime sql := t.GetSql(params) e := export.NewCsvExporter(d, f) e.(*export.CsvExporter).WaitGroup(&wg) wg.Add(1) - e.Export(sql+" limit "+strconv.Itoa(batch), t.PK) + e.Export(sql, t.PK) - count, last := e.Last() + count := e.Count() fmt.Printf("已导出 %d 条数据\n", batch*i+count) - if count == 0 { - return nil - } + total = total + count - if count < batch { + if over { + if count == 0 { + i = i - 1 + } break } - - params["last"] = last + beginTime = endTime time.Sleep(time.Microsecond * 30) } wg.Wait() //fmt.Println("tempDir", e.dirTemp) - //todo 合并csv文件，并删除 临时目录 - if err := e.mergeCsvToExcel(e.dirTemp, i, fileName); err != nil { - log.Printf("合并csv文件异常：%s", err.Error()) - return err + if total > 0 { //查询到数据 + //合并csv文件，并删除 临时目录 + if err := e.mergeCsvToExcel(e.dirTemp, i, fileName); err != nil { + log.Printf("合并csv文件异常：%s", err.Error()) + return err + } } - //重置临时路径 e.dirTemp = "" @@ -126,5 +138,6 @@ func (e *Csv) mergeCsvToExcel(path string, max int, out string) error { return err } - return m.Clear() + return nil + //return m.Clear() } diff --git a/cmd/cmd/export-all-job.go b/cmd/cmd/export-all-job.go index fd4598a..60c2e41 100644 --- a/cmd/cmd/export-all-job.go +++ b/cmd/cmd/export-all-job.go @@ -44,6 +44,8 @@ func exportAllJobRun(cmd *cobra.Command, args []string) { CmdError(cmd, "无效的参数：%s", err.Error()) } + allBegin := time.Now() + for _, job := range sys.Jobs { log.Printf("执行【%s】【%s】导出\n", sName, job.Name) b := time.Now() @@ -53,4 +55,6 @@ func exportAllJobRun(cmd *cobra.Command, args []string) { } log.Println("导出耗时：" + time.Now().Sub(b).String()) } + + log.Println("总耗时：" + time.Now().Sub(allBegin).String()) } diff --git a/cmd/cmd/export-all.go b/cmd/cmd/export-all.go index 2dcf317..e6c5f1a 100644 --- a/cmd/cmd/export-all.go +++ b/cmd/cmd/export-all.go @@ -35,6 +35,7 @@ func exportAllRun(cmd *cobra.Command, args []string) { if err != nil { CmdError(cmd, "无效的参数：%s", err.Error()) } + allBegin := time.Now() for _, sys := range c.Systems { for _, job := range sys.Jobs { log.Printf("执行【%s】【%s】导出\n", sys.Name, job.Name) @@ -46,5 +47,6 @@ func exportAllRun(cmd *cobra.Command, args []string) { log.Println("导出耗时：" + time.Now().Sub(b).String()) } } + log.Println("总耗时：" + time.Now().Sub(allBegin).String()) } diff --git a/cmd/cmd/merge_test.go b/cmd/cmd/merge_test.go index 2cb4798..69c6f6b 100644 --- a/cmd/cmd/merge_test.go +++ b/cmd/cmd/merge_test.go @@ -1,6 +1,8 @@ package cmd import ( + "github.com/stretchr/testify/assert" + "os" "testing" ) @@ -17,5 +19,10 @@ func TestMerge_Write(t *testing.T) { } func TestMerge_Save(t *testing.T) { - // m := NewMerge(os.TempDir()+"/3299772411",500000) + m := NewMerge( + Reader{Path: os.TempDir() + "/3299772411", Index: 102}, + Writer{File: "sss.xlsx", Limit: 1000000}, + ) + err := m.Merge() + assert.NoError(t, err) }