diff --git a/biz/export/csv_exporter.go b/biz/export/csv_exporter.go index 2104515..8041647 100644 --- a/biz/export/csv_exporter.go +++ b/biz/export/csv_exporter.go @@ -1,57 +1,68 @@ package export -import "fmt" +import ( + "fmt" + "sync" +) -type CSVExporter struct { +type CsvExporter struct { mFetcher DataFetcher file FileAdapter count int last interface{} + + wg *sync.WaitGroup } -func NewCVSExporter(fetcher DataFetcher, file FileAdapter) DataExporter { - return &CSVExporter{ +func NewCsvExporter(fetcher DataFetcher, file FileAdapter) DataExporter { + return &CsvExporter{ mFetcher: fetcher, file: file, } } -func (ee *CSVExporter) Fetcher(fetcher DataFetcher) { +func (ee *CsvExporter) Fetcher(fetcher DataFetcher) { ee.mFetcher = fetcher } -func (ee *CSVExporter) File(file FileAdapter) { +func (ee *CsvExporter) File(file FileAdapter) { ee.file = file } -func (ee *CSVExporter) Export(sql, pk string) error { +func (ee *CsvExporter) WaitGroup(wg *sync.WaitGroup) { + ee.wg = wg +} + +func (ee *CsvExporter) Export(sql, pk string) error { data, err := ee.mFetcher.Fetch(sql) if err != nil { return fmt.Errorf("数据获取错误:%w", err) } ee.count = len(data.Data) //fmt.Printf("Excel Exporter.Excel, got %v data\n", len(data)) - //ee.file.Open() - ee.file.WriteTitle(data.Title) - var last interface{} - for _, val := range data.Data { - last = val - ee.file.Write(last) - } + go func() { + ee.file.Open() + ee.file.WriteTitle(data.Title) - if row, ok := last.([]interface{}); ok { - ee.last = row[ee.getPkIndex(data.Title, pk)] - } + for _, val := range data.Data { + row := val + ee.file.Write(row) + } + ee.file.Close() + ee.wg.Done() + }() + + last := data.Data[ee.count-1] + ee.last = last[ee.getPkIndex(data.Title, pk)] - //ee.file.Close() return nil } -func (ee *CSVExporter) Last() (int, interface{}) { +func (ee *CsvExporter) Last() (int, interface{}) { return ee.count, ee.last } -func (ee *CSVExporter) getPkIndex(titles []string, pk string) int { +func (ee *CsvExporter) getPkIndex(titles []string, pk string) int { for i, title := range titles { if title == pk { return i diff --git a/biz/export/cvs.go b/biz/export/cvs.go index 5e45509..dd7227f 100644 --- a/biz/export/cvs.go +++ b/biz/export/cvs.go @@ -10,7 +10,7 @@ import ( "reflect" ) -type CSV struct { +type Csv struct { fc *os.File csv *csv.Writer f *File @@ -18,23 +18,23 @@ type CSV struct { titles []string } -func NewCSV(fileName string, limit int, param map[string]string) *CSV { - return &CSV{ - f: NewFile(fileName, limit, param), +func NewCsv(fileName string, param map[string]string) *Csv { + return &Csv{ + f: NewFile(fileName, 100000000, param), } } -func (e *CSV) slice() { +func (e *Csv) slice() { if e.f.slice() { e.reset() } } -func (e *CSV) SetParam(param map[string]string) { +func (e *Csv) SetParam(param map[string]string) { e.f.param = param } -func (e *CSV) reset() { +func (e *Csv) reset() { e.save() e.f.NextFile() e.Open() @@ -42,7 +42,7 @@ func (e *CSV) reset() { e.slice() } -func (e *CSV) Open() error { +func (e *Csv) Open() error { var err error if e.f.IsFileExist() { e.fc, err = os.OpenFile(e.f.FileName(), os.O_APPEND|os.O_WRONLY, 0644) @@ -62,7 +62,7 @@ func (e *CSV) Open() error { return nil } -func (e *CSV) getLineCount(file io.Reader) (line int) { +func (e *Csv) getLineCount(file io.Reader) (line int) { reader := bufio.NewReader(file) line = 0 for { @@ -77,13 +77,13 @@ func (e *CSV) getLineCount(file io.Reader) (line int) { return line } -func (e *CSV) save() error { +func (e *Csv) save() error { e.csv.Flush() e.fc.Close() return nil } -func (e *CSV) WriteTitle(titles []string) error { +func (e *Csv) WriteTitle(titles []string) error { if titles != nil { e.titles = titles @@ -96,7 +96,7 @@ func (e *CSV) WriteTitle(titles []string) error { return nil } -func (e *CSV) Write(data interface{}) error { +func (e *Csv) Write(data interface{}) error { if e.f.slice() { e.reset() } @@ -122,6 +122,6 @@ func (e *CSV) Write(data interface{}) error { return nil } -func (e *CSV) Close() error { +func (e *Csv) Close() error { return e.save() } diff --git a/cmd/cmd/excel.go b/cmd/cmd/excel.go index ee3db20..8f2488a 100644 --- a/cmd/cmd/excel.go +++ b/cmd/cmd/excel.go @@ -55,6 +55,7 @@ func (e *Excel) JobHandler(job config.Job, d export.DataFetcher, params map[stri func (e *Excel) TaskExport(d export.DataFetcher, t config.Task, params map[string]interface{}, f export.FileAdapter, batch int) error { //todo 最多分1000个批次进行处理 + f.Open() defer f.Close() for i := 0; i < 1000; i++ { diff --git a/cmd/cmd/root.go b/cmd/cmd/root.go index 4698946..a2dba94 100644 --- a/cmd/cmd/root.go +++ b/cmd/cmd/root.go @@ -86,7 +86,7 @@ func rootRun(cmd *cobra.Command, args []string) { CmdError(cmd, "无效的参数:%s", err.Error()) } b := time.Now() - ee := NewExcel(config.DefaultConfig) + ee := NewCsv(config.DefaultConfig) ee.Export(sName, jName, begin, end, batch) e := time.Now() fmt.Println("耗时:" + e.Sub(b).String()) diff --git a/go.mod b/go.mod index 7ab592b..ddd44fa 100644 --- a/go.mod +++ b/go.mod @@ -31,9 +31,12 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/richardlehane/mscfb v1.0.4 // indirect + github.com/richardlehane/msoleps v1.0.3 // indirect github.com/rogpeppe/fastuuid v1.2.0 // indirect github.com/rogpeppe/go-internal v1.6.1 // indirect github.com/shabbyrobe/xmlwriter v0.0.0-20200208144257-9fca06d00ffa // indirect @@ -41,8 +44,13 @@ require ( github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect - golang.org/x/sys v0.3.0 // indirect - golang.org/x/text v0.5.0 // indirect + github.com/xuri/efp v0.0.0-20220603152613-6918739fd470 // indirect + github.com/xuri/excelize/v2 v2.7.1 // indirect + github.com/xuri/nfp v0.0.0-20220409054826-5e722a1d9e22 // indirect + golang.org/x/crypto v0.8.0 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect )