diff --git a/biz/export/excel.go b/biz/export/excel.go deleted file mode 100644 index 0dc45af..0000000 --- a/biz/export/excel.go +++ /dev/null @@ -1,115 +0,0 @@ -package export - -import ( - "errors" - "github.com/tealeg/xlsx/v3" - "reflect" -) - -const Sheet_Name = "sheet_1" - -type Excel struct { - f *File - count int //总数 - isNew bool - titles []string - file *xlsx.File - sheet *xlsx.Sheet -} - -func NewExcel(fileName string, limit int, param map[string]string) *Excel { - return &Excel{ - f: NewFile(fileName, limit, param), - } -} - -func (e *Excel) slice() { - if e.f.slice() { - e.reset() - } -} - -func (e *Excel) reset() { - e.save() - e.f.NextFile() - e.Open() - e.WriteTitle(nil) - e.slice() -} - -func (e *Excel) Open() error { - var err error - if e.f.IsFileExist() { - e.file, err = xlsx.OpenFile(e.f.FileName()) - if err != nil { - return err - } - } else { - e.isNew = true - e.file = xlsx.NewFile() - } - - var ok bool - e.sheet, ok = e.file.Sheet[Sheet_Name] - if !ok { - e.sheet, err = e.file.AddSheet(Sheet_Name) - if err != nil { - return err - } - } else { - //需要处理偏移数据 - e.f.SetRow(e.sheet.MaxRow) - } - - return nil -} - -func (e *Excel) save() error { - return e.file.Save(e.f.FileName()) -} - -func (e *Excel) WriteTitle(titles []string) error { - if e.file == nil || e.sheet == nil { - return errors.New("没有执行open方法") - } - - if titles != nil { - e.titles = titles - } - - if e.titles != nil && e.isNew { - e.Write(e.titles) - e.isNew = false - } - return nil -} - -func (e *Excel) Write(data interface{}) error { - if e.f.slice() { - e.reset() - } - row := e.sheet.AddRow() - - v := reflect.ValueOf(data) - if v.Kind() == reflect.Ptr { - v = v.Elem() - } - if v.Kind() != reflect.Slice { - return errors.New("数据无效，不是切片类型") - } - - switch val := data.(type) { - case []string: - row.WriteSlice(val, -1) - case []interface{}: - for _, i := range data.([]interface{}) { - cell := row.AddCell() - cell.SetValue(i) - } - } - return nil -} - -func (e *Excel) Close() error { - return e.save() -} diff --git a/biz/export/excel_exporter.go b/biz/export/excel_exporter.go deleted file mode 100644 index aa1aa32..0000000 --- a/biz/export/excel_exporter.go +++ /dev/null @@ -1,67 +0,0 @@ -package export - -import ( - "fmt" - "log" -) - -type ExcelExporter struct { - mFetcher DataFetcher - file FileAdapter - count int - last interface{} -} - -func NewExcelExporter(fetcher DataFetcher, file FileAdapter) DataExporter { - return &ExcelExporter{ - mFetcher: fetcher, - file: file, - } -} - -func (ee *ExcelExporter) Fetcher(fetcher DataFetcher) { - ee.mFetcher = fetcher -} - -func (ee *ExcelExporter) File(file FileAdapter) { - ee.file = file -} - -func (ee *ExcelExporter) Export(sql, pk string) error { - data, err := ee.mFetcher.Fetch(sql) - if err != nil { - return fmt.Errorf("数据获取错误:%w", err) - } - ee.count = len(data.Data) - //fmt.Printf("Excel Exporter.Excel, got %v data\n", len(data)) - //ee.file.Open() - ee.file.WriteTitle(data.Title) - var last []string - for _, val := range data.Data { - last = val - ee.file.Write(last) - } - - if len(last) > 0 { - ee.last = last[getPkIndex(data.Title, pk)] - } - //ee.file.Close() - return nil -} - -func (ee *ExcelExporter) Last() (int, interface{}) { - return ee.count, ee.last -} - -func getPkIndex(titles []string, pk string) int { - if pk == "" { - log.Println("pk is not empty") - return -1 - } - for i, title := range titles { - if title == pk { - return i - } - } - return -1 -} diff --git a/biz/export/excel_exporter_test.go b/biz/export/excel_exporter_test.go deleted file mode 100644 index 2c464b7..0000000 --- a/biz/export/excel_exporter_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package export - -import ( - "github.com/stretchr/testify/assert" - "os" - "testing" -) - -func TestExcelExporter_Export(t *testing.T) { - data := NewMysqlDataFetcher("aaa") - pwd, _ := os.Getwd() - file := NewExcel(pwd+"/aa-{begin}.xlsx", 5, map[string]string{"begin": "202301"}) - e := NewExcelExporter(data, file) - file.Open() - err := e.Export("aa", "字段1") - file.Close() - assert.Nil(t, err) - - assert.FileExists(t, pwd+"/aa-202301_0.xlsx") - assert.FileExists(t, pwd+"/aa-202301_1.xlsx") - assert.NoFileExists(t, pwd+"/aa-202301_2.xlsx") - - _ = os.Remove(pwd + "/aa-202301_0.xlsx") - _ = os.Remove(pwd + "/aa-202301_1.xlsx") - -} diff --git a/biz/export/excel_test.go b/biz/export/excel_test.go deleted file mode 100644 index 1e0cad4..0000000 --- a/biz/export/excel_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package export - -import ( - "github.com/stretchr/testify/assert" - "os" - "testing" -) - -func TestExcel_Write(t *testing.T) { - pwd, _ := os.Getwd() - e := NewExcel(pwd+"/aa-{begin}.xlsx", 5, map[string]string{"begin": "202301"}) - - e.Open() - - e.WriteTitle([]string{"姓名", "年龄"}) - data := make([]interface{}, 2) - data[0] = "张三" - for i := 0; i < 9; i++ { - data[1] = 10 + i - e.Write(data) - } - e.Close() - - assert.FileExists(t, pwd+"/aa-202301_0.xlsx") - assert.FileExists(t, pwd+"/aa-202301_1.xlsx") - assert.NoFileExists(t, pwd+"/aa-202301_2.xlsx") - - _ = os.Remove(pwd + "/aa-202301_0.xlsx") - _ = os.Remove(pwd + "/aa-202301_1.xlsx") - -} diff --git a/biz/export/mysql_data_fetcher.go b/biz/export/mysql_data_fetcher.go deleted file mode 100644 index d8e48f3..0000000 --- a/biz/export/mysql_data_fetcher.go +++ /dev/null @@ -1,35 +0,0 @@ -package export - -import ( - "math/rand" - "strconv" -) - -type MysqlDataFetcher struct { - Config string -} - -func (mf *MysqlDataFetcher) Fetch(sql string) (*Data, error) { - rows := make([][]string, 0, 6) - // 插入6个随机数组成的切片，模拟查询要返回的数据集 - rows = append(rows, row(), row(), row(), row(), row(), row()) - return &Data{ - Title: []string{"字段1", "字段2", "字段3", "字段4", "字段5"}, - Data: rows, - }, nil -} - -func NewMysqlDataFetcher(configStr string) DataFetcher { - return &MysqlDataFetcher{ - Config: configStr, - } -} - -func row() []string { - strs := make([]string, 5) - nums := rand.Perm(5) - for i, num := range nums { - strs[i] = strconv.Itoa(num) - } - return strs -} diff --git a/cmd/cmd/excel.go b/cmd/cmd/excel.go deleted file mode 100644 index 8f2488a..0000000 --- a/cmd/cmd/excel.go +++ /dev/null @@ -1,83 +0,0 @@ -package cmd - -import ( - "excel_export/biz/config" - "excel_export/biz/db" - "excel_export/biz/export" - "fmt" - "strconv" - "time" -) - -type Excel struct { - conf *config.Config -} - -func NewExcel(conf *config.Config) *Excel { - return &Excel{ - conf: conf, - } -} - -func (e *Excel) Export(sysName, jobName string, begin, end time.Time, batch int) error { - job, dbStr, err := config.GetJob(e.conf, sysName, jobName) - if err != nil { - return err - } - - d, err := db.NewDb(dbStr) - if err != nil { - return err - } - - return e.JobHandler(job, d, map[string]interface{}{ - "begin": begin, - "end": end, - "last": 0, - }, batch) -} - -func (e *Excel) JobHandler(job config.Job, d export.DataFetcher, params map[string]interface{}, batch int) error { - - f := export.NewExcel(job.File, job.Size, map[string]string{ - "begin": params["begin"].(time.Time).Format("20060102"), - "end": params["end"].(time.Time).Format("20060102"), - }) - - for i, task := range job.Tasks { - fmt.Printf("执行导出任务：%d\n", i+1) - if err := e.TaskExport(d, task, params, f, batch); err != nil { - return err - } - } - return nil -} - -func (e *Excel) TaskExport(d export.DataFetcher, t config.Task, params map[string]interface{}, f export.FileAdapter, batch int) error { - //todo 最多分1000个批次进行处理 - - f.Open() - defer f.Close() - for i := 0; i < 1000; i++ { - sql := t.GetSql(params) - - e := export.NewExcelExporter(d, f) - - e.Export(sql+" limit "+strconv.Itoa(batch), t.PK) - - count, last := e.Last() - fmt.Printf("已导出 %d 条数据\n", batch*i+count) - - if count == 0 { - return nil - } - - if count < batch { - break - } - - params["last"] = last - time.Sleep(time.Microsecond * 30) - } - return nil -}