package l_export_async import ( "context" "time" ) // DataProviderFn 定义数据提供函数的类型别名 // 这里目前存在三种情况 // 1. 分页导出 limit ... offset // 2. 游标导出 id>... ORDER BY id LIMIT ...; // 3. 时间范围导出 time_column BETWEEN ... AND ... ORDER BY time_column LIMIT ...; type DataProviderFn func(ctx context.Context, pageNum, limit int) ([][]interface{}, error) // Uploader 这里主要是为了调用attachment.Upload方法 type Uploader struct { Host string System string Business string FieldFormName string } type ExportOption func(*ExportAsync) // WithCustomUploader 自定义上传配置 func WithCustomUploader(sys string, business string, fieldFormName string) ExportOption { return func(b *ExportAsync) { b.uploader = &Uploader{ System: sys, Business: business, FieldFormName: fieldFormName, } } } // WithCustomBatchSize 每一批次导出数量,数据库每页行数,默认10000行 func WithCustomBatchSize(batchSize int) ExportOption { return func(b *ExportAsync) { b.batchSize = batchSize } } // WithCustomBatchSize 每一批次导出数量,数据库每页行数,默认10000行 func WithTotalProcess(batchSize int) ExportOption { return func(b *ExportAsync) { b.batchSize = batchSize } } // WithCustomBufferSize csv转excel的批量写入缓冲区大小,逐行写入设置为0 func WithCustomBufferSize(bufferSize int) ExportOption { return func(b *ExportAsync) { b.csvToExcelBatch = bufferSize } } // WithCustomSuffixFileName 自定后缀 func WithCustomSuffixFileName(suffix string) ExportOption { return func(b *ExportAsync) { b.fileName = b.fileName + "_" + suffix } } // WithCustomWorkNum 自定义协程数 func WithCustomWorkNum(num int) ExportOption { return func(b *ExportAsync) { b.workerNum = num } } // WithCustomExtension 自定义扩展名 func WithCustomExtension(extension string) ExportOption { return func(b *ExportAsync) { b.extension = extension } } // WithCustomSheetName 自定义注脚 func WithCustomSheetName(sheetName string) ExportOption { return func(b *ExportAsync) { b.sheetName = sheetName } } // WithProcess 需要进行进度统计 func WithProcess(dataCount int) ExportOption { return func(b *ExportAsync) { b.dataCount = dataCount } } // WithLogPrint 日志输出组件 func WithLogPrint(logTool LogTool) ExportOption { return func(b *ExportAsync) { b.logTool = NewLogPrint(logTool) } } // WithMaxRowPerFile 每个Xlsx的行数,默认10000行 func WithMaxRowPerFile(maxRowPerFile int) ExportOption { return func(b *ExportAsync) { b.maxRowPerFile = maxRowPerFile } } // WithOffset 传统分页数据 func WithOffset(fetcher OffsetDataFetcher) ExportOption { return func(e *ExportAsync) { if e.batchSize == 0 { e.batchSize = DefaultBatch } e.setPageStrategy(&OffsetStrategy{ fetcher: fetcher, limit: e.batchSize, }) } } // WithCursor 游标分页数据 func WithCursor(fetcher CursorDataFetcher, initialCursor interface{}) ExportOption { return func(e *ExportAsync) { if e.batchSize == 0 { e.batchSize = DefaultBatch } e.setPageStrategy(&CursorStrategy{ fetcher: fetcher, limit: e.batchSize, initialCursor: initialCursor, }) } } // WithTimeRange 时间范围分页 func WithTimeRange(fetcher TimeRangeDataFetcher, startTime time.Time, timeRange time.Duration) ExportOption { return func(e *ExportAsync) { if e.batchSize == 0 { e.batchSize = DefaultBatch } e.setPageStrategy(&TimeRangeStrategy{ fetcher: fetcher, limit: e.batchSize, startTime: startTime, timeRange: timeRange, }) } } type Task struct { Id string `json:"id"` Name string `json:"name"` Process int32 `json:"process"` //需除10000,上限是100 //初始化10,生成csv60,合并数据20,上传数据10,ProcessScore Err string `json:"err"` Source string `json:"source"` RowCount int64 `json:"row_count"` } type ProcessScore int32 const ( INIT ProcessScore = 100000 CSV ProcessScore = 600000 XLSX ProcessScore = 200000 ATT ProcessScore = 100000 ) const CacheKey = "export_async_task" func (p ProcessScore) int() int32 { return int32(p) } func (p ProcessScore) float64() float64 { return float64(p) }