l-export-async/entity.go

package l_export_async

import (
	"context"
	"time"
)

// DataProviderFn 定义数据提供函数的类型别名
// 这里目前存在三种情况
// 1. 分页导出 limit ... offset
// 2. 游标导出 id>... ORDER BY id LIMIT ...;
// 3. 时间范围导出 time_column BETWEEN ... AND ... ORDER BY time_column LIMIT ...;

type DataProviderFn func(ctx context.Context, pageNum, limit int) ([][]interface{}, error)

// Uploader 这里主要是为了调用attachment.Upload方法
type Uploader struct {
	Host          string
	System        string
	Business      string
	FieldFormName string
}

type ExportOption func(*ExportAsync)

// WithCustomUploader 自定义上传配置
func WithCustomUploader(sys string, business string, fieldFormName string) ExportOption {
	return func(b *ExportAsync) {
		b.uploader = &Uploader{
			System:        sys,
			Business:      business,
			FieldFormName: fieldFormName,
		}
	}
}

// WithCustomBatchSize 每一批次导出数量，数据库每页行数，默认10000行
func WithCustomBatchSize(batchSize int) ExportOption {
	return func(b *ExportAsync) {
		b.batchSize = batchSize
	}
}

// WithCustomBatchSize 每一批次导出数量，数据库每页行数，默认10000行
func WithTotalProcess(batchSize int) ExportOption {
	return func(b *ExportAsync) {
		b.batchSize = batchSize
	}
}

// WithCustomBufferSize csv转excel的批量写入缓冲区大小,逐行写入设置为0
func WithCustomBufferSize(bufferSize int) ExportOption {
	return func(b *ExportAsync) {
		b.csvToExcelBatch = bufferSize
	}
}

// WithCustomSuffixFileName 自定后缀
func WithCustomSuffixFileName(suffix string) ExportOption {
	return func(b *ExportAsync) {
		b.fileName = b.fileName + "_" + suffix
	}
}

// WithCustomWorkNum 自定义协程数
func WithCustomWorkNum(num int) ExportOption {
	return func(b *ExportAsync) {
		b.workerNum = num
	}
}

// WithCustomExtension 自定义扩展名
func WithCustomExtension(extension string) ExportOption {
	return func(b *ExportAsync) {
		b.extension = extension
	}
}

// WithCustomSheetName 自定义注脚
func WithCustomSheetName(sheetName string) ExportOption {
	return func(b *ExportAsync) {
		b.sheetName = sheetName
	}
}

// WithProcess 需要进行进度统计
func WithProcess(dataCount int) ExportOption {
	return func(b *ExportAsync) {
		b.dataCount = dataCount
	}
}

// WithLogPrint 日志输出组件
func WithLogPrint(logTool LogTool) ExportOption {
	return func(b *ExportAsync) {
		b.logTool = NewLogPrint(logTool)
	}
}

// WithMaxRowPerFile 每个Xlsx的行数，默认10000行
func WithMaxRowPerFile(maxRowPerFile int) ExportOption {
	return func(b *ExportAsync) {
		b.maxRowPerFile = maxRowPerFile
	}
}

// WithOffset 传统分页数据
func WithOffset(fetcher OffsetDataFetcher) ExportOption {
	return func(e *ExportAsync) {
		if e.batchSize == 0 {
			e.batchSize = DefaultBatch
		}
		e.setPageStrategy(&OffsetStrategy{
			fetcher: fetcher,
			limit:   e.batchSize,
		})
	}
}

// WithCursor 游标分页数据
func WithCursor(fetcher CursorDataFetcher, initialCursor interface{}) ExportOption {
	return func(e *ExportAsync) {
		if e.batchSize == 0 {
			e.batchSize = DefaultBatch
		}
		e.setPageStrategy(&CursorStrategy{
			fetcher:       fetcher,
			limit:         e.batchSize,
			initialCursor: initialCursor,
		})
	}
}

// WithTimeRange 时间范围分页
func WithTimeRange(fetcher TimeRangeDataFetcher, startTime time.Time, timeRange time.Duration) ExportOption {
	return func(e *ExportAsync) {
		if e.batchSize == 0 {
			e.batchSize = DefaultBatch
		}
		e.setPageStrategy(&TimeRangeStrategy{
			fetcher:   fetcher,
			limit:     e.batchSize,
			startTime: startTime,
			timeRange: timeRange,
		})
	}
}

type Task struct {
	Id       string `json:"id"`
	Name     string `json:"name"`
	Process  int32  `json:"process"` //需除10000，上限是100 //初始化10，生成csv60，合并数据20，上传数据10,ProcessScore
	Err      string `json:"err"`
	Source   string `json:"source"`
	RowCount int64  `json:"row_count"`
}

type ProcessScore int32

const (
	INIT ProcessScore = 100000
	CSV  ProcessScore = 600000
	XLSX ProcessScore = 200000
	ATT  ProcessScore = 100000
)

const CacheKey = "export_async_task"

func (p ProcessScore) int() int32 {
	return int32(p)
}

func (p ProcessScore) float64() float64 {
	return float64(p)
}