166 lines
4.1 KiB
Go
166 lines
4.1 KiB
Go
// Package constants 定义系统常量、阈值和配置
|
||
package constants
|
||
|
||
import "time"
|
||
|
||
// ==================== 导出任务状态 ====================
|
||
|
||
// JobStatus 导出任务状态枚举
|
||
type JobStatus string
|
||
|
||
const (
|
||
JobStatusQueued JobStatus = "queued" // 排队中
|
||
JobStatusRunning JobStatus = "running" // 执行中
|
||
JobStatusCompleted JobStatus = "completed" // 已完成
|
||
JobStatusFailed JobStatus = "failed" // 失败
|
||
JobStatusCanceled JobStatus = "canceled" // 已取消
|
||
)
|
||
|
||
// ==================== 数据源类型 ====================
|
||
|
||
// Datasource 数据源类型
|
||
type Datasource string
|
||
|
||
const (
|
||
DatasourceMarketing Datasource = "marketing" // 营销系统
|
||
DatasourceYMT Datasource = "ymt" // 易码通
|
||
)
|
||
|
||
// ==================== 主表类型 ====================
|
||
|
||
// MainTable 主表类型
|
||
type MainTable string
|
||
|
||
const (
|
||
MainTableOrder MainTable = "order" // 订单表
|
||
MainTableOrderInfo MainTable = "order_info" // 订单信息表(YMT)
|
||
)
|
||
|
||
// ==================== 导出格式 ====================
|
||
|
||
// FileFormat 导出文件格式
|
||
type FileFormat string
|
||
|
||
const (
|
||
FileFormatCSV FileFormat = "csv"
|
||
FileFormatXLSX FileFormat = "xlsx"
|
||
)
|
||
|
||
// ==================== 导出阈值配置 ====================
|
||
|
||
// ExportThresholds 导出相关阈值
|
||
var ExportThresholds = struct {
|
||
// MaxRowsPerFile 单文件最大行数
|
||
MaxRowsPerFile int64
|
||
// PassScoreThreshold EXPLAIN评分通过阈值
|
||
PassScoreThreshold int
|
||
// ChunkDays 分块导出的天数步长
|
||
ChunkDays int
|
||
// ChunkThreshold 启用分块导出的行数阈值
|
||
ChunkThreshold int64
|
||
// ProgressUpdateInterval 进度更新间隔(行数)
|
||
ProgressUpdateInterval int64
|
||
// XlsxMaxRows xlsx格式最大行数，超过则强制使用csv
|
||
XlsxMaxRows int64
|
||
}{
|
||
MaxRowsPerFile: 500000, // 修改：单个文件最大5万行数据
|
||
PassScoreThreshold: 60,
|
||
ChunkDays: 10,
|
||
ChunkThreshold: 50000,
|
||
ProgressUpdateInterval: 1000,
|
||
XlsxMaxRows: 100000,
|
||
}
|
||
|
||
// BatchSizes 批量处理大小配置
|
||
var BatchSizes = struct {
|
||
// CSVDefault CSV默认批次大小
|
||
CSVDefault int
|
||
// XLSXDefault XLSX默认批次大小
|
||
XLSXDefault int
|
||
// SmallDataset 小数据集批次
|
||
SmallDataset int
|
||
// MediumDataset 中等数据集批次
|
||
MediumDataset int
|
||
// LargeDataset 大数据集批次
|
||
LargeDataset int
|
||
// HugeDataset 超大数据集批次
|
||
HugeDataset int
|
||
// MassiveDataset 海量数据集批次 (用于 5000万+ 数捥)
|
||
MassiveDataset int
|
||
}{
|
||
CSVDefault: 10000,
|
||
XLSXDefault: 5000,
|
||
SmallDataset: 10000,
|
||
MediumDataset: 20000,
|
||
LargeDataset: 50000,
|
||
HugeDataset: 200000, // 上汪一一次
|
||
MassiveDataset: 500000, // 新增：海量数据集专用
|
||
}
|
||
|
||
// ChooseBatchSize 根据估算行数和格式选择合適的批次大小
|
||
func ChooseBatchSize(estimate int64, format FileFormat) int {
|
||
if format == FileFormatXLSX {
|
||
return BatchSizes.XLSXDefault
|
||
}
|
||
if estimate <= 0 {
|
||
return BatchSizes.CSVDefault
|
||
}
|
||
if estimate < 50000 {
|
||
return BatchSizes.SmallDataset
|
||
}
|
||
if estimate < 200000 {
|
||
return BatchSizes.MediumDataset
|
||
}
|
||
if estimate < 500000 {
|
||
return BatchSizes.LargeDataset
|
||
}
|
||
if estimate >= 50000000 { // 5000万+ 数据
|
||
return BatchSizes.MassiveDataset
|
||
}
|
||
if estimate >= 2000000 {
|
||
return BatchSizes.HugeDataset
|
||
}
|
||
return BatchSizes.LargeDataset
|
||
}
|
||
|
||
// ==================== HTTP服务器配置 ====================
|
||
|
||
// ServerConfig HTTP服务器配置
|
||
var ServerConfig = struct {
|
||
DefaultPort string
|
||
ReadTimeout time.Duration
|
||
WriteTimeout time.Duration
|
||
}{
|
||
DefaultPort: "8077",
|
||
ReadTimeout: 15 * time.Second,
|
||
WriteTimeout: 60 * time.Second,
|
||
}
|
||
|
||
// ==================== 分页配置 ====================
|
||
|
||
// PaginationConfig 分页配置
|
||
var PaginationConfig = struct {
|
||
DefaultPageSize int
|
||
MaxPageSize int
|
||
}{
|
||
DefaultPageSize: 15,
|
||
MaxPageSize: 100,
|
||
}
|
||
|
||
// ==================== 存储配置 ====================
|
||
|
||
// StorageConfig 存储配置
|
||
var StorageConfig = struct {
|
||
Directory string
|
||
FilePrefix string
|
||
ZipSuffix string
|
||
CSVSuffix string
|
||
XLSXSuffix string
|
||
}{
|
||
Directory: "storage",
|
||
FilePrefix: "export_job_",
|
||
ZipSuffix: ".zip",
|
||
CSVSuffix: ".csv",
|
||
XLSXSuffix: ".xlsx",
|
||
}
|