feat(export): 根据预估行数限制xlsx格式导出
- 新增xlsx格式最大行数阈值配置,超过该阈值则导出格式强制为csv - 查询export_jobs表中作业的预估行数用于判断格式切换 - 记录格式切换时的日志,包含作业ID、预估行数及阈值信息 - 避免xlsx格式导出过大文件,提高导出效率和稳定性 - 在export常量中添加XlsxMaxRows配置项,默认值为100000行
This commit is contained in:
parent
b50a615d8b
commit
0de192c9b4
|
|
@ -393,6 +393,24 @@ func (a *ExportsAPI) runJob(id uint64, db *sql.DB, q string, args []interface{},
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 检查预估行数,如果超过阈值且格式是xlsx,强制改为csv
|
||||
if fmt == "xlsx" {
|
||||
var rowEstimate int64
|
||||
estRow := a.meta.QueryRow("SELECT row_estimate FROM export_jobs WHERE id=?", id)
|
||||
_ = estRow.Scan(&rowEstimate)
|
||||
if rowEstimate > constants.ExportThresholds.XlsxMaxRows {
|
||||
logging.JSON("INFO", map[string]interface{}{
|
||||
"event": "force_csv_format",
|
||||
"job_id": id,
|
||||
"row_estimate": rowEstimate,
|
||||
"threshold": constants.ExportThresholds.XlsxMaxRows,
|
||||
"reason": "row_estimate exceeds xlsx max rows, forcing csv format",
|
||||
})
|
||||
fmt = "csv"
|
||||
}
|
||||
}
|
||||
|
||||
rrepo.StartJob(a.meta, id)
|
||||
if fmt == "csv" {
|
||||
newBaseWriter := func() (exporter.RowWriter, error) {
|
||||
|
|
|
|||
|
|
@ -60,12 +60,15 @@ var ExportThresholds = struct {
|
|||
ChunkThreshold int64
|
||||
// ProgressUpdateInterval 进度更新间隔(行数)
|
||||
ProgressUpdateInterval int64
|
||||
// XlsxMaxRows xlsx格式最大行数,超过则强制使用csv
|
||||
XlsxMaxRows int64
|
||||
}{
|
||||
MaxRowsPerFile: 300000,
|
||||
PassScoreThreshold: 60,
|
||||
ChunkDays: 10,
|
||||
ChunkThreshold: 50000,
|
||||
ProgressUpdateInterval: 1000,
|
||||
XlsxMaxRows: 100000,
|
||||
}
|
||||
|
||||
// BatchSizes 批量处理大小配置
|
||||
|
|
|
|||
Loading…
Reference in New Issue