feat(export): 根据预估行数限制xlsx格式导出

- 新增xlsx格式最大行数阈值配置,超过该阈值则导出格式强制为csv
- 查询export_jobs表中作业的预估行数用于判断格式切换
- 记录格式切换时的日志,包含作业ID、预估行数及阈值信息
- 避免xlsx格式导出过大文件,提高导出效率和稳定性
- 在export常量中添加XlsxMaxRows配置项,默认值为100000行
This commit is contained in:
zhouyonggao 2025-12-18 20:58:23 +08:00
parent b50a615d8b
commit 0de192c9b4
2 changed files with 21 additions and 0 deletions

View File

@ -393,6 +393,24 @@ func (a *ExportsAPI) runJob(id uint64, db *sql.DB, q string, args []interface{},
}
}
}
// 检查预估行数如果超过阈值且格式是xlsx强制改为csv
if fmt == "xlsx" {
var rowEstimate int64
estRow := a.meta.QueryRow("SELECT row_estimate FROM export_jobs WHERE id=?", id)
_ = estRow.Scan(&rowEstimate)
if rowEstimate > constants.ExportThresholds.XlsxMaxRows {
logging.JSON("INFO", map[string]interface{}{
"event": "force_csv_format",
"job_id": id,
"row_estimate": rowEstimate,
"threshold": constants.ExportThresholds.XlsxMaxRows,
"reason": "row_estimate exceeds xlsx max rows, forcing csv format",
})
fmt = "csv"
}
}
rrepo.StartJob(a.meta, id)
if fmt == "csv" {
newBaseWriter := func() (exporter.RowWriter, error) {

View File

@ -60,12 +60,15 @@ var ExportThresholds = struct {
ChunkThreshold int64
// ProgressUpdateInterval 进度更新间隔(行数)
ProgressUpdateInterval int64
// XlsxMaxRows xlsx格式最大行数超过则强制使用csv
XlsxMaxRows int64
}{
MaxRowsPerFile: 300000,
PassScoreThreshold: 60,
ChunkDays: 10,
ChunkThreshold: 50000,
ProgressUpdateInterval: 1000,
XlsxMaxRows: 100000,
}
// BatchSizes 批量处理大小配置