feat(export): 根据预估行数限制xlsx格式导出
- 新增xlsx格式最大行数阈值配置,超过该阈值则导出格式强制为csv - 查询export_jobs表中作业的预估行数用于判断格式切换 - 记录格式切换时的日志,包含作业ID、预估行数及阈值信息 - 避免xlsx格式导出过大文件,提高导出效率和稳定性 - 在export常量中添加XlsxMaxRows配置项,默认值为100000行
This commit is contained in:
parent
b50a615d8b
commit
0de192c9b4
|
|
@ -393,6 +393,24 @@ func (a *ExportsAPI) runJob(id uint64, db *sql.DB, q string, args []interface{},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 检查预估行数,如果超过阈值且格式是xlsx,强制改为csv
|
||||||
|
if fmt == "xlsx" {
|
||||||
|
var rowEstimate int64
|
||||||
|
estRow := a.meta.QueryRow("SELECT row_estimate FROM export_jobs WHERE id=?", id)
|
||||||
|
_ = estRow.Scan(&rowEstimate)
|
||||||
|
if rowEstimate > constants.ExportThresholds.XlsxMaxRows {
|
||||||
|
logging.JSON("INFO", map[string]interface{}{
|
||||||
|
"event": "force_csv_format",
|
||||||
|
"job_id": id,
|
||||||
|
"row_estimate": rowEstimate,
|
||||||
|
"threshold": constants.ExportThresholds.XlsxMaxRows,
|
||||||
|
"reason": "row_estimate exceeds xlsx max rows, forcing csv format",
|
||||||
|
})
|
||||||
|
fmt = "csv"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
rrepo.StartJob(a.meta, id)
|
rrepo.StartJob(a.meta, id)
|
||||||
if fmt == "csv" {
|
if fmt == "csv" {
|
||||||
newBaseWriter := func() (exporter.RowWriter, error) {
|
newBaseWriter := func() (exporter.RowWriter, error) {
|
||||||
|
|
|
||||||
|
|
@ -60,12 +60,15 @@ var ExportThresholds = struct {
|
||||||
ChunkThreshold int64
|
ChunkThreshold int64
|
||||||
// ProgressUpdateInterval 进度更新间隔(行数)
|
// ProgressUpdateInterval 进度更新间隔(行数)
|
||||||
ProgressUpdateInterval int64
|
ProgressUpdateInterval int64
|
||||||
|
// XlsxMaxRows xlsx格式最大行数,超过则强制使用csv
|
||||||
|
XlsxMaxRows int64
|
||||||
}{
|
}{
|
||||||
MaxRowsPerFile: 300000,
|
MaxRowsPerFile: 300000,
|
||||||
PassScoreThreshold: 60,
|
PassScoreThreshold: 60,
|
||||||
ChunkDays: 10,
|
ChunkDays: 10,
|
||||||
ChunkThreshold: 50000,
|
ChunkThreshold: 50000,
|
||||||
ProgressUpdateInterval: 1000,
|
ProgressUpdateInterval: 1000,
|
||||||
|
XlsxMaxRows: 100000,
|
||||||
}
|
}
|
||||||
|
|
||||||
// BatchSizes 批量处理大小配置
|
// BatchSizes 批量处理大小配置
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue