fix(exports): 优化导出逻辑以支持动态时间范围分片

- 在 CountRowsFastChunked 函数中添加时间跨度计算,支持根据时间跨度选择按天或按周分片
- 新增 SplitByWeeks 函数以实现按周分割时间范围
- 增强日志记录,提供分片信息以便于后续分析和调试
This commit is contained in:
zhouyonggao 2025-12-19 02:33:59 +08:00
parent ccb12da7f3
commit 61a3b17c09
1 changed files with 48 additions and 1 deletions

View File

@ -195,7 +195,30 @@ func CountRowsFastChunked(db *sql.DB, ds, main string, filters map[string]interf
if start == "" || end == "" { if start == "" || end == "" {
return CountRowsFast(db, ds, main, filters) return CountRowsFast(db, ds, main, filters)
} }
ranges := SplitByDays(start, end, 15) // 计算时间跨度(天数)
layout := "2006-01-02 15:04:05"
st, err1 := time.Parse(layout, start)
en, err2 := time.Parse(layout, end)
if err1 != nil || err2 != nil {
return CountRowsFast(db, ds, main, filters)
}
daysDiff := int(en.Sub(st).Hours() / 24)
// 如果时间跨度超过 15 天半个月使用按周分片7天否则使用按天分片15天
var ranges [][2]string
if daysDiff > 15 {
ranges = SplitByWeeks(start, end)
logging.JSON("INFO", map[string]interface{}{
"event": "count_chunked_by_weeks",
"datasource": ds,
"main": main,
"days_diff": daysDiff,
"chunks": len(ranges),
})
} else {
ranges = SplitByDays(start, end, 15)
}
var total int64 var total int64
for _, rg := range ranges { for _, rg := range ranges {
fl := map[string]interface{}{} fl := map[string]interface{}{}
@ -232,6 +255,30 @@ func SplitByDays(startStr, endStr string, stepDays int) [][2]string {
return out return out
} }
// SplitByWeeks 按周7天分割时间范围返回多个时间区间
func SplitByWeeks(startStr, endStr string) [][2]string {
layout := "2006-01-02 15:04:05"
s := strings.TrimSpace(startStr)
e := strings.TrimSpace(endStr)
st, err1 := time.Parse(layout, s)
en, err2 := time.Parse(layout, e)
if err1 != nil || err2 != nil || !en.After(st) {
return [][2]string{{s, e}}
}
var out [][2]string
cur := st
weekDuration := 7 * 24 * time.Hour
for cur.Before(en) {
nxt := cur.Add(weekDuration)
if nxt.After(en) {
nxt = en
}
out = append(out, [2]string{cur.Format(layout), nxt.Format(layout)})
cur = nxt
}
return out
}
type RowTransform func([]string) []string type RowTransform func([]string) []string
type RollCallback func(path string, size int64, partRows int64) error type RollCallback func(path string, size int64, partRows int64) error
type ProgressCallback func(totalRows int64) error type ProgressCallback func(totalRows int64) error