fix(exports): 优化导出逻辑以支持动态时间范围分片
- 在 CountRowsFastChunked 函数中添加时间跨度计算,支持根据时间跨度选择按天或按周分片 - 新增 SplitByWeeks 函数以实现按周分割时间范围 - 增强日志记录,提供分片信息以便于后续分析和调试
This commit is contained in:
parent
ccb12da7f3
commit
61a3b17c09
|
|
@ -195,7 +195,30 @@ func CountRowsFastChunked(db *sql.DB, ds, main string, filters map[string]interf
|
|||
if start == "" || end == "" {
|
||||
return CountRowsFast(db, ds, main, filters)
|
||||
}
|
||||
ranges := SplitByDays(start, end, 15)
|
||||
// 计算时间跨度(天数)
|
||||
layout := "2006-01-02 15:04:05"
|
||||
st, err1 := time.Parse(layout, start)
|
||||
en, err2 := time.Parse(layout, end)
|
||||
if err1 != nil || err2 != nil {
|
||||
return CountRowsFast(db, ds, main, filters)
|
||||
}
|
||||
daysDiff := int(en.Sub(st).Hours() / 24)
|
||||
|
||||
// 如果时间跨度超过 15 天(半个月),使用按周分片(7天),否则使用按天分片(15天)
|
||||
var ranges [][2]string
|
||||
if daysDiff > 15 {
|
||||
ranges = SplitByWeeks(start, end)
|
||||
logging.JSON("INFO", map[string]interface{}{
|
||||
"event": "count_chunked_by_weeks",
|
||||
"datasource": ds,
|
||||
"main": main,
|
||||
"days_diff": daysDiff,
|
||||
"chunks": len(ranges),
|
||||
})
|
||||
} else {
|
||||
ranges = SplitByDays(start, end, 15)
|
||||
}
|
||||
|
||||
var total int64
|
||||
for _, rg := range ranges {
|
||||
fl := map[string]interface{}{}
|
||||
|
|
@ -232,6 +255,30 @@ func SplitByDays(startStr, endStr string, stepDays int) [][2]string {
|
|||
return out
|
||||
}
|
||||
|
||||
// SplitByWeeks 按周(7天)分割时间范围,返回多个时间区间
|
||||
func SplitByWeeks(startStr, endStr string) [][2]string {
|
||||
layout := "2006-01-02 15:04:05"
|
||||
s := strings.TrimSpace(startStr)
|
||||
e := strings.TrimSpace(endStr)
|
||||
st, err1 := time.Parse(layout, s)
|
||||
en, err2 := time.Parse(layout, e)
|
||||
if err1 != nil || err2 != nil || !en.After(st) {
|
||||
return [][2]string{{s, e}}
|
||||
}
|
||||
var out [][2]string
|
||||
cur := st
|
||||
weekDuration := 7 * 24 * time.Hour
|
||||
for cur.Before(en) {
|
||||
nxt := cur.Add(weekDuration)
|
||||
if nxt.After(en) {
|
||||
nxt = en
|
||||
}
|
||||
out = append(out, [2]string{cur.Format(layout), nxt.Format(layout)})
|
||||
cur = nxt
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type RowTransform func([]string) []string
|
||||
type RollCallback func(path string, size int64, partRows int64) error
|
||||
type ProgressCallback func(totalRows int64) error
|
||||
|
|
|
|||
Loading…
Reference in New Issue