fix(exports): 优化导出逻辑以支持动态时间范围分片
- 在 CountRowsFastChunked 函数中添加时间跨度计算,支持根据时间跨度选择按天或按周分片 - 新增 SplitByWeeks 函数以实现按周分割时间范围 - 增强日志记录,提供分片信息以便于后续分析和调试
This commit is contained in:
parent
ccb12da7f3
commit
61a3b17c09
|
|
@ -195,7 +195,30 @@ func CountRowsFastChunked(db *sql.DB, ds, main string, filters map[string]interf
|
||||||
if start == "" || end == "" {
|
if start == "" || end == "" {
|
||||||
return CountRowsFast(db, ds, main, filters)
|
return CountRowsFast(db, ds, main, filters)
|
||||||
}
|
}
|
||||||
ranges := SplitByDays(start, end, 15)
|
// 计算时间跨度(天数)
|
||||||
|
layout := "2006-01-02 15:04:05"
|
||||||
|
st, err1 := time.Parse(layout, start)
|
||||||
|
en, err2 := time.Parse(layout, end)
|
||||||
|
if err1 != nil || err2 != nil {
|
||||||
|
return CountRowsFast(db, ds, main, filters)
|
||||||
|
}
|
||||||
|
daysDiff := int(en.Sub(st).Hours() / 24)
|
||||||
|
|
||||||
|
// 如果时间跨度超过 15 天(半个月),使用按周分片(7天),否则使用按天分片(15天)
|
||||||
|
var ranges [][2]string
|
||||||
|
if daysDiff > 15 {
|
||||||
|
ranges = SplitByWeeks(start, end)
|
||||||
|
logging.JSON("INFO", map[string]interface{}{
|
||||||
|
"event": "count_chunked_by_weeks",
|
||||||
|
"datasource": ds,
|
||||||
|
"main": main,
|
||||||
|
"days_diff": daysDiff,
|
||||||
|
"chunks": len(ranges),
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
ranges = SplitByDays(start, end, 15)
|
||||||
|
}
|
||||||
|
|
||||||
var total int64
|
var total int64
|
||||||
for _, rg := range ranges {
|
for _, rg := range ranges {
|
||||||
fl := map[string]interface{}{}
|
fl := map[string]interface{}{}
|
||||||
|
|
@ -232,6 +255,30 @@ func SplitByDays(startStr, endStr string, stepDays int) [][2]string {
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SplitByWeeks 按周(7天)分割时间范围,返回多个时间区间
|
||||||
|
func SplitByWeeks(startStr, endStr string) [][2]string {
|
||||||
|
layout := "2006-01-02 15:04:05"
|
||||||
|
s := strings.TrimSpace(startStr)
|
||||||
|
e := strings.TrimSpace(endStr)
|
||||||
|
st, err1 := time.Parse(layout, s)
|
||||||
|
en, err2 := time.Parse(layout, e)
|
||||||
|
if err1 != nil || err2 != nil || !en.After(st) {
|
||||||
|
return [][2]string{{s, e}}
|
||||||
|
}
|
||||||
|
var out [][2]string
|
||||||
|
cur := st
|
||||||
|
weekDuration := 7 * 24 * time.Hour
|
||||||
|
for cur.Before(en) {
|
||||||
|
nxt := cur.Add(weekDuration)
|
||||||
|
if nxt.After(en) {
|
||||||
|
nxt = en
|
||||||
|
}
|
||||||
|
out = append(out, [2]string{cur.Format(layout), nxt.Format(layout)})
|
||||||
|
cur = nxt
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
type RowTransform func([]string) []string
|
type RowTransform func([]string) []string
|
||||||
type RollCallback func(path string, size int64, partRows int64) error
|
type RollCallback func(path string, size int64, partRows int64) error
|
||||||
type ProgressCallback func(totalRows int64) error
|
type ProgressCallback func(totalRows int64) error
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue