refactor(exporter): 优化按时间跨度的分块计数策略

- 根据时间跨度自适应选择分块方式,提升查询效率
- ≤15天直接查询,避免无谓分片
- 15-30天按天分片(步长15天)
- 30-90天按周分片(7天)
- >90天新增按月分片(30天)策略
- 新增 SplitByMonths 函数实现月度分片逻辑
- 日志事件更新为按分片策略动态记录
This commit is contained in:
zhouyonggao 2025-12-19 18:15:09 +08:00
parent db8abb437b
commit bd38bdc374
1 changed files with 41 additions and 7 deletions

View File

@ -204,19 +204,28 @@ func CountRowsFastChunked(db *sql.DB, ds, main string, filters map[string]interf
} }
daysDiff := int(en.Sub(st).Hours() / 24) daysDiff := int(en.Sub(st).Hours() / 24)
// 如果时间跨度超过 15 天半个月使用按周分片7天否则使用按天分片15天 // 优化:根据时间跨度自适应分块策略
// ≤15天: 直接查询≤30天: 按天分片,>30天: 按月分片30天
var ranges [][2]string var ranges [][2]string
if daysDiff > 15 { if daysDiff <= 15 {
// 15天内直接查询不分片
return CountRowsFast(db, ds, main, filters)
} else if daysDiff <= 30 {
// 15-30天按天分片步长15
ranges = SplitByDays(start, end, 15)
} else if daysDiff <= 90 {
// 30-90天按周分片7天
ranges = SplitByWeeks(start, end) ranges = SplitByWeeks(start, end)
} else {
// >90天按月分片30天
ranges = SplitByMonths(start, end)
logging.JSON("INFO", map[string]interface{}{ logging.JSON("INFO", map[string]interface{}{
"event": "count_chunked_by_weeks", "event": "count_chunked_by_months",
"datasource": ds, "datasource": ds,
"main": main, "main": main,
"days_diff": daysDiff, "days_diff": daysDiff,
"chunks": len(ranges), "chunks": len(ranges),
}) })
} else {
ranges = SplitByDays(start, end, 15)
} }
var total int64 var total int64
@ -279,6 +288,31 @@ func SplitByWeeks(startStr, endStr string) [][2]string {
return out return out
} }
// SplitByMonths 按月30天分割时间范围用于超長时间跨度的分块統計
func SplitByMonths(startStr, endStr string) [][2]string {
layout := "2006-01-02 15:04:05"
s := strings.TrimSpace(startStr)
e := strings.TrimSpace(endStr)
st, err1 := time.Parse(layout, s)
en, err2 := time.Parse(layout, e)
if err1 != nil || err2 != nil || !en.After(st) {
return [][2]string{{s, e}}
}
var out [][2]string
cur := st
// 按月分片
for cur.Before(en) {
// 下个月的第一天
nxt := time.Date(cur.Year(), cur.Month()+1, 1, 0, 0, 0, 0, cur.Location())
if nxt.After(en) {
nxt = en
}
out = append(out, [2]string{cur.Format(layout), nxt.Format(layout)})
cur = nxt
}
return out
}
type RowTransform func([]string) []string type RowTransform func([]string) []string
type RollCallback func(path string, size int64, partRows int64) error type RollCallback func(path string, size int64, partRows int64) error
type ProgressCallback func(totalRows int64) error type ProgressCallback func(totalRows int64) error