refactor(exporter): 优化按时间跨度的分块计数策略
- 根据时间跨度自适应选择分块方式,提升查询效率 - ≤15天直接查询,避免无谓分片 - 15-30天按天分片(步长15天) - 30-90天按周分片(7天) - >90天新增按月分片(30天)策略 - 新增 SplitByMonths 函数实现月度分片逻辑 - 日志事件更新为按分片策略动态记录
This commit is contained in:
parent
db8abb437b
commit
bd38bdc374
|
|
@ -204,19 +204,28 @@ func CountRowsFastChunked(db *sql.DB, ds, main string, filters map[string]interf
|
|||
}
|
||||
daysDiff := int(en.Sub(st).Hours() / 24)
|
||||
|
||||
// 如果时间跨度超过 15 天(半个月),使用按周分片(7天),否则使用按天分片(15天)
|
||||
// 优化:根据时间跨度自适应分块策略
|
||||
// ≤15天: 直接查询,≤30天: 按天分片,>30天: 按月分片(30天)
|
||||
var ranges [][2]string
|
||||
if daysDiff > 15 {
|
||||
if daysDiff <= 15 {
|
||||
// 15天内直接查询,不分片
|
||||
return CountRowsFast(db, ds, main, filters)
|
||||
} else if daysDiff <= 30 {
|
||||
// 15-30天:按天分片(步长15)
|
||||
ranges = SplitByDays(start, end, 15)
|
||||
} else if daysDiff <= 90 {
|
||||
// 30-90天:按周分片(7天)
|
||||
ranges = SplitByWeeks(start, end)
|
||||
} else {
|
||||
// >90天:按月分片(30天)
|
||||
ranges = SplitByMonths(start, end)
|
||||
logging.JSON("INFO", map[string]interface{}{
|
||||
"event": "count_chunked_by_weeks",
|
||||
"event": "count_chunked_by_months",
|
||||
"datasource": ds,
|
||||
"main": main,
|
||||
"days_diff": daysDiff,
|
||||
"chunks": len(ranges),
|
||||
})
|
||||
} else {
|
||||
ranges = SplitByDays(start, end, 15)
|
||||
}
|
||||
|
||||
var total int64
|
||||
|
|
@ -279,6 +288,31 @@ func SplitByWeeks(startStr, endStr string) [][2]string {
|
|||
return out
|
||||
}
|
||||
|
||||
// SplitByMonths 按月(30天)分割时间范围,用于超長时间跨度的分块統計
|
||||
func SplitByMonths(startStr, endStr string) [][2]string {
|
||||
layout := "2006-01-02 15:04:05"
|
||||
s := strings.TrimSpace(startStr)
|
||||
e := strings.TrimSpace(endStr)
|
||||
st, err1 := time.Parse(layout, s)
|
||||
en, err2 := time.Parse(layout, e)
|
||||
if err1 != nil || err2 != nil || !en.After(st) {
|
||||
return [][2]string{{s, e}}
|
||||
}
|
||||
var out [][2]string
|
||||
cur := st
|
||||
// 按月分片
|
||||
for cur.Before(en) {
|
||||
// 下个月的第一天
|
||||
nxt := time.Date(cur.Year(), cur.Month()+1, 1, 0, 0, 0, 0, cur.Location())
|
||||
if nxt.After(en) {
|
||||
nxt = en
|
||||
}
|
||||
out = append(out, [2]string{cur.Format(layout), nxt.Format(layout)})
|
||||
cur = nxt
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type RowTransform func([]string) []string
|
||||
type RollCallback func(path string, size int64, partRows int64) error
|
||||
type ProgressCallback func(totalRows int64) error
|
||||
|
|
|
|||
Loading…
Reference in New Issue