refactor(exporter): 优化按时间跨度的分块计数策略
- 根据时间跨度自适应选择分块方式,提升查询效率 - ≤15天直接查询,避免无谓分片 - 15-30天按天分片(步长15天) - 30-90天按周分片(7天) - >90天新增按月分片(30天)策略 - 新增 SplitByMonths 函数实现月度分片逻辑 - 日志事件更新为按分片策略动态记录
This commit is contained in:
parent
db8abb437b
commit
bd38bdc374
|
|
@ -204,19 +204,28 @@ func CountRowsFastChunked(db *sql.DB, ds, main string, filters map[string]interf
|
||||||
}
|
}
|
||||||
daysDiff := int(en.Sub(st).Hours() / 24)
|
daysDiff := int(en.Sub(st).Hours() / 24)
|
||||||
|
|
||||||
// 如果时间跨度超过 15 天(半个月),使用按周分片(7天),否则使用按天分片(15天)
|
// 优化:根据时间跨度自适应分块策略
|
||||||
|
// ≤15天: 直接查询,≤30天: 按天分片,>30天: 按月分片(30天)
|
||||||
var ranges [][2]string
|
var ranges [][2]string
|
||||||
if daysDiff > 15 {
|
if daysDiff <= 15 {
|
||||||
|
// 15天内直接查询,不分片
|
||||||
|
return CountRowsFast(db, ds, main, filters)
|
||||||
|
} else if daysDiff <= 30 {
|
||||||
|
// 15-30天:按天分片(步长15)
|
||||||
|
ranges = SplitByDays(start, end, 15)
|
||||||
|
} else if daysDiff <= 90 {
|
||||||
|
// 30-90天:按周分片(7天)
|
||||||
ranges = SplitByWeeks(start, end)
|
ranges = SplitByWeeks(start, end)
|
||||||
|
} else {
|
||||||
|
// >90天:按月分片(30天)
|
||||||
|
ranges = SplitByMonths(start, end)
|
||||||
logging.JSON("INFO", map[string]interface{}{
|
logging.JSON("INFO", map[string]interface{}{
|
||||||
"event": "count_chunked_by_weeks",
|
"event": "count_chunked_by_months",
|
||||||
"datasource": ds,
|
"datasource": ds,
|
||||||
"main": main,
|
"main": main,
|
||||||
"days_diff": daysDiff,
|
"days_diff": daysDiff,
|
||||||
"chunks": len(ranges),
|
"chunks": len(ranges),
|
||||||
})
|
})
|
||||||
} else {
|
|
||||||
ranges = SplitByDays(start, end, 15)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var total int64
|
var total int64
|
||||||
|
|
@ -279,6 +288,31 @@ func SplitByWeeks(startStr, endStr string) [][2]string {
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SplitByMonths 按月(30天)分割时间范围,用于超長时间跨度的分块統計
|
||||||
|
func SplitByMonths(startStr, endStr string) [][2]string {
|
||||||
|
layout := "2006-01-02 15:04:05"
|
||||||
|
s := strings.TrimSpace(startStr)
|
||||||
|
e := strings.TrimSpace(endStr)
|
||||||
|
st, err1 := time.Parse(layout, s)
|
||||||
|
en, err2 := time.Parse(layout, e)
|
||||||
|
if err1 != nil || err2 != nil || !en.After(st) {
|
||||||
|
return [][2]string{{s, e}}
|
||||||
|
}
|
||||||
|
var out [][2]string
|
||||||
|
cur := st
|
||||||
|
// 按月分片
|
||||||
|
for cur.Before(en) {
|
||||||
|
// 下个月的第一天
|
||||||
|
nxt := time.Date(cur.Year(), cur.Month()+1, 1, 0, 0, 0, 0, cur.Location())
|
||||||
|
if nxt.After(en) {
|
||||||
|
nxt = en
|
||||||
|
}
|
||||||
|
out = append(out, [2]string{cur.Format(layout), nxt.Format(layout)})
|
||||||
|
cur = nxt
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
type RowTransform func([]string) []string
|
type RowTransform func([]string) []string
|
||||||
type RollCallback func(path string, size int64, partRows int64) error
|
type RollCallback func(path string, size int64, partRows int64) error
|
||||||
type ProgressCallback func(totalRows int64) error
|
type ProgressCallback func(totalRows int64) error
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue