From 6cec327340c9f0c86e22fa9cd4a68d02fd1eeca5 Mon Sep 17 00:00:00 2001
From: zhouyonggao <1971162852@qq.com>
Date: Fri, 19 Dec 2025 17:55:55 +0800
Subject: [PATCH] =?UTF-8?q?feat(export):=20=E4=BC=98=E5=8C=96=E5=AF=BC?=
 =?UTF-8?q?=E5=87=BA=E6=95=B0=E6=8D=AE=E5=A4=84=E7=90=86=E8=83=BD=E5=8A=9B?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- 将单个文件最大行数从30万提升至50万
- 新增海量数据集批次大小（500万+行）支持
- 将超大数据集批次大小由10万调整至20万
- 根据估算行数新增海量数据集批次选择逻辑
- 增加进度同步间隔至每10万行，减少同步频率
- 保持同步最长时间限制不变，优化性能表现
---
 server/internal/constants/constants.go | 22 ++++++++++++++--------
 server/internal/repo/export_repo.go    |  4 ++--
 2 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/server/internal/constants/constants.go b/server/internal/constants/constants.go
index ebcfc4b..974bf30 100644
--- a/server/internal/constants/constants.go
+++ b/server/internal/constants/constants.go
@@ -63,7 +63,7 @@ var ExportThresholds = struct {
 	// XlsxMaxRows xlsx格式最大行数，超过则强制使用csv
 	XlsxMaxRows int64
 }{
-	MaxRowsPerFile:         300000,
+	MaxRowsPerFile:         500000, // 修改：单个文件最大5万行数据
 	PassScoreThreshold:     60,
 	ChunkDays:              10,
 	ChunkThreshold:         50000,
@@ -85,16 +85,19 @@ var BatchSizes = struct {
 	LargeDataset int
 	// HugeDataset 超大数据集批次
 	HugeDataset int
+	// MassiveDataset 海量数据集批次 (用于 5000万+ 数捥)
+	MassiveDataset int
 }{
-	CSVDefault:    10000,
-	XLSXDefault:   5000,
-	SmallDataset:  10000,
-	MediumDataset: 20000,
-	LargeDataset:  50000,
-	HugeDataset:   100000,
+	CSVDefault:     10000,
+	XLSXDefault:    5000,
+	SmallDataset:   10000,
+	MediumDataset:  20000,
+	LargeDataset:   50000,
+	HugeDataset:    200000, // 上汪一一次
+	MassiveDataset: 500000, // 新增：海量数据集专用
 }
 
-// ChooseBatchSize 根据估算行数和格式选择合适的批次大小
+// ChooseBatchSize 根据估算行数和格式选择合適的批次大小
 func ChooseBatchSize(estimate int64, format FileFormat) int {
 	if format == FileFormatXLSX {
 		return BatchSizes.XLSXDefault
@@ -111,6 +114,9 @@ func ChooseBatchSize(estimate int64, format FileFormat) int {
 	if estimate < 500000 {
 		return BatchSizes.LargeDataset
 	}
+	if estimate >= 50000000 { // 5000万+ 数据
+		return BatchSizes.MassiveDataset
+	}
 	if estimate >= 2000000 {
 		return BatchSizes.HugeDataset
 	}
diff --git a/server/internal/repo/export_repo.go b/server/internal/repo/export_repo.go
index 467b63c..86ddc84 100644
--- a/server/internal/repo/export_repo.go
+++ b/server/internal/repo/export_repo.go
@@ -459,8 +459,8 @@ func NewProgressTracker(jobID uint64, metaDB *sql.DB) *ProgressTracker {
 		lastSyncRows: 0,
 		lastSyncTime: time.Now(),
 		metaDB:       metaDB,
-		syncInterval: 10000, // 每10000行同步一次
-		timeLimitMS:  5000,  // 最长5秒同步一次
+		syncInterval: 100000, // 每100000行同步一次
+		timeLimitMS:  5000,   // 最长5秒同步一次
 	}
 }