From d094dbd5ffac56f67d026498343e6849d5c3a6ab Mon Sep 17 00:00:00 2001 From: ziming Date: Tue, 10 Jun 2025 10:17:16 +0800 Subject: [PATCH 01/36] query --- internal/pkg/rdsmq/rdsmq.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/pkg/rdsmq/rdsmq.go b/internal/pkg/rdsmq/rdsmq.go index 3ef5b7c..6712883 100644 --- a/internal/pkg/rdsmq/rdsmq.go +++ b/internal/pkg/rdsmq/rdsmq.go @@ -89,11 +89,13 @@ func (r *ConsumeConfig) Start(ctx context.Context) { } func (r *ConsumeConfig) consumer(ctx context.Context, queueName string, value string) { - if err := r.Fn(ctx, value); err == nil { - r.Logger.Errorf("BLPop on %s Failed to process message [%s] after retry %d times, err[%v]\n", queueName, value, r.RetryNum, err) + err := r.Fn(ctx, value) + + if err == nil { return } + r.Logger.Errorf("BLPop on %s Failed to process message [%s] after retry %d times, err[%v]\n", queueName, value, r.RetryNum, err) if r.RetryNum > 0 { r.retry(ctx, queueName, value) } From 68b95dc87c6dd74a652971e34623658c9c53e16f Mon Sep 17 00:00:00 2001 From: ziming Date: Tue, 10 Jun 2025 10:39:17 +0800 Subject: [PATCH 02/36] query --- internal/biz/wechat_notify.go | 7 +------ internal/biz/wechat_query.go | 20 +++++++++++--------- internal/pkg/helper/utils_test.go | 16 ++++++++++++++++ 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/internal/biz/wechat_notify.go b/internal/biz/wechat_notify.go index 833ca14..8c288f4 100644 --- a/internal/biz/wechat_notify.go +++ b/internal/biz/wechat_notify.go @@ -118,12 +118,7 @@ func (v *VoucherBiz) expired(ctx context.Context, order *bo.OrderBo) error { func (v *VoucherBiz) notify(ctx context.Context, order *bo.OrderBo) error { - if order.Type.IsCmb() { - - return v.cmbNotify(ctx, order.ID) - } - - return fmt.Errorf("未知渠道订单类型:%s", order.Type.GetText()) + return v.cmbNotify(ctx, order.ID) } func (v *VoucherBiz) cmbNotify(ctx context.Context, orderId uint64) error { diff --git a/internal/biz/wechat_query.go b/internal/biz/wechat_query.go index a7711aa..37db077 100644 --- a/internal/biz/wechat_query.go +++ b/internal/biz/wechat_query.go @@ -63,31 +63,32 @@ func (v *VoucherBiz) WechatQuery(ctx context.Context, msg string) error { n := 0 num := 0 + notifyNum := 0 err := v.OrderRepo.FinSucByStockIdInBatches(ctx, req, func(ctx context.Context, rows []*bo.OrderBo) error { n += 1 for _, order := range rows { num += 1 - if err := v.wechatQuery(ctx, order); err != nil { + if err := v.wechatQuery(ctx, order, ¬ifyNum); err != nil { log.Errorf("微信查询券订单状态发生错误,msg:%s,orderNo:%s,couponId:%s,appId:%s,openId:%s,err:%v", msg, order.OrderNo, order.VoucherNo, order.AppID, order.Account, err) } } - log.Warnf("微信券查询处理第:%d组,已执行条数:%d,执行开始时间:%s，已耗时:%s", n, num, startStr, time.Now().Sub(start).String()) + log.Warnf("微信券查询处理第:%d组,已执行条数:%d,核销通知条数:%d,执行开始时间:%s，已耗时:%s", n, num, notifyNum, startStr, time.Now().Sub(start).String()) return nil }) - log.Warnf("微信券查询处理耗时:%s,处理%d组,处理%d单,msg:%s", time.Now().Sub(start).String(), n, num, msg) - fmt.Printf("微信券查询处理耗时:%s,处理%d组,处理%d单,msg:%s", time.Now().Sub(start).String(), n, num, msg) + log.Warnf("微信券查询处理耗时:%s,处理%d组,处理%d单,核销通知条数:%d,msg:%s", time.Now().Sub(start).String(), n, num, notifyNum, msg) + fmt.Printf("微信券查询处理耗时:%s,处理%d组,处理%d单,核销通知条数:%d,msg:%s", time.Now().Sub(start).String(), n, num, notifyNum, msg) return err } -func (v *VoucherBiz) wechatQuery(ctx context.Context, order *bo.OrderBo) error { +func (v *VoucherBiz) wechatQuery(ctx context.Context, order *bo.OrderBo, notifyNum *int) error { status, err := v.WechatCpnRepo.Query(ctx, order) if err != nil { @@ -95,7 +96,7 @@ func (v *VoucherBiz) wechatQuery(ctx context.Context, order *bo.OrderBo) error { } if status.IsUse() { - return v.queryUsed(ctx, order) + return v.queryUsed(ctx, order, notifyNum) } else if status.IsExpired() { return v.expired(ctx, order) } @@ -103,11 +104,12 @@ func (v *VoucherBiz) wechatQuery(ctx context.Context, order *bo.OrderBo) error { return nil } -func (v *VoucherBiz) queryUsed(ctx context.Context, order *bo.OrderBo) error { +func (v *VoucherBiz) queryUsed(ctx context.Context, order *bo.OrderBo, notifyNum *int) error { + + *notifyNum += 1 if order.Status.IsUse() { - log.Warnf("券状态已是已使用，忽略不处理,orderNo:%s", order.OrderNo) - return nil + return v.notify(ctx, order) } if err := v.OrderRepo.Used(ctx, order.ID); err != nil { diff --git a/internal/pkg/helper/utils_test.go b/internal/pkg/helper/utils_test.go index 8f14839..e596e66 100644 --- a/internal/pkg/helper/utils_test.go +++ b/internal/pkg/helper/utils_test.go @@ -25,3 +25,19 @@ func TestNoticeTime(t *testing.T) { t.Logf("startTime:%s,endTime:%s", startTime, endTime) } + +func TestNum(t *testing.T) { + useNum := 0 + used(&useNum) + t.Log(useNum) +} + +func used(useNum *int) { + queryUsed(useNum) + queryUsed(useNum) + *useNum += 1 +} + +func queryUsed(useNum *int) { + *useNum += 1 +} From f418a277c321433e42b6294f1a3dcb3d59309cbc Mon Sep 17 00:00:00 2001 From: ziming Date: Tue, 10 Jun 2025 14:07:01 +0800 Subject: [PATCH 03/36] query --- internal/biz/wechat_query.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/biz/wechat_query.go b/internal/biz/wechat_query.go index 37db077..284dfe9 100644 --- a/internal/biz/wechat_query.go +++ b/internal/biz/wechat_query.go @@ -77,13 +77,15 @@ func (v *VoucherBiz) WechatQuery(ctx context.Context, msg string) error { } - log.Warnf("微信券查询处理第:%d组,已执行条数:%d,核销通知条数:%d,执行开始时间:%s，已耗时:%s", n, num, notifyNum, startStr, time.Now().Sub(start).String()) + groupTime := time.Now() + log.Warnf("微信券查询处理第:%d组,已执行条数:%d,核销通知条数:%d,执行开始时间:%s，当前时间:%s,已耗时:%s", n, num, notifyNum, startStr, groupTime.String(), groupTime.Sub(start).String()) return nil }) - log.Warnf("微信券查询处理耗时:%s,处理%d组,处理%d单,核销通知条数:%d,msg:%s", time.Now().Sub(start).String(), n, num, notifyNum, msg) - fmt.Printf("微信券查询处理耗时:%s,处理%d组,处理%d单,核销通知条数:%d,msg:%s", time.Now().Sub(start).String(), n, num, notifyNum, msg) + endTime := time.Now() + log.Warnf("微信券查询处理耗时:%s,结束时间:%s,处理%d组,处理%d单,核销通知条数:%d,msg:%s", endTime.Sub(start).String(), endTime.String(), n, num, notifyNum, msg) + fmt.Printf("微信券查询处理耗时:%s,结束时间%s,处理%d组,处理%d单,核销通知条数:%d,msg:%s", endTime.Sub(start).String(), endTime.String(), n, num, notifyNum, msg) return err } From 87222148ec8506f3fa866af9a5a84d776ed2862e Mon Sep 17 00:00:00 2001 From: ziming Date: Tue, 10 Jun 2025 19:01:15 +0800 Subject: [PATCH 04/36] query --- go.mod | 2 + go.sum | 4 + internal/biz/timeslice/manager.go | 104 +++++++++++++++++++++++++ internal/biz/timeslice/manager_test.go | 78 +++++++++++++++++++ internal/biz/timeslice/model.go | 24 ++++++ internal/biz/voucher.go | 4 +- internal/biz/wechat_query.go | 24 ++++-- internal/biz/wechatrepo/cpn.go | 1 + internal/data/wechatrepoimpl/cpn.go | 28 +++++++ internal/pkg/helper/utils.go | 13 ++++ internal/pkg/helper/utils_test.go | 5 ++ internal/pkg/script/script.go | 66 ++++++++++++++++ internal/pkg/script/script_test.go | 44 +++++++++++ 13 files changed, 387 insertions(+), 10 deletions(-) create mode 100644 internal/biz/timeslice/manager.go create mode 100644 internal/biz/timeslice/manager_test.go create mode 100644 internal/biz/timeslice/model.go create mode 100644 internal/pkg/script/script.go create mode 100644 internal/pkg/script/script_test.go diff --git a/go.mod b/go.mod index cfddba6..34d68d0 100644 --- a/go.mod +++ b/go.mod @@ -63,6 +63,8 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/mux v1.8.1 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af // indirect diff --git a/go.sum b/go.sum index 80fa89b..8561a05 100644 --- a/go.sum +++ b/go.sum @@ -150,6 +150,10 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grokify/html-strip-tags-go v0.0.1 h1:0fThFwLbW7P/kOiTBs03FsJSV9RM2M/Q/MOnCQxKMo0= github.com/grokify/html-strip-tags-go v0.0.1/go.mod h1:2Su6romC5/1VXOQMaWL2yb618ARB8iVo6/DR99A6d78= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= diff --git a/internal/biz/timeslice/manager.go b/internal/biz/timeslice/manager.go new file mode 100644 index 0000000..c908e44 --- /dev/null +++ b/internal/biz/timeslice/manager.go @@ -0,0 +1,104 @@ +package timeslice + +import ( + "context" + "fmt" + "github.com/hashicorp/go-multierror" + "golang.org/x/sync/errgroup" + "time" +) + +type ManagerSrv struct { + callback func(ctx context.Context, req *Task) error +} + +func NewManager(callback func(ctx context.Context, req *Task) error) *ManagerSrv { + return &ManagerSrv{callback: callback} +} + +func (m *ManagerSrv) Run(ctx context.Context, req *Manager) (int, error) { + + if req.StartTime.After(req.EndTime) { + return 0, fmt.Errorf("start_time不能大于end_time") + } + + totalHours := req.EndTime.Sub(req.StartTime).Hours() + taskCount := int(totalHours / 2) + + // 如果剩余时间不足2小时，增加任务数 + if totalHours-float64(taskCount)*float64(2) > 0 { + taskCount++ + } + + processReq := &Process{ + manager: req, + taskCount: taskCount, + } + + return taskCount, m.process(ctx, processReq) +} + +func (m *ManagerSrv) process(ctx context.Context, req *Process) error { + + if req.taskCount == 0 { + return fmt.Errorf("该时间范围无可执行任务次数，请检查时间范围") + } + + if req.manager.GoNum == 0 { + return fmt.Errorf("协程数量不能为0") + } + if req.manager.GoNum > 100 { + return fmt.Errorf("协程数量不能大于100") + } + + // 设置最大并发任务数为 5 + eg := new(errgroup.Group) + eg.SetLimit(req.manager.GoNum) + errs := make([]error, 0) // 用于存储所有错误 + + // 为每个任务分配开始和结束时间 + for i := 0; i < req.taskCount; i++ { + + currentStart := req.manager.StartTime.Add(time.Duration(i) * 2 * time.Hour) + currentEnd := currentStart.Add(2 * time.Hour) + if currentEnd.After(req.manager.EndTime) { + currentEnd = req.manager.EndTime + } + + eg.Go(func() error { + + defer func() { + if err := recover(); err != nil { + errs = append(errs, fmt.Errorf("panic: %v", err)) + } + }() + + taskID := i + 1 + taskReq := &Task{ + CurrentStartTime: currentStart, + CurrentEndTime: currentEnd, + TaskID: taskID, + ProductNo: req.manager.ProductNo, + } + + if err := m.callback(ctx, taskReq); err != nil { + errs = append(errs, err) + return err + } + + return nil + }) + } + + // 等待所有任务完成 + _ = eg.Wait() + + var result error + + // 收集错误 + for _, err2 := range errs { + result = multierror.Append(result, err2) + } + + return result +} diff --git a/internal/biz/timeslice/manager_test.go b/internal/biz/timeslice/manager_test.go new file mode 100644 index 0000000..0b73271 --- /dev/null +++ b/internal/biz/timeslice/manager_test.go @@ -0,0 +1,78 @@ +package timeslice + +import ( + "context" + "fmt" + "golang.org/x/sync/errgroup" + "math/rand" + "testing" + "time" +) + +func ProcessTasks() error { + eg := new(errgroup.Group) + eg.SetLimit(5) + + for i := 0; i < 5; i++ { + eg.Go(func() error { + // 任务逻辑... + time.Sleep(100 * time.Millisecond) + + return fmt.Errorf("任务失败") + }) + } + + return eg.Wait() // 仅返回第一个错误 +} + +func TestNewManager(t *testing.T) { + + // 解析起始时间和结束时间 + start, err := time.Parse(time.DateTime, "2023-01-01 00:00:00") + if err != nil { + t.Fatalf("查询失败: %v", err) + return + } + + //end, err := time.Parse(time.DateTime, "2023-01-31 02:00:01") + end, err := time.Parse(time.DateTime, "2023-01-02 02:00:01") + if err != nil { + t.Fatalf("查询失败: %v", err) + return + } + + var results []string + callback := func(ctx context.Context, req *Task) error { + // 模拟任务执行，休眠随机时间 + time.Sleep(time.Duration(rand.Intn(5)) * time.Second) + // 生成任务执行结果 + result := fmt.Sprintf("任务批次 %d-%s: %s 至 %s 处理完成", req.TaskID, req.ProductNo, req.CurrentStartTime.Format(time.DateTime), req.CurrentEndTime.Format(time.DateTime)) + results = append(results, result) + //return nil + return fmt.Errorf("任务执行失败:%d-%s,时间%s-%s", req.TaskID, req.ProductNo, req.CurrentStartTime.Format(time.DateTime), req.CurrentEndTime.Format(time.DateTime)) + } + + startTime := time.Now() + startStr := time.Now().String() + + srv := NewManager(callback) + taskCount, err := srv.Run(context.Background(), &Manager{ + StartTime: start, + EndTime: end, + ProductNo: "123456", + GoNum: 2, + }) + + // 输出结果 + fmt.Printf("总任务数:%d\n", taskCount) + for _, result := range results { + fmt.Printf("%v\n", result) + } + + if err != nil { + t.Error(err) + } + + endTime := time.Now() + fmt.Printf("处理耗时:%s,开始处理时间:%s,结束时间%s", startStr, endTime.Sub(startTime).String(), endTime.String()) +} diff --git a/internal/biz/timeslice/model.go b/internal/biz/timeslice/model.go new file mode 100644 index 0000000..ca4b4a5 --- /dev/null +++ b/internal/biz/timeslice/model.go @@ -0,0 +1,24 @@ +package timeslice + +import ( + "time" +) + +type Manager struct { + StartTime time.Time + EndTime time.Time + ProductNo string + GoNum int +} + +type Process struct { + manager *Manager + taskCount int +} + +type Task struct { + CurrentStartTime time.Time + CurrentEndTime time.Time + TaskID int + ProductNo string +} diff --git a/internal/biz/voucher.go b/internal/biz/voucher.go index 400a505..7c92587 100644 --- a/internal/biz/voucher.go +++ b/internal/biz/voucher.go @@ -60,11 +60,11 @@ func NewVoucherBiz( } } -func (this *VoucherBiz) Get(stockNo string) bool { +func (this *VoucherBiz) Get(uid string) bool { this.mu.Lock() defer this.mu.Unlock() - if _, ok := this.queryMap[stockNo]; ok { + if _, ok := this.queryMap[uid]; ok { return ok } diff --git a/internal/biz/wechat_query.go b/internal/biz/wechat_query.go index 284dfe9..f389912 100644 --- a/internal/biz/wechat_query.go +++ b/internal/biz/wechat_query.go @@ -6,16 +6,17 @@ import ( "fmt" "github.com/go-kratos/kratos/v2/log" "github.com/go-kratos/kratos/v2/transport/http" + "github.com/nacos-group/nacos-sdk-go/util" "time" "voucher/internal/biz/bo" "voucher/internal/biz/do" ) -func (v *VoucherBiz) PushWechatQuery(ctx http.Context, req *do.WechatQuery) error { +func (v *VoucherBiz) uid(_ context.Context, msg string) string { + return util.Md5(msg) +} - if v.Get("CMB_WECHAT_QUERY") { - return fmt.Errorf("此台服务队列正在处理中,ip:%s", ctx.Header().Get("X-Forwarded-For")) - } +func (v *VoucherBiz) PushWechatQuery(ctx http.Context, req *do.WechatQuery) error { if req.ProductNo != "" { _, err := v.ProductRepo.GetByProductNo(ctx, req.ProductNo) @@ -34,11 +35,18 @@ func (v *VoucherBiz) PushWechatQuery(ctx http.Context, req *do.WechatQuery) erro return err } - v.Add("CMB_WECHAT_QUERY") + strMsg := string(msg) - _, err = v.rdb.Rdb.RPush(ctx, queue.Name, msg).Result() + uid := v.uid(ctx, strMsg) + if v.Get(uid) { + return fmt.Errorf("此台服务队列正在处理中,key:%s,ip:%s", uid, ctx.Header().Get("X-Forwarded-For")) + } + + v.Add(uid) + + _, err = v.rdb.Rdb.RPush(ctx, queue.Name, strMsg).Result() if err != nil { - v.Remove("CMB_WECHAT_QUERY") + v.Remove(uid) return fmt.Errorf("添加到队列失败:%v", err) } @@ -47,7 +55,7 @@ func (v *VoucherBiz) PushWechatQuery(ctx http.Context, req *do.WechatQuery) erro func (v *VoucherBiz) WechatQuery(ctx context.Context, msg string) error { - defer v.Remove("CMB_WECHAT_QUERY") + defer v.Remove(v.uid(ctx, msg)) var req *do.WechatQuery diff --git a/internal/biz/wechatrepo/cpn.go b/internal/biz/wechatrepo/cpn.go index 2813652..20adf29 100644 --- a/internal/biz/wechatrepo/cpn.go +++ b/internal/biz/wechatrepo/cpn.go @@ -10,6 +10,7 @@ import ( type WechatCpnRepo interface { Order(ctx context.Context, order *bo.OrderBo) (couponId string, err error) Query(ctx context.Context, order *bo.OrderBo) (vo.OrderStatus, error) + QueryCoupon(ctx context.Context, orderWechat *bo.OrderBo) (*cashcoupons.Coupon, error) QueryProduct(ctx context.Context, stockCreatorMchId, stockId string) (*cashcoupons.Stock, error) QueryCallback(ctx context.Context) (*cashcoupons.Callback, error) SetCallback(ctx context.Context, url string) (*cashcoupons.SetCallbackResponse, error) diff --git a/internal/data/wechatrepoimpl/cpn.go b/internal/data/wechatrepoimpl/cpn.go index 0bc9ea8..dd0cbdd 100644 --- a/internal/data/wechatrepoimpl/cpn.go +++ b/internal/data/wechatrepoimpl/cpn.go @@ -126,6 +126,34 @@ func (c *CpnRepoImpl) Query(ctx context.Context, orderWechat *bo.OrderBo) (vo.Or return CpnStatus(*resp.Status).GetStatus() } +func (c *CpnRepoImpl) QueryCoupon(ctx context.Context, orderWechat *bo.OrderBo) (*cashcoupons.Coupon, error) { + + req := cashcoupons.QueryCouponRequest{ + CouponId: core.String(orderWechat.VoucherNo), + Appid: core.String(orderWechat.AppID), + Openid: core.String(orderWechat.Account), + } + + client, err := c.GetClient(ctx) + if err != nil { + return nil, err + } + + svc := cashcoupons.CouponApiService{Client: client} + + resp, result, err := svc.QueryCoupon(ctx, req) + if err != nil { + + if result.Response != nil && result.Response.Body != nil { + return nil, c.bodyErr(ctx, result) + } + + return nil, err + } + + return resp, nil +} + func (c *CpnRepoImpl) QueryProduct(ctx context.Context, stockCreatorMchId, stockId string) (*cashcoupons.Stock, error) { if stockCreatorMchId == "" || stockId == "" { diff --git a/internal/pkg/helper/utils.go b/internal/pkg/helper/utils.go index 7677cd6..bd9f033 100644 --- a/internal/pkg/helper/utils.go +++ b/internal/pkg/helper/utils.go @@ -1,6 +1,8 @@ package helper import ( + "crypto/md5" + "encoding/hex" "hash/fnv" "math" "os" @@ -18,3 +20,14 @@ func HashMod(hashStr string) int { hashValue := hash.Sum32() return int(math.Mod(float64(hashValue), 32)) } + +func Md5(str string) string { + // 创建一个 MD5 哈希对象 + hash := md5.New() + // 写入待加密的数据 + hash.Write([]byte(str)) + // 获取 MD5 哈希值 + hashBytes := hash.Sum(nil) + // 将 MD5 哈希值转换为16进制字符串 + return hex.EncodeToString(hashBytes) +} diff --git a/internal/pkg/helper/utils_test.go b/internal/pkg/helper/utils_test.go index e596e66..b8d2096 100644 --- a/internal/pkg/helper/utils_test.go +++ b/internal/pkg/helper/utils_test.go @@ -41,3 +41,8 @@ func used(useNum *int) { func queryUsed(useNum *int) { *useNum += 1 } + +func TestMd5(t *testing.T) { + s := Md5(`{"product_no":"","start_time":"2025-04-20 09:00:00","end_time":"2025-05-01 00:00:00"}`) + t.Log(s) +} diff --git a/internal/pkg/script/script.go b/internal/pkg/script/script.go new file mode 100644 index 0000000..cf06e4e --- /dev/null +++ b/internal/pkg/script/script.go @@ -0,0 +1,66 @@ +package script + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "time" +) + +func script(startTime, endTime time.Time, duration time.Duration) error { + + // 每指定间隔时间发送一次请求 + for t := startTime; t.Before(endTime); t = t.Add(duration) { + end := t.Add(duration) // 计算每次请求的结束时间 + + // 发送请求 + if err := sendRequest(t, end); err != nil { + fmt.Printf("Error sending request: %v\n", err) + } + + // 等待一段时间后再发送下一个请求 + time.Sleep(1 * time.Second) // 可以根据需要调整间隔时间 + } + + return nil +} + +func sendRequest(startTime, endTime time.Time) error { + + url := "https://gateway.dev.cdlsxd.cn/voucher/cmb/v1/orderQuery" + + // 创建请求体 + requestBody := map[string]interface{}{ + "product_no": "", + "start_time": startTime.Format(time.DateTime), + "end_time": endTime.Format(time.DateTime), + } + + // 将请求体转换为 JSON 格式 + jsonData, err := json.Marshal(requestBody) + if err != nil { + return fmt.Errorf("failed to marshal JSON: %v", err) + } + + // 发送 POST 请求 + resp, err := http.Post(url, "application/json", bytes.NewBuffer(jsonData)) + if err != nil { + return fmt.Errorf("failed to send POST request: %v", err) + } + defer resp.Body.Close() + + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("读取响应体失败: %w", err) + } + + if resp.StatusCode == http.StatusOK { + fmt.Printf("Request sent successfully,body:%s", string(bodyBytes)) + } else { + return fmt.Errorf("failed with status code: %d", resp.StatusCode) + } + + return nil +} diff --git a/internal/pkg/script/script_test.go b/internal/pkg/script/script_test.go new file mode 100644 index 0000000..a557e56 --- /dev/null +++ b/internal/pkg/script/script_test.go @@ -0,0 +1,44 @@ +package script + +import ( + "testing" + "time" +) + +func Test_script(t *testing.T) { + + startTime, err := time.Parse(time.DateTime, "2025-05-01 00:00:00") + if err != nil { + t.Error(err) + return + } + + endTime, err := time.Parse(time.DateTime, "2025-05-31 00:00:00") + if err != nil { + t.Error(err) + return + } + + duration := 24 * time.Hour + + if err = script(startTime, endTime, duration); err != nil { + t.Error(err) + } +} + +func Test_script2(t *testing.T) { + + startTime, err := time.Parse(time.DateTime, "2025-05-31 00:00:00") + if err != nil { + t.Error(err) + return + } + + endTime := time.Now() + + duration := 1 * time.Hour + + if err = script(startTime, endTime, duration); err != nil { + t.Error(err) + } +} From ac009695973cc23eb52205e0811ae4c3f8f1f647 Mon Sep 17 00:00:00 2001 From: ziming Date: Wed, 11 Jun 2025 14:27:10 +0800 Subject: [PATCH 05/36] query --- internal/biz/timeslice/manager.go | 11 +- internal/biz/timeslice/manager_test.go | 167 ++++++++++++++++++++++++- internal/biz/timeslice/model.go | 27 +++- 3 files changed, 194 insertions(+), 11 deletions(-) diff --git a/internal/biz/timeslice/manager.go b/internal/biz/timeslice/manager.go index c908e44..aa7e0c6 100644 --- a/internal/biz/timeslice/manager.go +++ b/internal/biz/timeslice/manager.go @@ -54,6 +54,7 @@ func (m *ManagerSrv) process(ctx context.Context, req *Process) error { // 设置最大并发任务数为 5 eg := new(errgroup.Group) eg.SetLimit(req.manager.GoNum) + errs := make([]error, 0) // 用于存储所有错误 // 为每个任务分配开始和结束时间 @@ -61,9 +62,11 @@ func (m *ManagerSrv) process(ctx context.Context, req *Process) error { currentStart := req.manager.StartTime.Add(time.Duration(i) * 2 * time.Hour) currentEnd := currentStart.Add(2 * time.Hour) + if currentEnd.After(req.manager.EndTime) { currentEnd = req.manager.EndTime } + taskID := i + 1 eg.Go(func() error { @@ -73,17 +76,15 @@ func (m *ManagerSrv) process(ctx context.Context, req *Process) error { } }() - taskID := i + 1 taskReq := &Task{ CurrentStartTime: currentStart, CurrentEndTime: currentEnd, TaskID: taskID, - ProductNo: req.manager.ProductNo, + Process: req, } if err := m.callback(ctx, taskReq); err != nil { errs = append(errs, err) - return err } return nil @@ -91,7 +92,9 @@ func (m *ManagerSrv) process(ctx context.Context, req *Process) error { } // 等待所有任务完成 - _ = eg.Wait() + if err := eg.Wait(); err != nil { + return fmt.Errorf("任务执行失败: %v", err) + } var result error diff --git a/internal/biz/timeslice/manager_test.go b/internal/biz/timeslice/manager_test.go index 0b73271..2864dc9 100644 --- a/internal/biz/timeslice/manager_test.go +++ b/internal/biz/timeslice/manager_test.go @@ -2,9 +2,12 @@ package timeslice import ( "context" + "errors" "fmt" + "github.com/hashicorp/go-multierror" "golang.org/x/sync/errgroup" "math/rand" + "sync" "testing" "time" ) @@ -44,12 +47,12 @@ func TestNewManager(t *testing.T) { var results []string callback := func(ctx context.Context, req *Task) error { // 模拟任务执行，休眠随机时间 - time.Sleep(time.Duration(rand.Intn(5)) * time.Second) + time.Sleep(time.Duration(rand.Intn(3)) * time.Second) // 生成任务执行结果 - result := fmt.Sprintf("任务批次 %d-%s: %s 至 %s 处理完成", req.TaskID, req.ProductNo, req.CurrentStartTime.Format(time.DateTime), req.CurrentEndTime.Format(time.DateTime)) + result := fmt.Sprintf("任务批次 %d-%s: %s 至 %s 处理完成", req.TaskID, req.Process.manager.ProductNo, req.CurrentStartTime.Format(time.DateTime), req.CurrentEndTime.Format(time.DateTime)) results = append(results, result) //return nil - return fmt.Errorf("任务执行失败:%d-%s,时间%s-%s", req.TaskID, req.ProductNo, req.CurrentStartTime.Format(time.DateTime), req.CurrentEndTime.Format(time.DateTime)) + return fmt.Errorf("任务执行失败:%d-%s,时间%s-%s", req.TaskID, req.Process.manager.ProductNo, req.CurrentStartTime.Format(time.DateTime), req.CurrentEndTime.Format(time.DateTime)) } startTime := time.Now() @@ -59,7 +62,7 @@ func TestNewManager(t *testing.T) { taskCount, err := srv.Run(context.Background(), &Manager{ StartTime: start, EndTime: end, - ProductNo: "123456", + ProductNo: "no123456", GoNum: 2, }) @@ -70,9 +73,161 @@ func TestNewManager(t *testing.T) { } if err != nil { - t.Error(err) + multiErr, ok := err.(*multierror.Error) + if ok { + for i, err := range multiErr.Errors { + fmt.Printf("错误 %d: %v\n", i+1, err) + } + } else { + // 不是多错误，单独处理 + t.Error("单个错误", err) + } } endTime := time.Now() - fmt.Printf("处理耗时:%s,开始处理时间:%s,结束时间%s", startStr, endTime.Sub(startTime).String(), endTime.String()) + fmt.Printf("处理耗时:%s,开始处理时间:%s,结束时间%s\n", startStr, endTime.Sub(startTime).String(), endTime.String()) +} + +func TestBatchCallBackFunc(t *testing.T) { + // 初始化开始时间和结束时间 + start, err := time.Parse(time.DateTime, "2025-01-01 01:00:00") + if err != nil { + t.Fatalf("%v", err) + return + } + end, err := time.Parse(time.DateTime, "2025-01-02 02:00:01") + if err != nil { + t.Fatalf("%v", err) + return + } + + // 确保开始时间小于结束时间 + if start.After(end) { + t.Fatalf("start_time不能大于end_time") + return + } + + var wg sync.WaitGroup + // 按照每天的时间单位分片 + duration := 24 * time.Hour // 每个时间段为一天 + + // 循环处理时间段 + for current := start; current.Before(end); current = current.Add(duration) { + // 计算当前时间段的结束时间 + next := current.Add(duration) + + // 如果下一个结束时间超过了实际的结束时间，则调整为实际结束时间 + if next.After(end) { + next = end + } + + t.Logf("处理时间: %s到%s", current.Format(time.DateTime), next.Format(time.DateTime)) + + wg.Add(1) + // 启动goroutine处理每个时间段 + go func(startTime, endTime time.Time) { + defer func() { + wg.Done() + if err2 := recover(); err2 != nil { + t.Error("panic", err2) + } + }() + if err3 := CallbackFunc(startTime, endTime); err3 != nil { + t.Errorf("任务执行失败:%v\n", err3) + } + }(current, next) + } + + // 等待所有的 goroutine 完成 + wg.Wait() +} + +func CallbackFunc(start, end time.Time) error { + + nowTime := time.Now() + + managerStartStr := start.Format(time.DateTime) + managerEndStr := end.Format(time.DateTime) + + req := &Manager{ + StartTime: start, + EndTime: end, + ProductNo: "no123456", + GoNum: 2, + } + + taskCount, err := NewManager(callbackFunc).Run(context.Background(), req) + if err != nil { + var multiErr *multierror.Error + if errors.As(err, &multiErr) { + for i, err := range multiErr.Errors { + fmt.Printf("%s到%s,错误 %d: %v\n", managerStartStr, managerEndStr, i+1, err) + } + } else { + // 不是多错误，单独处理 + fmt.Printf("单个错误%v", err) + } + } + + fmt.Printf("%s到%s,总任务数:%d\n", managerStartStr, managerEndStr, taskCount) + + endTime := time.Now() + fmt.Printf("%s到%s,处理耗时:%s,开始处理时间:%s,结束时间%s\n", managerStartStr, managerEndStr, endTime.Sub(nowTime).String(), nowTime.Format(time.DateTime), endTime.Format(time.DateTime)) + + return nil +} + +func callbackFunc(_ context.Context, req *Task) error { + + managerStartTimeStr := req.Process.manager.StartTime.Format(time.DateTime) + managerEndTimeStr := req.Process.manager.EndTime.Format(time.DateTime) + + currentStartTimeStr := req.CurrentStartTime.Format(time.DateTime) + currentEndTimeStr := req.CurrentEndTime.Format(time.DateTime) + + start := time.Now() + startStr := start.Format(time.DateTime) + + n := 0 + num := 0 + notifyNum := 0 + + for i := 0; i < 3; i++ { + + groupStartTime := time.Now() + // 模拟任务执行，休眠随机时间 + time.Sleep(time.Duration(rand.Intn(3)) * time.Second) + + n += 1 + num = num + (i+1)*100 + + logFields := map[string]interface{}{ + "处理条数": num, + "通知条数": notifyNum, + "耗时": time.Now().Sub(groupStartTime).String(), + "任务处理开始时间": currentStartTimeStr, + "任务处理结束时间": currentEndTimeStr, + } + fmt.Printf("%s到%s,第%d个任务，第%d组,处理完毕, %+v\n", managerStartTimeStr, managerEndTimeStr, req.TaskID, i+1, logFields) + } + + end := time.Now() + logFields := map[string]interface{}{ + "总处理组数": n, + "总处理条数": num, + "总通知条数": notifyNum, + "执行任务开始时间": startStr, + "执行任务结束时间": end.Format(time.DateTime), + "总处理耗时": end.Sub(start).String(), + "任务处理开始时间": currentStartTimeStr, + "任务处理结束时间": currentEndTimeStr, + } + + // 生成任务执行结果 + result := fmt.Sprintf("%s到%s，第%d个任务,处理完毕,%+v\n", managerStartTimeStr, managerEndTimeStr, req.TaskID, logFields) + + fmt.Printf(result) + + //return fmt.Errorf(result) + return nil } diff --git a/internal/biz/timeslice/model.go b/internal/biz/timeslice/model.go index ca4b4a5..8e87526 100644 --- a/internal/biz/timeslice/model.go +++ b/internal/biz/timeslice/model.go @@ -1,6 +1,7 @@ package timeslice import ( + "encoding/json" "time" ) @@ -11,14 +12,38 @@ type Manager struct { GoNum int } +func (t *Manager) String() (string, error) { + b, err := json.Marshal(t) + if err != nil { + return "", err + } + return string(b), nil +} + type Process struct { manager *Manager taskCount int } +func (t *Process) String() (string, error) { + b, err := json.Marshal(t) + if err != nil { + return "", err + } + return string(b), nil +} + type Task struct { + Process *Process CurrentStartTime time.Time CurrentEndTime time.Time TaskID int - ProductNo string +} + +func (t *Task) String() (string, error) { + b, err := json.Marshal(t) + if err != nil { + return "", err + } + return string(b), nil } From 769af27bf9b4aaff9ee12e0e2561fe69e4e05c53 Mon Sep 17 00:00:00 2001 From: ziming Date: Wed, 11 Jun 2025 14:29:03 +0800 Subject: [PATCH 06/36] query --- internal/biz/timeslice/model.go | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/internal/biz/timeslice/model.go b/internal/biz/timeslice/model.go index 8e87526..65ef733 100644 --- a/internal/biz/timeslice/model.go +++ b/internal/biz/timeslice/model.go @@ -1,7 +1,6 @@ package timeslice import ( - "encoding/json" "time" ) @@ -12,38 +11,14 @@ type Manager struct { GoNum int } -func (t *Manager) String() (string, error) { - b, err := json.Marshal(t) - if err != nil { - return "", err - } - return string(b), nil -} - type Process struct { manager *Manager taskCount int } -func (t *Process) String() (string, error) { - b, err := json.Marshal(t) - if err != nil { - return "", err - } - return string(b), nil -} - type Task struct { Process *Process CurrentStartTime time.Time CurrentEndTime time.Time TaskID int } - -func (t *Task) String() (string, error) { - b, err := json.Marshal(t) - if err != nil { - return "", err - } - return string(b), nil -} From 153dd55b4db69a2b389106cf5438d18d8b20a9c0 Mon Sep 17 00:00:00 2001 From: ziming Date: Wed, 11 Jun 2025 14:30:00 +0800 Subject: [PATCH 07/36] query --- internal/{biz => pkg}/timeslice/manager.go | 0 internal/{biz => pkg}/timeslice/manager_test.go | 0 internal/{biz => pkg}/timeslice/model.go | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename internal/{biz => pkg}/timeslice/manager.go (100%) rename internal/{biz => pkg}/timeslice/manager_test.go (100%) rename internal/{biz => pkg}/timeslice/model.go (100%) diff --git a/internal/biz/timeslice/manager.go b/internal/pkg/timeslice/manager.go similarity index 100% rename from internal/biz/timeslice/manager.go rename to internal/pkg/timeslice/manager.go diff --git a/internal/biz/timeslice/manager_test.go b/internal/pkg/timeslice/manager_test.go similarity index 100% rename from internal/biz/timeslice/manager_test.go rename to internal/pkg/timeslice/manager_test.go diff --git a/internal/biz/timeslice/model.go b/internal/pkg/timeslice/model.go similarity index 100% rename from internal/biz/timeslice/model.go rename to internal/pkg/timeslice/model.go From 3aa1ad9520f0731a13e081e909abd07cc7dec94e Mon Sep 17 00:00:00 2001 From: ziming Date: Wed, 11 Jun 2025 17:03:41 +0800 Subject: [PATCH 08/36] timeSliceQueryPush --- READEME.md | 55 +-------- cmd/server/wire.go | 2 + configs/config.yaml | 12 +- configs/config_test.yaml | 7 +- internal/biz/timeslicequery/README.md | 14 +++ internal/biz/timeslicequery/base.go | 84 ++++++++++++++ internal/biz/timeslicequery/execute.go | 86 ++++++++++++++ internal/biz/timeslicequery/mq.go | 118 ++++++++++++++++++++ internal/biz/timeslicequery/provider_set.go | 7 ++ internal/biz/timeslicequery/query.go | 66 +++++++++++ internal/conf/conf.pb.go | 90 ++++++++------- internal/conf/conf.proto | 3 +- internal/data/repoimpl/order.go | 2 +- internal/pkg/timeslice/manager.go | 50 +++++---- internal/pkg/timeslice/manager_test.go | 10 +- internal/pkg/timeslice/model.go | 19 ++-- internal/server/http.go | 1 + internal/service/cmb.go | 105 +++-------------- internal/service/script.go | 118 ++++++++++++++++++++ internal/service/voucher.go | 15 ++- internal/service/wechat_query.go | 37 ++++++ 21 files changed, 670 insertions(+), 231 deletions(-) create mode 100644 internal/biz/timeslicequery/README.md create mode 100644 internal/biz/timeslicequery/base.go create mode 100644 internal/biz/timeslicequery/execute.go create mode 100644 internal/biz/timeslicequery/mq.go create mode 100644 internal/biz/timeslicequery/provider_set.go create mode 100644 internal/biz/timeslicequery/query.go create mode 100644 internal/service/script.go diff --git a/READEME.md b/READEME.md index 96a1103..69e7a99 100755 --- a/READEME.md +++ b/READEME.md @@ -1,59 +1,8 @@ -#

营销系统后台API

+#

招行立减金券系统

-### 参与开发 -[请参阅](https://tvd8jq9lqkp.feishu.cn/wiki/LNWVweZ64iY2UBkkTkZcezy0n5h?from=from_copylink) * * * ### 主要工作 -+ 后台接口API ++ 发券API * * * -### 规则说明 -+ 路由前缀都为 __/admin__ 开始，路由规则全小写+下划线,例如：/admin/v1/demo_1 -* * * -### 构建部署 -+ 采用多阶段构建，以获得最小体积的容器镜像 -````bash -cd /项目根目录 && make deploy folder=./configs_dev marketing=marketing_backend container_name=marketing_backend http_port=8090 -```` -* * * -### docker环境下开发 -+ 一、[下载Docker Desktop安装程序](https://www.docker.com/products/docker-desktop) -+ 二、在项目根目录下执行命令 - ```shell - docker build -f Dockerfile_win -t 镜像名称 . - docker run --privileged -itd --name 容器名称 --restart=always -v ./:/src 镜像名称 - docker ps - docker exec -it 容器名称 sh - make init - make all - ``` - -### windows非docker开发 -1 安装插件(配置goproxy,GOPROXY=https://goproxy.cn,direct) -```shell -go install google.golang.org/protobuf/cmd/protoc-gen-go@latest -go install github.com/go-kratos/kratos/cmd/kratos/v2@latest -go install github.com/go-kratos/kratos/cmd/protoc-gen-go-http/v2@latest -go install github.com/google/gnostic/cmd/protoc-gen-openapi@latest -go install github.com/google/wire/cmd/wire@latest -go install github.com/go-kratos/kratos/cmd/protoc-gen-go-errors/v2@latest -go install gorm.io/gen/tools/gentool@latest -``` -2生成相应rpo - -命令：kratos proto client api/helloworld/v1/demo.proto - -位置：api和internal下面的conf - -3 wire生成依赖 -cd cmd/server -wire - -4 配置编译 - -![img_1.png](img.png) - -5 生成service -kratos proto server api/helloworld/v1/demo.proto -t internal/service - diff --git a/cmd/server/wire.go b/cmd/server/wire.go index f03001e..6c27594 100644 --- a/cmd/server/wire.go +++ b/cmd/server/wire.go @@ -12,6 +12,7 @@ import ( "github.com/robfig/cron" "voucher/internal/biz" "voucher/internal/biz/cmb" + "voucher/internal/biz/timeslicequery" "voucher/internal/conf" "voucher/internal/data" "voucher/internal/data/mixrepoimpl" @@ -33,6 +34,7 @@ func wireApp(*conf.Bootstrap, log.Logger, *log2.AccessLogger) (*kratos.App, func repoimpl.ProviderRepoImplSet, wechatrepoimpl.ProviderWechatReposImplSet, mixrepoimpl.ProviderMixRepoImplSet, + timeslicequery.ProviderSetTimeSliceQuery, log2.NewLogHelper, cron.New, newApp, diff --git a/configs/config.yaml b/configs/config.yaml index decdbea..14cc902 100644 --- a/configs/config.yaml +++ b/configs/config.yaml @@ -85,18 +85,24 @@ cron: command: "0 0 1 * * ?" # 每天凌晨1点执行一次 rdsMQ: - wechatQuery: #发放结算 + wechatQuery: name: "wechatQuery" retryNum: 1 #重试次数 numWorkers: 2 #协程数量，不配置默认为10 waitTime: 1s #处理完成后等待时间 + isOpen: false #是否启动消费 true/false + wechatTimeSliceQuery: + name: "wechatTimeSliceQuery" + retryNum: 1 #重试次数 + numWorkers: 3 #协程数量，不配置默认为10 + waitTime: 1s #处理完成后等待时间 isOpen: true #是否启动消费 true/false - orderRetry: #发放结算 + orderRetry: name: "orderRetry" retryNum: 1 #重试次数 numWorkers: 2 #协程数量，不配置默认为10 waitTime: 1s #处理完成后等待时间 - isOpen: true #是否启动消费 true/false + isOpen: false #是否启动消费 true/false #配置日志 logs: diff --git a/configs/config_test.yaml b/configs/config_test.yaml index 9718289..7229860 100644 --- a/configs/config_test.yaml +++ b/configs/config_test.yaml @@ -91,7 +91,12 @@ rdsMQ: numWorkers: 1 #协程数量，不配置默认为10 waitTime: 1s #处理完成后等待时间 isOpen: true #是否启动消费 true/false - + wechatTimeSliceQuery: + name: "wechatTimeSliceQuery" + retryNum: 1 #重试次数 + numWorkers: 3 #协程数量，不配置默认为10 + waitTime: 1s #处理完成后等待时间 + isOpen: true #是否启动消费 true/false #配置日志 logs: business: business.log #业务日志路径：如果不写日志，则不配置或配置为空 diff --git a/internal/biz/timeslicequery/README.md b/internal/biz/timeslicequery/README.md new file mode 100644 index 0000000..e0d8ef2 --- /dev/null +++ b/internal/biz/timeslicequery/README.md @@ -0,0 +1,14 @@ +#

券状态同步

+* * * +### 主要工作 ++ 券状态查询同步 +* * * +### 规则说明 ++ 按照时间分片处理，按照2小时为一个时间片启用一个协程消费处理 ++ 协程最大可同时运行指定数量，设置为2 +* * * +### 使用方式 ++ 消费处理，按照时间范围上报消费 ++ 每次请求按照时间片分别启用2个协程消费处理，也就是每个请求可以同时启用2个协程消费处理 ++ 请不要无休止的访问，请按照时间片进行访问，并且不要重复的时间片访问，增加系统负载，特殊发券日期量较大，建议缩短时间范围上报，多切分上报处理 +* * * \ No newline at end of file diff --git a/internal/biz/timeslicequery/base.go b/internal/biz/timeslicequery/base.go new file mode 100644 index 0000000..81155bd --- /dev/null +++ b/internal/biz/timeslicequery/base.go @@ -0,0 +1,84 @@ +package timeslicequery + +import ( + "github.com/nacos-group/nacos-sdk-go/util" + "sync" + "voucher/internal/biz/cmb" + "voucher/internal/biz/mixrepos" + "voucher/internal/biz/repo" + "voucher/internal/biz/wechatrepo" + "voucher/internal/conf" + "voucher/internal/data" +) + +type Query struct { + mu sync.RWMutex + queryMap map[string]bool + + bc *conf.Bootstrap + rdb *data.Rdb + cmb *cmb.Cmb + + productRepo repo.ProductRepo + orderRepo repo.OrderRepo + + wechatCpnRepo wechatrepo.WechatCpnRepo + mqSendMixRepo mixrepos.MQSendMixRepo +} + +func NewQuery( + bc *conf.Bootstrap, + rdb *data.Rdb, + cmb *cmb.Cmb, + productRepo repo.ProductRepo, + orderRepo repo.OrderRepo, + wechatCpnRepo wechatrepo.WechatCpnRepo, + mqSendMixRepo mixrepos.MQSendMixRepo) *Query { + return &Query{ + queryMap: make(map[string]bool), + bc: bc, + rdb: rdb, + cmb: cmb, + productRepo: productRepo, + orderRepo: orderRepo, + wechatCpnRepo: wechatCpnRepo, + mqSendMixRepo: mqSendMixRepo} +} + +func (v *Query) uid(no string) string { + return util.Md5("query" + no) +} + +func (this *Query) GetAll() map[string]bool { + return this.queryMap +} + +func (this *Query) Get(uid string) bool { + + this.mu.Lock() + defer this.mu.Unlock() + + if _, ok := this.queryMap[uid]; ok { + return ok + } + + return false +} + +func (this *Query) Add(uid string) { + + this.mu.Lock() + defer this.mu.Unlock() + + this.queryMap[uid] = true +} + +func (this *Query) Remove(uid string) { + + this.mu.Lock() + defer this.mu.Unlock() + + if _, ok := this.queryMap[uid]; ok { + delete(this.queryMap, uid) + } +} diff --git a/internal/biz/timeslicequery/execute.go b/internal/biz/timeslicequery/execute.go new file mode 100644 index 0000000..c143c6b --- /dev/null +++ b/internal/biz/timeslicequery/execute.go @@ -0,0 +1,86 @@ +package timeslicequery + +import ( + "context" + "fmt" + "github.com/go-kratos/kratos/v2/log" + "time" + "voucher/internal/biz/bo" + "voucher/internal/biz/do" + "voucher/internal/pkg/timeslice" +) + +func (v *Query) execute(ctx context.Context, req *timeslice.Manager) error { + + managerStartStr := req.StartTime.Format(time.DateTime) + managerEndStr := req.EndTime.Format(time.DateTime) + + taskCount, err := timeslice.NewManager(v.callbackFunc).Run(ctx, req) + if err != nil { + log.Errorf("%s到%s,发生错误:%v", managerStartStr, managerEndStr, err) + } + + fmt.Printf("%s到%s,总任务数:%d\n", managerStartStr, managerEndStr, taskCount) + log.Warnf("%s到%s,总任务数:%d", managerStartStr, managerEndStr, taskCount) + + return nil +} + +func (v *Query) callbackFunc(ctx context.Context, req *timeslice.Task) error { + + startTimeStr := req.Process.Manager.StartTime.Format(time.DateTime) + endTimeStr := req.Process.Manager.EndTime.Format(time.DateTime) + + currentStartTimeStr := req.CurrentStartTime.Format(time.DateTime) + currentEndTimeStr := req.CurrentEndTime.Format(time.DateTime) + + start := time.Now() + startStr := start.Format(time.DateTime) + + x := &do.WechatQuery{ + StartTime: currentStartTimeStr, + EndTime: currentEndTimeStr, + ProductNo: req.Process.Manager.ProductNo, + } + + n := 0 + num := 0 + notifyNum := 0 + + err := v.orderRepo.FinSucByStockIdInBatches(ctx, x, func(ctx context.Context, rows []*bo.OrderBo) error { + + n += 1 + for _, order := range rows { + + num += 1 + if err := v.wechatQuery(ctx, order, ¬ifyNum); err != nil { + logFields := map[string]string{ + "order_no": order.OrderNo, + "coupon_id": order.VoucherNo, + "open_id": order.Account, + "err": err.Error(), + } + log.Errorf("%s到%s,第%d个任务，第%d组,发生错误:+v", startTimeStr, endTimeStr, req.TaskID, n, logFields) + } + + } + + return nil + }) + + end := time.Now() + + logFields := map[string]interface{}{ + "总处理组数": n, + "总处理条数": num, + "总通知条数": notifyNum, + "执行任务开始时间": startStr, + "执行任务结束时间": end.Format(time.DateTime), + "任务处理开始时间": currentStartTimeStr, + "任务处理结束时间": currentEndTimeStr, + "总处理耗时": end.Sub(start).String(), + } + log.Warnf("%s到%s,第%d个任务,处理完毕:%+v", startTimeStr, endTimeStr, req.TaskID, logFields) + + return err +} diff --git a/internal/biz/timeslicequery/mq.go b/internal/biz/timeslicequery/mq.go new file mode 100644 index 0000000..6e0c171 --- /dev/null +++ b/internal/biz/timeslicequery/mq.go @@ -0,0 +1,118 @@ +package timeslicequery + +import ( + "context" + "encoding/json" + "fmt" + "github.com/go-kratos/kratos/v2/log" + "github.com/go-kratos/kratos/v2/transport/http" + "time" + "voucher/internal/biz/do" + "voucher/internal/pkg/timeslice" +) + +func (v *Query) Push(ctx http.Context, req *do.WechatQuery) (string, error) { + + if req.StartTime == "" || req.EndTime == "" { + return "", fmt.Errorf("时间参数不能为空") + } + + queue := v.bc.RdsMQ.GetWechatTimeSliceQuery() + if queue == nil { + return "", fmt.Errorf("队列不存在") + } + + if queue.Name == "" { + return "", fmt.Errorf("队列不存在") + } + + if queue.IsOpen == false { + return "", fmt.Errorf("队列未开启") + } + + if req.ProductNo != "" { + _, err := v.productRepo.GetByProductNo(ctx, req.ProductNo) + if err != nil { + return "", err + } + } + + b, err := json.Marshal(req) + if err != nil { + return "", err + } + + strMsg := string(b) + + uid := v.uid(strMsg) + if v.Get(uid) { + return "", fmt.Errorf("此台服务队列正在处理中,%s-%s,ip:%s", uid, strMsg, ctx.Header().Get("X-Forwarded-For")) + } + + v.Add(uid) + + _, err = v.rdb.Rdb.RPush(ctx, queue.Name, strMsg).Result() + if err != nil { + v.Remove(uid) + return "", fmt.Errorf("添加到队列失败:%v", err) + } + + return strMsg, nil +} + +func (v *Query) getManager(_ context.Context, msg string) (*timeslice.Manager, error) { + + var req *do.WechatQuery + + if err := json.Unmarshal([]byte(msg), &req); err != nil { + return nil, err + } + + if req.StartTime == "" || req.EndTime == "" { + return nil, fmt.Errorf("时间参数不能为空") + } + + start, err := time.Parse(time.DateTime, req.StartTime) + if err != nil { + return nil, err + } + end, err := time.Parse(time.DateTime, req.EndTime) + if err != nil { + return nil, err + } + + return ×lice.Manager{ + StartTime: start, + EndTime: end, + ProductNo: req.ProductNo, + GoNum: 2, // 协程数量 + TimeSliceHours: 2, // 时间间隔 + }, nil +} + +func (v *Query) Consumer(ctx context.Context, msg string) error { + + defer v.Remove(v.uid(msg)) + + req, err := v.getManager(ctx, msg) + if err != nil { + return err + } + + executeStart := time.Now() + executeStartStr := executeStart.Format(time.DateTime) + + log.Warnf("微信券查询处理开始:%s,msg:%s", executeStartStr, msg) + fmt.Printf("微信券查询处理开始:%s,msg:%s", executeStartStr, msg) + + if err = v.execute(ctx, req); err != nil { + log.Errorf("微信券查询处理失败:%s,msg:%s,err:%v", executeStartStr, msg, err) + return fmt.Errorf("微信券查询处理失败:%s,msg:%s,err:%v", executeStartStr, msg, err) + } + + executeEnd := time.Now() + log.Warnf("微信券查询处理耗时:%s,结束时间%s,msg:%s", executeEnd.Sub(executeStart).String(), executeEnd.String(), msg) + fmt.Printf("微信券查询处理耗时:%s,结束时间%s,msg:%s", executeEnd.Sub(executeStart).String(), executeEnd.String(), msg) + + return nil +} diff --git a/internal/biz/timeslicequery/provider_set.go b/internal/biz/timeslicequery/provider_set.go new file mode 100644 index 0000000..72ffd16 --- /dev/null +++ b/internal/biz/timeslicequery/provider_set.go @@ -0,0 +1,7 @@ +package timeslicequery + +import ( + "github.com/google/wire" +) + +var ProviderSetTimeSliceQuery = wire.NewSet(NewQuery) diff --git a/internal/biz/timeslicequery/query.go b/internal/biz/timeslicequery/query.go new file mode 100644 index 0000000..1b626f1 --- /dev/null +++ b/internal/biz/timeslicequery/query.go @@ -0,0 +1,66 @@ +package timeslicequery + +import ( + "context" + "github.com/go-kratos/kratos/v2/log" + "voucher/internal/biz/bo" +) + +func (v *Query) wechatQuery(ctx context.Context, order *bo.OrderBo, notifyNum *int) error { + + status, err := v.wechatCpnRepo.Query(ctx, order) + if err != nil { + return err + } + + if status.IsUse() { + return v.queryUsed(ctx, order, notifyNum) + } else if status.IsExpired() { + return v.queryExpired(ctx, order) + } + + return nil +} + +func (v *Query) queryUsed(ctx context.Context, order *bo.OrderBo, notifyNum *int) error { + + *notifyNum += 1 + + if order.Status.IsUse() { + return v.notify(ctx, order) + } + + if err := v.orderRepo.Used(ctx, order.ID); err != nil { + return err + } + + return v.notify(ctx, order) +} + +func (v *Query) queryExpired(ctx context.Context, order *bo.OrderBo) error { + + if order.Status.IsExpired() { + log.Warnf("券状态已是已过期，忽略不处理,orderNo:%s", order.OrderNo) + return nil + } + + if err := v.orderRepo.Expired(ctx, order.ID); err != nil { + return err + } + + return nil // 过期不做通知 +} + +func (v *Query) notify(ctx context.Context, order *bo.OrderBo) error { + + order, err := v.orderRepo.GetByID(ctx, order.ID) + if err != nil { + return err + } + + if _, err = v.cmb.Notify(ctx, order); err != nil { + return err + } + + return nil +} diff --git a/internal/conf/conf.pb.go b/internal/conf/conf.pb.go index dc691b8..0dcadd7 100644 --- a/internal/conf/conf.pb.go +++ b/internal/conf/conf.pb.go @@ -840,8 +840,9 @@ type RdsMQ struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - WechatQuery *RdsMQ_Queue `protobuf:"bytes,1,opt,name=wechatQuery,proto3" json:"wechatQuery,omitempty"` - WechatRetry *RdsMQ_Queue `protobuf:"bytes,2,opt,name=wechatRetry,proto3" json:"wechatRetry,omitempty"` + WechatQuery *RdsMQ_Queue `protobuf:"bytes,1,opt,name=wechatQuery,proto3" json:"wechatQuery,omitempty"` + WechatTimeSliceQuery *RdsMQ_Queue `protobuf:"bytes,2,opt,name=wechatTimeSliceQuery,proto3" json:"wechatTimeSliceQuery,omitempty"` + WechatRetry *RdsMQ_Queue `protobuf:"bytes,3,opt,name=wechatRetry,proto3" json:"wechatRetry,omitempty"` } func (x *RdsMQ) Reset() { @@ -883,6 +884,13 @@ func (x *RdsMQ) GetWechatQuery() *RdsMQ_Queue { return nil } +func (x *RdsMQ) GetWechatTimeSliceQuery() *RdsMQ_Queue { + if x != nil { + return x.WechatTimeSliceQuery + } + return nil +} + func (x *RdsMQ) GetWechatRetry() *RdsMQ_Queue { if x != nil { return x.WechatRetry @@ -1552,32 +1560,37 @@ var file_conf_conf_proto_rawDesc = []byte{ 0x6b, 0x65, 0x79, 0x12, 0x35, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x76, 0x6f, 0x75, 0x63, 0x68, 0x65, 0x72, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x43, 0x72, 0x6f, 0x6e, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, - 0x4d, 0x61, 0x70, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xae, + 0x4d, 0x61, 0x70, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xff, 0x02, 0x0a, 0x05, 0x52, 0x64, 0x73, 0x4d, 0x51, 0x12, 0x3d, 0x0a, 0x0b, 0x77, 0x65, 0x63, 0x68, 0x61, 0x74, 0x51, 0x75, 0x65, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x76, 0x6f, 0x75, 0x63, 0x68, 0x65, 0x72, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52, 0x64, 0x73, 0x4d, 0x51, 0x2e, 0x51, 0x75, 0x65, 0x75, 0x65, 0x52, 0x0b, 0x77, 0x65, 0x63, 0x68, - 0x61, 0x74, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x3d, 0x0a, 0x0b, 0x77, 0x65, 0x63, 0x68, 0x61, - 0x74, 0x52, 0x65, 0x74, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x76, - 0x6f, 0x75, 0x63, 0x68, 0x65, 0x72, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52, 0x64, - 0x73, 0x4d, 0x51, 0x2e, 0x51, 0x75, 0x65, 0x75, 0x65, 0x52, 0x0b, 0x77, 0x65, 0x63, 0x68, 0x61, - 0x74, 0x52, 0x65, 0x74, 0x72, 0x79, 0x1a, 0xa6, 0x01, 0x0a, 0x05, 0x51, 0x75, 0x65, 0x75, 0x65, - 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x69, 0x73, 0x4f, 0x70, 0x65, 0x6e, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x69, 0x73, 0x4f, 0x70, 0x65, 0x6e, 0x12, 0x1a, 0x0a, 0x08, - 0x72, 0x65, 0x74, 0x72, 0x79, 0x4e, 0x75, 0x6d, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, - 0x72, 0x65, 0x74, 0x72, 0x79, 0x4e, 0x75, 0x6d, 0x12, 0x1e, 0x0a, 0x0a, 0x6e, 0x75, 0x6d, 0x57, - 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x6e, 0x75, - 0x6d, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x12, 0x35, 0x0a, 0x08, 0x77, 0x61, 0x69, 0x74, - 0x54, 0x69, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x77, 0x61, 0x69, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x22, - 0x3a, 0x0a, 0x04, 0x4c, 0x6f, 0x67, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x62, 0x75, 0x73, 0x69, 0x6e, - 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x62, 0x75, 0x73, 0x69, 0x6e, - 0x65, 0x73, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x06, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x42, 0x17, 0x5a, 0x15, 0x76, - 0x6f, 0x75, 0x63, 0x68, 0x65, 0x72, 0x2f, 0x63, 0x70, 0x6e, 0x2f, 0x63, 0x6f, 0x6e, 0x66, 0x3b, - 0x63, 0x6f, 0x6e, 0x66, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x61, 0x74, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x4f, 0x0a, 0x14, 0x77, 0x65, 0x63, 0x68, 0x61, + 0x74, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x6c, 0x69, 0x63, 0x65, 0x51, 0x75, 0x65, 0x72, 0x79, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x76, 0x6f, 0x75, 0x63, 0x68, 0x65, 0x72, 0x2e, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52, 0x64, 0x73, 0x4d, 0x51, 0x2e, 0x51, 0x75, 0x65, + 0x75, 0x65, 0x52, 0x14, 0x77, 0x65, 0x63, 0x68, 0x61, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x6c, + 0x69, 0x63, 0x65, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x3d, 0x0a, 0x0b, 0x77, 0x65, 0x63, 0x68, + 0x61, 0x74, 0x52, 0x65, 0x74, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, + 0x76, 0x6f, 0x75, 0x63, 0x68, 0x65, 0x72, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52, + 0x64, 0x73, 0x4d, 0x51, 0x2e, 0x51, 0x75, 0x65, 0x75, 0x65, 0x52, 0x0b, 0x77, 0x65, 0x63, 0x68, + 0x61, 0x74, 0x52, 0x65, 0x74, 0x72, 0x79, 0x1a, 0xa6, 0x01, 0x0a, 0x05, 0x51, 0x75, 0x65, 0x75, + 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x69, 0x73, 0x4f, 0x70, 0x65, 0x6e, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x69, 0x73, 0x4f, 0x70, 0x65, 0x6e, 0x12, 0x1a, 0x0a, + 0x08, 0x72, 0x65, 0x74, 0x72, 0x79, 0x4e, 0x75, 0x6d, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x08, 0x72, 0x65, 0x74, 0x72, 0x79, 0x4e, 0x75, 0x6d, 0x12, 0x1e, 0x0a, 0x0a, 0x6e, 0x75, 0x6d, + 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x6e, + 0x75, 0x6d, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x12, 0x35, 0x0a, 0x08, 0x77, 0x61, 0x69, + 0x74, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x77, 0x61, 0x69, 0x74, 0x54, 0x69, 0x6d, 0x65, + 0x22, 0x3a, 0x0a, 0x04, 0x4c, 0x6f, 0x67, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x62, 0x75, 0x73, 0x69, + 0x6e, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x62, 0x75, 0x73, 0x69, + 0x6e, 0x65, 0x73, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x42, 0x17, 0x5a, 0x15, + 0x76, 0x6f, 0x75, 0x63, 0x68, 0x65, 0x72, 0x2f, 0x63, 0x70, 0x6e, 0x2f, 0x63, 0x6f, 0x6e, 0x66, + 0x3b, 0x63, 0x6f, 0x6e, 0x66, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1632,20 +1645,21 @@ var file_conf_conf_proto_depIdxs = []int32{ 15, // 13: voucher.config.RocketMQ.eventMap:type_name -> voucher.config.RocketMQ.EventMapEntry 17, // 14: voucher.config.Cron.commandMap:type_name -> voucher.config.Cron.CommandMapEntry 18, // 15: voucher.config.RdsMQ.wechatQuery:type_name -> voucher.config.RdsMQ.Queue - 18, // 16: voucher.config.RdsMQ.wechatRetry:type_name -> voucher.config.RdsMQ.Queue - 19, // 17: voucher.config.Server.HTTP.timeout:type_name -> google.protobuf.Duration - 19, // 18: voucher.config.Data.Database.maxLifetime:type_name -> google.protobuf.Duration - 19, // 19: voucher.config.Data.Redis.readTimeout:type_name -> google.protobuf.Duration - 19, // 20: voucher.config.Data.Redis.writeTimeout:type_name -> google.protobuf.Duration - 19, // 21: voucher.config.Data.Redis.connMaxIdleTime:type_name -> google.protobuf.Duration - 4, // 22: voucher.config.RocketMQ.EventMapEntry.value:type_name -> voucher.config.EventMap - 16, // 23: voucher.config.Cron.CommandMapEntry.value:type_name -> voucher.config.Cron.CommandMap - 19, // 24: voucher.config.RdsMQ.Queue.waitTime:type_name -> google.protobuf.Duration - 25, // [25:25] is the sub-list for method output_type - 25, // [25:25] is the sub-list for method input_type - 25, // [25:25] is the sub-list for extension type_name - 25, // [25:25] is the sub-list for extension extendee - 0, // [0:25] is the sub-list for field type_name + 18, // 16: voucher.config.RdsMQ.wechatTimeSliceQuery:type_name -> voucher.config.RdsMQ.Queue + 18, // 17: voucher.config.RdsMQ.wechatRetry:type_name -> voucher.config.RdsMQ.Queue + 19, // 18: voucher.config.Server.HTTP.timeout:type_name -> google.protobuf.Duration + 19, // 19: voucher.config.Data.Database.maxLifetime:type_name -> google.protobuf.Duration + 19, // 20: voucher.config.Data.Redis.readTimeout:type_name -> google.protobuf.Duration + 19, // 21: voucher.config.Data.Redis.writeTimeout:type_name -> google.protobuf.Duration + 19, // 22: voucher.config.Data.Redis.connMaxIdleTime:type_name -> google.protobuf.Duration + 4, // 23: voucher.config.RocketMQ.EventMapEntry.value:type_name -> voucher.config.EventMap + 16, // 24: voucher.config.Cron.CommandMapEntry.value:type_name -> voucher.config.Cron.CommandMap + 19, // 25: voucher.config.RdsMQ.Queue.waitTime:type_name -> google.protobuf.Duration + 26, // [26:26] is the sub-list for method output_type + 26, // [26:26] is the sub-list for method input_type + 26, // [26:26] is the sub-list for extension type_name + 26, // [26:26] is the sub-list for extension extendee + 0, // [0:26] is the sub-list for field type_name } func init() { file_conf_conf_proto_init() } diff --git a/internal/conf/conf.proto b/internal/conf/conf.proto index cb15acf..772e7fc 100644 --- a/internal/conf/conf.proto +++ b/internal/conf/conf.proto @@ -128,7 +128,8 @@ message RdsMQ { google.protobuf.Duration waitTime = 5; } Queue wechatQuery = 1; - Queue wechatRetry = 2; + Queue wechatTimeSliceQuery = 2; + Queue wechatRetry = 3; } message Logs { diff --git a/internal/data/repoimpl/order.go b/internal/data/repoimpl/order.go index 6c803bc..f4dfea0 100644 --- a/internal/data/repoimpl/order.go +++ b/internal/data/repoimpl/order.go @@ -39,7 +39,7 @@ func (p *OrderRepoImpl) FinSucByStockIdInBatches(ctx context.Context, req *do.We tx = tx.Where("product_no = ?", req.ProductNo) } if req.StartTime != "" { - tx = tx.Where("receive_success_time >= ?", req.StartTime) + tx = tx.Where("receive_success_time > ?", req.StartTime) } if req.EndTime != "" { tx = tx.Where("receive_success_time <= ?", req.EndTime) diff --git a/internal/pkg/timeslice/manager.go b/internal/pkg/timeslice/manager.go index aa7e0c6..2d221ac 100644 --- a/internal/pkg/timeslice/manager.go +++ b/internal/pkg/timeslice/manager.go @@ -8,11 +8,15 @@ import ( "time" ) +const TimeSliceHours = 2 + +type Callback func(ctx context.Context, req *Task) error + type ManagerSrv struct { - callback func(ctx context.Context, req *Task) error + callback Callback } -func NewManager(callback func(ctx context.Context, req *Task) error) *ManagerSrv { +func NewManager(callback Callback) *ManagerSrv { return &ManagerSrv{callback: callback} } @@ -22,17 +26,24 @@ func (m *ManagerSrv) Run(ctx context.Context, req *Manager) (int, error) { return 0, fmt.Errorf("start_time不能大于end_time") } - totalHours := req.EndTime.Sub(req.StartTime).Hours() - taskCount := int(totalHours / 2) + if req.GoNum == 0 { + return 0, fmt.Errorf("协程数量不能为0") + } + if req.GoNum > 100 { + return 0, fmt.Errorf("协程数量不能大于100") + } - // 如果剩余时间不足2小时，增加任务数 - if totalHours-float64(taskCount)*float64(2) > 0 { + totalHours := req.EndTime.Sub(req.StartTime).Hours() + taskCount := int(totalHours / TimeSliceHours) + + // 如果剩余时间不足 TimeSliceHours 小时，增加任务数 + if totalHours-float64(taskCount)*float64(TimeSliceHours) > 0 { taskCount++ } processReq := &Process{ - manager: req, - taskCount: taskCount, + Manager: req, + TaskCount: taskCount, } return taskCount, m.process(ctx, processReq) @@ -40,31 +51,24 @@ func (m *ManagerSrv) Run(ctx context.Context, req *Manager) (int, error) { func (m *ManagerSrv) process(ctx context.Context, req *Process) error { - if req.taskCount == 0 { + if req.TaskCount == 0 { return fmt.Errorf("该时间范围无可执行任务次数，请检查时间范围") } - if req.manager.GoNum == 0 { - return fmt.Errorf("协程数量不能为0") - } - if req.manager.GoNum > 100 { - return fmt.Errorf("协程数量不能大于100") - } - // 设置最大并发任务数为 5 eg := new(errgroup.Group) - eg.SetLimit(req.manager.GoNum) + eg.SetLimit(req.Manager.GoNum) errs := make([]error, 0) // 用于存储所有错误 - // 为每个任务分配开始和结束时间 - for i := 0; i < req.taskCount; i++ { + // 为每个任务按指定的时间片 TimeSliceHours 分配开始和结束时间 + for i := 0; i < req.TaskCount; i++ { - currentStart := req.manager.StartTime.Add(time.Duration(i) * 2 * time.Hour) - currentEnd := currentStart.Add(2 * time.Hour) + currentStart := req.Manager.StartTime.Add(time.Duration(i) * TimeSliceHours * time.Hour) + currentEnd := currentStart.Add(TimeSliceHours * time.Hour) - if currentEnd.After(req.manager.EndTime) { - currentEnd = req.manager.EndTime + if currentEnd.After(req.Manager.EndTime) { + currentEnd = req.Manager.EndTime } taskID := i + 1 diff --git a/internal/pkg/timeslice/manager_test.go b/internal/pkg/timeslice/manager_test.go index 2864dc9..2cda4ae 100644 --- a/internal/pkg/timeslice/manager_test.go +++ b/internal/pkg/timeslice/manager_test.go @@ -49,10 +49,10 @@ func TestNewManager(t *testing.T) { // 模拟任务执行，休眠随机时间 time.Sleep(time.Duration(rand.Intn(3)) * time.Second) // 生成任务执行结果 - result := fmt.Sprintf("任务批次 %d-%s: %s 至 %s 处理完成", req.TaskID, req.Process.manager.ProductNo, req.CurrentStartTime.Format(time.DateTime), req.CurrentEndTime.Format(time.DateTime)) + result := fmt.Sprintf("任务批次 %d-%s: %s 至 %s 处理完成", req.TaskID, req.Process.Manager.ProductNo, req.CurrentStartTime.Format(time.DateTime), req.CurrentEndTime.Format(time.DateTime)) results = append(results, result) //return nil - return fmt.Errorf("任务执行失败:%d-%s,时间%s-%s", req.TaskID, req.Process.manager.ProductNo, req.CurrentStartTime.Format(time.DateTime), req.CurrentEndTime.Format(time.DateTime)) + return fmt.Errorf("任务执行失败:%d-%s,时间%s-%s", req.TaskID, req.Process.Manager.ProductNo, req.CurrentStartTime.Format(time.DateTime), req.CurrentEndTime.Format(time.DateTime)) } startTime := time.Now() @@ -179,8 +179,8 @@ func CallbackFunc(start, end time.Time) error { func callbackFunc(_ context.Context, req *Task) error { - managerStartTimeStr := req.Process.manager.StartTime.Format(time.DateTime) - managerEndTimeStr := req.Process.manager.EndTime.Format(time.DateTime) + managerStartTimeStr := req.Process.Manager.StartTime.Format(time.DateTime) + managerEndTimeStr := req.Process.Manager.EndTime.Format(time.DateTime) currentStartTimeStr := req.CurrentStartTime.Format(time.DateTime) currentEndTimeStr := req.CurrentEndTime.Format(time.DateTime) @@ -225,9 +225,7 @@ func callbackFunc(_ context.Context, req *Task) error { // 生成任务执行结果 result := fmt.Sprintf("%s到%s，第%d个任务,处理完毕,%+v\n", managerStartTimeStr, managerEndTimeStr, req.TaskID, logFields) - fmt.Printf(result) - //return fmt.Errorf(result) return nil } diff --git a/internal/pkg/timeslice/model.go b/internal/pkg/timeslice/model.go index 65ef733..35dfb12 100644 --- a/internal/pkg/timeslice/model.go +++ b/internal/pkg/timeslice/model.go @@ -5,20 +5,21 @@ import ( ) type Manager struct { - StartTime time.Time - EndTime time.Time - ProductNo string - GoNum int + StartTime time.Time // 开始时间 + EndTime time.Time // 结束时间 + ProductNo string // 产品编号 + GoNum int // 并发数 + TimeSliceHours int // 时间片"小时" } type Process struct { - manager *Manager - taskCount int + Manager *Manager + TaskCount int // 任务数 } type Task struct { Process *Process - CurrentStartTime time.Time - CurrentEndTime time.Time - TaskID int + CurrentStartTime time.Time // 时间片开始时间 + CurrentEndTime time.Time // 时间片结束时间 + TaskID int // 任务ID } diff --git a/internal/server/http.go b/internal/server/http.go index 3109eed..5bdbec5 100644 --- a/internal/server/http.go +++ b/internal/server/http.go @@ -40,6 +40,7 @@ func NewHTTPServer( srv.Route("/voucher/").POST("queryOrder/{order_no}", cmb.QueryOrder) srv.Route("/voucher/").POST("registerTag/{product_no}", cmb.RegisterTag) srv.Route("/voucher/").POST("pushWechatQuery", cmb.PushWechatQuery) + srv.Route("/voucher/").POST("timeSliceQueryPush", cmb.TimeSliceQueryPush) srv.Route("/voucher/").POST("pushWechatRetry/{product_no}", cmb.PushWechatRetry) v1.RegisterCmbHTTPServer(srv, cmb) diff --git a/internal/service/cmb.go b/internal/service/cmb.go index 7936dbc..cb5d52f 100644 --- a/internal/service/cmb.go +++ b/internal/service/cmb.go @@ -2,19 +2,16 @@ package service import ( "context" - "encoding/json" "fmt" "github.com/go-kratos/kratos/v2/log" "github.com/go-kratos/kratos/v2/transport/http" "github.com/robfig/cron" - "io" http2 "net/http" - "strconv" v1 "voucher/api/v1" "voucher/internal/biz" "voucher/internal/biz/bo" - "voucher/internal/biz/do" "voucher/internal/biz/mixrepos" + "voucher/internal/biz/timeslicequery" "voucher/internal/biz/vo" "voucher/internal/biz/wechatrepo" "voucher/internal/conf" @@ -23,11 +20,12 @@ import ( var _ v1.CmbHTTPServer = (*CmbService)(nil) type CmbService struct { - bc *conf.Bootstrap - cron *cron.Cron - VoucherBiz *biz.VoucherBiz - CmbMixRepo mixrepos.CmbMixRepo - WechatCpnRepo wechatrepo.WechatCpnRepo + bc *conf.Bootstrap + cron *cron.Cron + VoucherBiz *biz.VoucherBiz + CmbMixRepo mixrepos.CmbMixRepo + WechatCpnRepo wechatrepo.WechatCpnRepo + timeSliceQuery *timeslicequery.Query } func NewCmbService( @@ -36,13 +34,15 @@ func NewCmbService( VoucherBiz *biz.VoucherBiz, CmbMixRepo mixrepos.CmbMixRepo, WechatCpnRepo wechatrepo.WechatCpnRepo, + timeSliceQuery *timeslicequery.Query, ) *CmbService { return &CmbService{ - bc: bc, - cron: cron, - VoucherBiz: VoucherBiz, - CmbMixRepo: CmbMixRepo, - WechatCpnRepo: WechatCpnRepo, + bc: bc, + cron: cron, + VoucherBiz: VoucherBiz, + CmbMixRepo: CmbMixRepo, + WechatCpnRepo: WechatCpnRepo, + timeSliceQuery: timeSliceQuery, } } @@ -63,37 +63,6 @@ func (c *CmbService) GetResponse(ctx context.Context, replyBizContent []byte) (* return reply, nil } -func (this *CmbService) NotifyRetry(ctx http.Context) error { - id := ctx.Vars().Get("id") - if id == "" { - return fmt.Errorf("id is empty") - } - - orderNotifyId, err := strconv.ParseUint(id, 10, 64) - if err != nil { - return err - } - - return this.VoucherBiz.PushNotifyRetryDelayMQ(ctx, 1, orderNotifyId) -} - -func (this *CmbService) QueryOrder(ctx http.Context) error { - - orderNo := ctx.Vars().Get("order_no") - if orderNo == "" { - return fmt.Errorf("orderNo is empty") - } - - str, err := this.VoucherBiz.QueryOrder(ctx, orderNo) - if err != nil { - return err - } - - return ctx.JSON(http2.StatusOK, map[string]interface{}{ - "data": str, - }) -} - func (c *CmbService) OrderRetry(ctx context.Context, request *v1.OrderRetryRequest) (*v1.Empty, error) { return nil, c.VoucherBiz.OrderRetry(ctx, request.GetTransactionIds()) @@ -115,49 +84,3 @@ func (this *CmbService) RegisterTag(ctx http.Context) error { "data": productNo, }) } - -func (this *CmbService) PushWechatQuery(ctx http.Context) error { - - bodyBytes, err := io.ReadAll(ctx.Request().Body) - if err != nil { - return err - } - - var req *do.WechatQuery - if err = json.Unmarshal(bodyBytes, &req); err != nil { - return err - } - - if req == nil { - return fmt.Errorf("req is empty") - } - - if req.StartTime == "" || req.EndTime == "" { - return fmt.Errorf("start_time or end_time is empty") - } - - if err = this.VoucherBiz.PushWechatQuery(ctx, req); err != nil { - return err - } - - return ctx.JSON(http2.StatusOK, map[string]interface{}{ - "data": req, - }) -} - -func (this *CmbService) PushWechatRetry(ctx http.Context) error { - - productNo := ctx.Vars().Get("product_no") - if productNo == "" { - return fmt.Errorf("product_no is empty") - } - - err := this.VoucherBiz.PushWechatRetry(ctx, productNo) - if err != nil { - return err - } - - return ctx.JSON(http2.StatusOK, map[string]interface{}{ - "data": productNo, - }) -} diff --git a/internal/service/script.go b/internal/service/script.go new file mode 100644 index 0000000..e1ea10f --- /dev/null +++ b/internal/service/script.go @@ -0,0 +1,118 @@ +package service + +import ( + "encoding/json" + "fmt" + "github.com/go-kratos/kratos/v2/transport/http" + "io" + http2 "net/http" + "strconv" + "voucher/internal/biz/do" +) + +func (this *CmbService) NotifyRetry(ctx http.Context) error { + id := ctx.Vars().Get("id") + if id == "" { + return fmt.Errorf("id is empty") + } + + orderNotifyId, err := strconv.ParseUint(id, 10, 64) + if err != nil { + return err + } + + return this.VoucherBiz.PushNotifyRetryDelayMQ(ctx, 1, orderNotifyId) +} + +func (this *CmbService) QueryOrder(ctx http.Context) error { + + orderNo := ctx.Vars().Get("order_no") + if orderNo == "" { + return fmt.Errorf("orderNo is empty") + } + + str, err := this.VoucherBiz.QueryOrder(ctx, orderNo) + if err != nil { + return err + } + + return ctx.JSON(http2.StatusOK, map[string]interface{}{ + "data": str, + }) +} + +func (this *CmbService) PushWechatQuery(ctx http.Context) error { + + bodyBytes, err := io.ReadAll(ctx.Request().Body) + if err != nil { + return err + } + + var req *do.WechatQuery + if err = json.Unmarshal(bodyBytes, &req); err != nil { + return err + } + + if req == nil { + return fmt.Errorf("req is empty") + } + + if req.StartTime == "" || req.EndTime == "" { + return fmt.Errorf("start_time or end_time is empty") + } + + if err = this.VoucherBiz.PushWechatQuery(ctx, req); err != nil { + return err + } + + return ctx.JSON(http2.StatusOK, map[string]interface{}{ + "data": req, + }) +} + +func (this *CmbService) TimeSliceQueryPush(ctx http.Context) error { + + bodyBytes, err := io.ReadAll(ctx.Request().Body) + if err != nil { + return err + } + + var req *do.WechatQuery + if err = json.Unmarshal(bodyBytes, &req); err != nil { + return err + } + + if req == nil { + return fmt.Errorf("req is empty") + } + + if req.StartTime == "" || req.EndTime == "" { + return fmt.Errorf("start_time or end_time is empty") + } + + reps, err := this.timeSliceQuery.Push(ctx, req) + if err != nil { + return err + } + + return ctx.JSON(http2.StatusOK, map[string]interface{}{ + "data": reps, + }) +} + +func (this *CmbService) PushWechatRetry(ctx http.Context) error { + + productNo := ctx.Vars().Get("product_no") + if productNo == "" { + return fmt.Errorf("product_no is empty") + } + + err := this.VoucherBiz.PushWechatRetry(ctx, productNo) + if err != nil { + return err + } + + return ctx.JSON(http2.StatusOK, map[string]interface{}{ + "data": productNo, + }) +} diff --git a/internal/service/voucher.go b/internal/service/voucher.go index c663d55..83f1acb 100644 --- a/internal/service/voucher.go +++ b/internal/service/voucher.go @@ -10,6 +10,7 @@ import ( "time" "voucher/internal/biz" "voucher/internal/biz/bo" + "voucher/internal/biz/timeslicequery" "voucher/internal/conf" "voucher/internal/data" "voucher/internal/pkg/mq" @@ -21,6 +22,8 @@ type VoucherService struct { VoucherBiz *biz.VoucherBiz rdb *data.Rdb logHelper *log.Helper + + timeSliceQuery *timeslicequery.Query } func NewVoucherService( @@ -29,13 +32,15 @@ func NewVoucherService( VoucherBiz *biz.VoucherBiz, rdb *data.Rdb, logHelper *log.Helper, + timeSliceQuery *timeslicequery.Query, ) *VoucherService { return &VoucherService{ - bc: bc, - cron: cron, - VoucherBiz: VoucherBiz, - rdb: rdb, - logHelper: logHelper, + bc: bc, + cron: cron, + VoucherBiz: VoucherBiz, + rdb: rdb, + logHelper: logHelper, + timeSliceQuery: timeSliceQuery, } } diff --git a/internal/service/wechat_query.go b/internal/service/wechat_query.go index 390964e..6e44485 100644 --- a/internal/service/wechat_query.go +++ b/internal/service/wechat_query.go @@ -43,3 +43,40 @@ func (s *VoucherService) WechatQueryHandle(ctx context.Context, msg string) erro return nil } + +func (s *VoucherService) GetWechatTimeSliceQueryConfig() *rdsmq.ConsumeConfig { + + queue := s.bc.RdsMQ.GetWechatTimeSliceQuery() + if queue == nil { + return nil + } + + if !queue.GetIsOpen() { + log.Warn(fmt.Sprintf("[%s]RdsMQ is not open", queue.Name)) + return nil + } + + return &rdsmq.ConsumeConfig{ + Rdb: s.rdb.Rdb, + QueueName: queue.Name, + NumWorkers: queue.NumWorkers, + WaitTime: queue.GetWaitTime().AsDuration(), + RetryNum: queue.RetryNum, + Fn: s.WechatTimeSliceQueryHandle, + Logger: s.logHelper, + } +} + +func (s *VoucherService) WechatTimeSliceQueryHandle(ctx context.Context, msg string) error { + + if msg == "" { + s.logHelper.Errorf("wechat TimeSlice query error: batchNo is empty") + return nil + } + + if err := s.timeSliceQuery.Consumer(ctx, msg); err != nil { + s.logHelper.Errorf("wechat TimeSlice query msg:%s error: %v", msg, err) + } + + return nil +} From d7a1f7ce863e6d15565973c88f4faa5e85b56390 Mon Sep 17 00:00:00 2001 From: ziming Date: Wed, 11 Jun 2025 17:31:18 +0800 Subject: [PATCH 09/36] timeSliceQueryPush --- internal/biz/timeslicequery/execute.go | 3 +-- internal/pkg/timeslice/manager.go | 31 +++++++++++++++++++++----- internal/server/rds_consume.go | 4 ++++ 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/internal/biz/timeslicequery/execute.go b/internal/biz/timeslicequery/execute.go index c143c6b..d0928ba 100644 --- a/internal/biz/timeslicequery/execute.go +++ b/internal/biz/timeslicequery/execute.go @@ -71,13 +71,12 @@ func (v *Query) callbackFunc(ctx context.Context, req *timeslice.Task) error { end := time.Now() logFields := map[string]interface{}{ + "任务处理时间": currentStartTimeStr + "到" + currentEndTimeStr, "总处理组数": n, "总处理条数": num, "总通知条数": notifyNum, "执行任务开始时间": startStr, "执行任务结束时间": end.Format(time.DateTime), - "任务处理开始时间": currentStartTimeStr, - "任务处理结束时间": currentEndTimeStr, "总处理耗时": end.Sub(start).String(), } log.Warnf("%s到%s,第%d个任务,处理完毕:%+v", startTimeStr, endTimeStr, req.TaskID, logFields) diff --git a/internal/pkg/timeslice/manager.go b/internal/pkg/timeslice/manager.go index 2d221ac..4e3e8a5 100644 --- a/internal/pkg/timeslice/manager.go +++ b/internal/pkg/timeslice/manager.go @@ -5,10 +5,14 @@ import ( "fmt" "github.com/hashicorp/go-multierror" "golang.org/x/sync/errgroup" + "sync" "time" ) -const TimeSliceHours = 2 +const ( + TimeSliceHours = 2 + maxGlobalGoroutines = 1000 +) type Callback func(ctx context.Context, req *Task) error @@ -29,8 +33,8 @@ func (m *ManagerSrv) Run(ctx context.Context, req *Manager) (int, error) { if req.GoNum == 0 { return 0, fmt.Errorf("协程数量不能为0") } - if req.GoNum > 100 { - return 0, fmt.Errorf("协程数量不能大于100") + if req.GoNum > maxGlobalGoroutines { + return 0, fmt.Errorf("协程数量不能大于%d", maxGlobalGoroutines) } totalHours := req.EndTime.Sub(req.StartTime).Hours() @@ -58,8 +62,9 @@ func (m *ManagerSrv) process(ctx context.Context, req *Process) error { // 设置最大并发任务数为 5 eg := new(errgroup.Group) eg.SetLimit(req.Manager.GoNum) + var mu sync.Mutex - errs := make([]error, 0) // 用于存储所有错误 + errs := make([]error, 0, req.TaskCount) // 为每个任务按指定的时间片 TimeSliceHours 分配开始和结束时间 for i := 0; i < req.TaskCount; i++ { @@ -74,9 +79,21 @@ func (m *ManagerSrv) process(ctx context.Context, req *Process) error { eg.Go(func() error { + select { + case <-ctx.Done(): + mu.Lock() + errs = append(errs, fmt.Errorf("任务 %d 被上下文取消", taskID)) + mu.Unlock() + return ctx.Err() + default: + // 继续执行 + } + defer func() { if err := recover(); err != nil { - errs = append(errs, fmt.Errorf("panic: %v", err)) + mu.Lock() + errs = append(errs, fmt.Errorf("任务 %d panic: %v", taskID, err)) + mu.Unlock() } }() @@ -88,7 +105,9 @@ func (m *ManagerSrv) process(ctx context.Context, req *Process) error { } if err := m.callback(ctx, taskReq); err != nil { - errs = append(errs, err) + mu.Lock() + errs = append(errs, fmt.Errorf("任务 %d 执行失败: %v", taskID, err)) + mu.Unlock() } return nil diff --git a/internal/server/rds_consume.go b/internal/server/rds_consume.go index 895f7d3..0a34025 100644 --- a/internal/server/rds_consume.go +++ b/internal/server/rds_consume.go @@ -30,6 +30,10 @@ func NewRdbConsumer( manager.Add(cf) } + if cf1 := voucherService.GetWechatTimeSliceQueryConfig(); cf1 != nil { + manager.Add(cf1) + } + if cf2 := voucherService.GetWechatRetryConfig(); cf2 != nil { manager.Add(cf2) } From 644075a199305d5996032ca2d92fd1305fe31e8e Mon Sep 17 00:00:00 2001 From: ziming Date: Wed, 11 Jun 2025 17:45:21 +0800 Subject: [PATCH 10/36] timeSliceQueryPush --- internal/biz/timeslicequery/execute.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/internal/biz/timeslicequery/execute.go b/internal/biz/timeslicequery/execute.go index d0928ba..24838f8 100644 --- a/internal/biz/timeslicequery/execute.go +++ b/internal/biz/timeslicequery/execute.go @@ -46,6 +46,7 @@ func (v *Query) callbackFunc(ctx context.Context, req *timeslice.Task) error { n := 0 num := 0 notifyNum := 0 + errNum := 0 err := v.orderRepo.FinSucByStockIdInBatches(ctx, x, func(ctx context.Context, rows []*bo.OrderBo) error { @@ -54,13 +55,20 @@ func (v *Query) callbackFunc(ctx context.Context, req *timeslice.Task) error { num += 1 if err := v.wechatQuery(ctx, order, ¬ifyNum); err != nil { + + errNum += 1 + logFields := map[string]string{ "order_no": order.OrderNo, "coupon_id": order.VoucherNo, "open_id": order.Account, "err": err.Error(), } - log.Errorf("%s到%s,第%d个任务，第%d组,发生错误:+v", startTimeStr, endTimeStr, req.TaskID, n, logFields) + log.Errorf("%s到%s,第%d个任务，第%d组,发生错误:%+v", startTimeStr, endTimeStr, req.TaskID, n, logFields) + + if errNum > 20 { + return fmt.Errorf("%s到%s,第%d个任务，已经连续发生20次错误%+v", startTimeStr, endTimeStr, req.TaskID, logFields) + } } } From d3be7d733be25a44d4cbd10ab06bdb68e25701b2 Mon Sep 17 00:00:00 2001 From: ziming Date: Wed, 11 Jun 2025 18:11:45 +0800 Subject: [PATCH 11/36] timeSliceQueryPush --- internal/biz/timeslicequery/execute.go | 18 ++++++------------ internal/pkg/script/script.go | 12 ++++++++---- internal/pkg/script/script_test.go | 4 ++-- internal/service/script.go | 4 +--- 4 files changed, 17 insertions(+), 21 deletions(-) diff --git a/internal/biz/timeslicequery/execute.go b/internal/biz/timeslicequery/execute.go index 24838f8..b33acdb 100644 --- a/internal/biz/timeslicequery/execute.go +++ b/internal/biz/timeslicequery/execute.go @@ -35,7 +35,6 @@ func (v *Query) callbackFunc(ctx context.Context, req *timeslice.Task) error { currentEndTimeStr := req.CurrentEndTime.Format(time.DateTime) start := time.Now() - startStr := start.Format(time.DateTime) x := &do.WechatQuery{ StartTime: currentStartTimeStr, @@ -43,14 +42,12 @@ func (v *Query) callbackFunc(ctx context.Context, req *timeslice.Task) error { ProductNo: req.Process.Manager.ProductNo, } - n := 0 num := 0 notifyNum := 0 errNum := 0 err := v.orderRepo.FinSucByStockIdInBatches(ctx, x, func(ctx context.Context, rows []*bo.OrderBo) error { - n += 1 for _, order := range rows { num += 1 @@ -64,7 +61,7 @@ func (v *Query) callbackFunc(ctx context.Context, req *timeslice.Task) error { "open_id": order.Account, "err": err.Error(), } - log.Errorf("%s到%s,第%d个任务，第%d组,发生错误:%+v", startTimeStr, endTimeStr, req.TaskID, n, logFields) + log.Errorf("%s到%s,taskId:%d,发生错误:%+v", startTimeStr, endTimeStr, req.TaskID, logFields) if errNum > 20 { return fmt.Errorf("%s到%s,第%d个任务，已经连续发生20次错误%+v", startTimeStr, endTimeStr, req.TaskID, logFields) @@ -79,15 +76,12 @@ func (v *Query) callbackFunc(ctx context.Context, req *timeslice.Task) error { end := time.Now() logFields := map[string]interface{}{ - "任务处理时间": currentStartTimeStr + "到" + currentEndTimeStr, - "总处理组数": n, - "总处理条数": num, - "总通知条数": notifyNum, - "执行任务开始时间": startStr, - "执行任务结束时间": end.Format(time.DateTime), - "总处理耗时": end.Sub(start).String(), + "searchTime": currentStartTimeStr + "到" + currentEndTimeStr, + "num": num, + "notifyNum": notifyNum, + "duration": end.Sub(start).String(), } - log.Warnf("%s到%s,第%d个任务,处理完毕:%+v", startTimeStr, endTimeStr, req.TaskID, logFields) + log.Warnf("%s到%s,taskId:%d,处理完毕:%+v", startTimeStr, endTimeStr, req.TaskID, logFields) return err } diff --git a/internal/pkg/script/script.go b/internal/pkg/script/script.go index cf06e4e..240d57c 100644 --- a/internal/pkg/script/script.go +++ b/internal/pkg/script/script.go @@ -9,6 +9,12 @@ import ( "time" ) +const ( + URL = "http://127.0.0.1:15000/voucher/timeSliceQueryPush" + DEV_URL = "http://open.cszfan.com/voucher/cmb/timeSliceQueryPush" + PRO_URL = "https://voucher.86698.cn/voucher/cmb/timeSliceQueryPush" +) + func script(startTime, endTime time.Time, duration time.Duration) error { // 每指定间隔时间发送一次请求 @@ -29,10 +35,8 @@ func script(startTime, endTime time.Time, duration time.Duration) error { func sendRequest(startTime, endTime time.Time) error { - url := "https://gateway.dev.cdlsxd.cn/voucher/cmb/v1/orderQuery" - // 创建请求体 - requestBody := map[string]interface{}{ + requestBody := map[string]string{ "product_no": "", "start_time": startTime.Format(time.DateTime), "end_time": endTime.Format(time.DateTime), @@ -45,7 +49,7 @@ func sendRequest(startTime, endTime time.Time) error { } // 发送 POST 请求 - resp, err := http.Post(url, "application/json", bytes.NewBuffer(jsonData)) + resp, err := http.Post(URL, "application/json", bytes.NewBuffer(jsonData)) if err != nil { return fmt.Errorf("failed to send POST request: %v", err) } diff --git a/internal/pkg/script/script_test.go b/internal/pkg/script/script_test.go index a557e56..e2fc243 100644 --- a/internal/pkg/script/script_test.go +++ b/internal/pkg/script/script_test.go @@ -13,13 +13,13 @@ func Test_script(t *testing.T) { return } - endTime, err := time.Parse(time.DateTime, "2025-05-31 00:00:00") + endTime, err := time.Parse(time.DateTime, "2025-05-01 10:00:00") if err != nil { t.Error(err) return } - duration := 24 * time.Hour + duration := 5 * time.Hour if err = script(startTime, endTime, duration); err != nil { t.Error(err) diff --git a/internal/service/script.go b/internal/service/script.go index e1ea10f..0a48089 100644 --- a/internal/service/script.go +++ b/internal/service/script.go @@ -95,9 +95,7 @@ func (this *CmbService) TimeSliceQueryPush(ctx http.Context) error { return err } - return ctx.JSON(http2.StatusOK, map[string]interface{}{ - "data": reps, - }) + return ctx.JSON(http2.StatusOK, reps) } func (this *CmbService) PushWechatRetry(ctx http.Context) error { From 343df76e718e2d7ee87caabe45e3cd4f9f5f32d5 Mon Sep 17 00:00:00 2001 From: ziming Date: Thu, 12 Jun 2025 09:24:21 +0800 Subject: [PATCH 12/36] timeSliceQueryPush --- internal/biz/do/rds_mq.go | 9 +++++ internal/biz/timeslicequery/mq.go | 36 +++++++++++------ internal/data/repoimpl/order.go | 3 ++ internal/pkg/script/script.go | 33 ++++++---------- internal/pkg/script/script_test.go | 45 +++++++++++++++++++-- internal/pkg/timeslice/manager.go | 15 ++++--- internal/pkg/timeslice/manager_test.go | 54 +++++++++++++------------- internal/pkg/timeslice/model.go | 18 ++++++++- internal/service/script.go | 2 +- 9 files changed, 142 insertions(+), 73 deletions(-) diff --git a/internal/biz/do/rds_mq.go b/internal/biz/do/rds_mq.go index b89b5a7..6efb705 100644 --- a/internal/biz/do/rds_mq.go +++ b/internal/biz/do/rds_mq.go @@ -4,4 +4,13 @@ type WechatQuery struct { ProductNo string `json:"product_no"` StartTime string `json:"start_time"` EndTime string `json:"end_time"` + OrderNo string `json:"order_no"` +} + +type RdsWechatQuery struct { + ProductNo string `json:"product_no"` + StartTime string `json:"start_time"` + EndTime string `json:"end_time"` + GoNum int `json:"go_num"` // 并发数 + TimeSliceHours int64 `json:"time_slice_hours"` // 时间片"小时" } diff --git a/internal/biz/timeslicequery/mq.go b/internal/biz/timeslicequery/mq.go index 6e0c171..3fb2fcc 100644 --- a/internal/biz/timeslicequery/mq.go +++ b/internal/biz/timeslicequery/mq.go @@ -11,7 +11,7 @@ import ( "voucher/internal/pkg/timeslice" ) -func (v *Query) Push(ctx http.Context, req *do.WechatQuery) (string, error) { +func (v *Query) Push(ctx http.Context, req *do.RdsWechatQuery) (string, error) { if req.StartTime == "" || req.EndTime == "" { return "", fmt.Errorf("时间参数不能为空") @@ -60,9 +60,9 @@ func (v *Query) Push(ctx http.Context, req *do.WechatQuery) (string, error) { return strMsg, nil } -func (v *Query) getManager(_ context.Context, msg string) (*timeslice.Manager, error) { +func (v *Query) getManager(msg string) (*timeslice.Manager, error) { - var req *do.WechatQuery + var req *do.RdsWechatQuery if err := json.Unmarshal([]byte(msg), &req); err != nil { return nil, err @@ -81,29 +81,41 @@ func (v *Query) getManager(_ context.Context, msg string) (*timeslice.Manager, e return nil, err } - return ×lice.Manager{ + m := ×lice.Manager{ StartTime: start, EndTime: end, ProductNo: req.ProductNo, - GoNum: 2, // 协程数量 - TimeSliceHours: 2, // 时间间隔 - }, nil + GoNum: timeslice.DefaultGoNum, // 协程数量 + TimeSliceHours: timeslice.DefaultTimeSliceHours, // 时间间隔 + } + + if req.GoNum > 0 { + m.GoNum = req.GoNum + } + + if req.TimeSliceHours > 0 { + m.TimeSliceHours = req.TimeSliceHours + } + + return m, nil } func (v *Query) Consumer(ctx context.Context, msg string) error { defer v.Remove(v.uid(msg)) - req, err := v.getManager(ctx, msg) + req, err := v.getManager(msg) if err != nil { return err } + reqStr := req.String() + executeStart := time.Now() executeStartStr := executeStart.Format(time.DateTime) - log.Warnf("微信券查询处理开始:%s,msg:%s", executeStartStr, msg) - fmt.Printf("微信券查询处理开始:%s,msg:%s", executeStartStr, msg) + log.Warnf("微信券查询处理开始:%s,manager:%s", executeStartStr, reqStr) + fmt.Printf("微信券查询处理开始:%s,manager:%s", executeStartStr, reqStr) if err = v.execute(ctx, req); err != nil { log.Errorf("微信券查询处理失败:%s,msg:%s,err:%v", executeStartStr, msg, err) @@ -111,8 +123,8 @@ func (v *Query) Consumer(ctx context.Context, msg string) error { } executeEnd := time.Now() - log.Warnf("微信券查询处理耗时:%s,结束时间%s,msg:%s", executeEnd.Sub(executeStart).String(), executeEnd.String(), msg) - fmt.Printf("微信券查询处理耗时:%s,结束时间%s,msg:%s", executeEnd.Sub(executeStart).String(), executeEnd.String(), msg) + log.Warnf("微信券查询处理耗时:%s,结束时间%s,manager:%s", executeEnd.Sub(executeStart).String(), executeEnd.String(), reqStr) + fmt.Printf("微信券查询处理耗时:%s,结束时间%s,manager:%s", executeEnd.Sub(executeStart).String(), executeEnd.String(), reqStr) return nil } diff --git a/internal/data/repoimpl/order.go b/internal/data/repoimpl/order.go index f4dfea0..713dd57 100644 --- a/internal/data/repoimpl/order.go +++ b/internal/data/repoimpl/order.go @@ -44,6 +44,9 @@ func (p *OrderRepoImpl) FinSucByStockIdInBatches(ctx context.Context, req *do.We if req.EndTime != "" { tx = tx.Where("receive_success_time <= ?", req.EndTime) } + if req.OrderNo != "" { + tx = tx.Where("order_no = ?", req.OrderNo) + } var results = make([]*model.Order, 0) diff --git a/internal/pkg/script/script.go b/internal/pkg/script/script.go index 240d57c..b6258b6 100644 --- a/internal/pkg/script/script.go +++ b/internal/pkg/script/script.go @@ -2,7 +2,6 @@ package script import ( "bytes" - "encoding/json" "fmt" "io" "net/http" @@ -15,14 +14,20 @@ const ( PRO_URL = "https://voucher.86698.cn/voucher/cmb/timeSliceQueryPush" ) -func script(startTime, endTime time.Time, duration time.Duration) error { +const ( + SINGLE_URL = "http://127.0.0.1:15000/voucher/pushWechatQuery" + DEV_SINGLE_URL = "http://open.cszfan.com/voucher/cmb/pushWechatQuery" + PRO_SINGLE_URL = "https://voucher.86698.cn/voucher/cmb/pushWechatQuery" +) + +func script(startTime, endTime time.Time, duration time.Duration, body []byte, URL string) error { // 每指定间隔时间发送一次请求 for t := startTime; t.Before(endTime); t = t.Add(duration) { end := t.Add(duration) // 计算每次请求的结束时间 // 发送请求 - if err := sendRequest(t, end); err != nil { + if err := sendRequest(t, end, body, URL); err != nil { fmt.Printf("Error sending request: %v\n", err) } @@ -33,23 +38,9 @@ func script(startTime, endTime time.Time, duration time.Duration) error { return nil } -func sendRequest(startTime, endTime time.Time) error { +func sendRequest(startTime, endTime time.Time, body []byte, URL string) error { - // 创建请求体 - requestBody := map[string]string{ - "product_no": "", - "start_time": startTime.Format(time.DateTime), - "end_time": endTime.Format(time.DateTime), - } - - // 将请求体转换为 JSON 格式 - jsonData, err := json.Marshal(requestBody) - if err != nil { - return fmt.Errorf("failed to marshal JSON: %v", err) - } - - // 发送 POST 请求 - resp, err := http.Post(URL, "application/json", bytes.NewBuffer(jsonData)) + resp, err := http.Post(URL, "application/json", bytes.NewBuffer(body)) if err != nil { return fmt.Errorf("failed to send POST request: %v", err) } @@ -62,9 +53,7 @@ func sendRequest(startTime, endTime time.Time) error { if resp.StatusCode == http.StatusOK { fmt.Printf("Request sent successfully,body:%s", string(bodyBytes)) - } else { - return fmt.Errorf("failed with status code: %d", resp.StatusCode) } - return nil + return fmt.Errorf("failed with status code: %d", resp.StatusCode) } diff --git a/internal/pkg/script/script_test.go b/internal/pkg/script/script_test.go index e2fc243..1b35f8d 100644 --- a/internal/pkg/script/script_test.go +++ b/internal/pkg/script/script_test.go @@ -1,6 +1,7 @@ package script import ( + "encoding/json" "testing" "time" ) @@ -21,7 +22,24 @@ func Test_script(t *testing.T) { duration := 5 * time.Hour - if err = script(startTime, endTime, duration); err != nil { + // 创建请求体 + requestBody := map[string]any{ + "go_num": 2, // 并发数量 + "time_slice_hours": 2, // 时间间隔 + "order_no": "", + "product_no": "", + "start_time": startTime.Format(time.DateTime), + "end_time": endTime.Format(time.DateTime), + } + + // 将请求体转换为 JSON 格式 + bodyBytes, err := json.Marshal(requestBody) + if err != nil { + t.Error(err) + return + } + + if err = script(startTime, endTime, duration, bodyBytes, URL); err != nil { t.Error(err) } } @@ -34,11 +52,30 @@ func Test_script2(t *testing.T) { return } - endTime := time.Now() + endTime, err := time.Parse(time.DateTime, "2025-05-01 10:00:00") + if err != nil { + t.Error(err) + return + } - duration := 1 * time.Hour + duration := 50 * time.Hour - if err = script(startTime, endTime, duration); err != nil { + // 创建请求体 + requestBody := map[string]any{ + "order_no": "", + "product_no": "", + "start_time": startTime.Format(time.DateTime), + "end_time": endTime.Format(time.DateTime), + } + + // 将请求体转换为 JSON 格式 + bodyBytes, err := json.Marshal(requestBody) + if err != nil { + t.Error(err) + return + } + + if err = script(startTime, endTime, duration, bodyBytes, SINGLE_URL); err != nil { t.Error(err) } } diff --git a/internal/pkg/timeslice/manager.go b/internal/pkg/timeslice/manager.go index 4e3e8a5..6ca40b0 100644 --- a/internal/pkg/timeslice/manager.go +++ b/internal/pkg/timeslice/manager.go @@ -10,8 +10,9 @@ import ( ) const ( - TimeSliceHours = 2 - maxGlobalGoroutines = 1000 + DefaultGoNum = 2 + DefaultTimeSliceHours = 2 + maxGlobalGoroutines = 1000 ) type Callback func(ctx context.Context, req *Task) error @@ -37,11 +38,13 @@ func (m *ManagerSrv) Run(ctx context.Context, req *Manager) (int, error) { return 0, fmt.Errorf("协程数量不能大于%d", maxGlobalGoroutines) } + timeSliceHours := float64(req.TimeSliceHours) + totalHours := req.EndTime.Sub(req.StartTime).Hours() - taskCount := int(totalHours / TimeSliceHours) + taskCount := int(totalHours / timeSliceHours) // 如果剩余时间不足 TimeSliceHours 小时，增加任务数 - if totalHours-float64(taskCount)*float64(TimeSliceHours) > 0 { + if totalHours-float64(taskCount)*timeSliceHours > 0 { taskCount++ } @@ -69,8 +72,8 @@ func (m *ManagerSrv) process(ctx context.Context, req *Process) error { // 为每个任务按指定的时间片 TimeSliceHours 分配开始和结束时间 for i := 0; i < req.TaskCount; i++ { - currentStart := req.Manager.StartTime.Add(time.Duration(i) * TimeSliceHours * time.Hour) - currentEnd := currentStart.Add(TimeSliceHours * time.Hour) + currentStart := req.Manager.StartTime.Add(time.Duration(i) * time.Duration(req.Manager.TimeSliceHours) * time.Hour) + currentEnd := currentStart.Add(time.Duration(req.Manager.TimeSliceHours) * time.Hour) if currentEnd.After(req.Manager.EndTime) { currentEnd = req.Manager.EndTime diff --git a/internal/pkg/timeslice/manager_test.go b/internal/pkg/timeslice/manager_test.go index 2cda4ae..7600515 100644 --- a/internal/pkg/timeslice/manager_test.go +++ b/internal/pkg/timeslice/manager_test.go @@ -60,10 +60,11 @@ func TestNewManager(t *testing.T) { srv := NewManager(callback) taskCount, err := srv.Run(context.Background(), &Manager{ - StartTime: start, - EndTime: end, - ProductNo: "no123456", - GoNum: 2, + StartTime: start, + EndTime: end, + ProductNo: "no123456", + GoNum: 2, + TimeSliceHours: 2, }) // 输出结果 @@ -95,7 +96,7 @@ func TestBatchCallBackFunc(t *testing.T) { t.Fatalf("%v", err) return } - end, err := time.Parse(time.DateTime, "2025-01-02 02:00:01") + end, err := time.Parse(time.DateTime, "2025-01-01 05:00:01") if err != nil { t.Fatalf("%v", err) return @@ -121,7 +122,7 @@ func TestBatchCallBackFunc(t *testing.T) { next = end } - t.Logf("处理时间: %s到%s", current.Format(time.DateTime), next.Format(time.DateTime)) + t.Logf("\n处理时间: %s到%s", current.Format(time.DateTime), next.Format(time.DateTime)) wg.Add(1) // 启动goroutine处理每个时间段 @@ -150,10 +151,11 @@ func CallbackFunc(start, end time.Time) error { managerEndStr := end.Format(time.DateTime) req := &Manager{ - StartTime: start, - EndTime: end, - ProductNo: "no123456", - GoNum: 2, + StartTime: start, + EndTime: end, + ProductNo: "no123456", + GoNum: 2, + TimeSliceHours: 1, } taskCount, err := NewManager(callbackFunc).Run(context.Background(), req) @@ -188,8 +190,8 @@ func callbackFunc(_ context.Context, req *Task) error { start := time.Now() startStr := start.Format(time.DateTime) - n := 0 - num := 0 + groupNum := 0 + allNum := 0 notifyNum := 0 for i := 0; i < 3; i++ { @@ -198,34 +200,32 @@ func callbackFunc(_ context.Context, req *Task) error { // 模拟任务执行，休眠随机时间 time.Sleep(time.Duration(rand.Intn(3)) * time.Second) - n += 1 - num = num + (i+1)*100 + groupNum += 1 + allNum = allNum + (i+1)*100 logFields := map[string]interface{}{ - "处理条数": num, - "通知条数": notifyNum, - "耗时": time.Now().Sub(groupStartTime).String(), - "任务处理开始时间": currentStartTimeStr, - "任务处理结束时间": currentEndTimeStr, + "duration": time.Now().Sub(groupStartTime).String(), + "searchTime": currentStartTimeStr + "到" + currentEndTimeStr, } fmt.Printf("%s到%s,第%d个任务，第%d组,处理完毕, %+v\n", managerStartTimeStr, managerEndTimeStr, req.TaskID, i+1, logFields) } end := time.Now() logFields := map[string]interface{}{ - "总处理组数": n, - "总处理条数": num, - "总通知条数": notifyNum, - "执行任务开始时间": startStr, - "执行任务结束时间": end.Format(time.DateTime), - "总处理耗时": end.Sub(start).String(), - "任务处理开始时间": currentStartTimeStr, - "任务处理结束时间": currentEndTimeStr, + "groupNum": groupNum, + "allNum": allNum, + "startTime": startStr, + "endTime": end.Format(time.DateTime), + "searchTime": currentStartTimeStr + "到" + currentEndTimeStr, + "notifyNum": notifyNum, + "duration": end.Sub(start).String(), } // 生成任务执行结果 result := fmt.Sprintf("%s到%s，第%d个任务,处理完毕,%+v\n", managerStartTimeStr, managerEndTimeStr, req.TaskID, logFields) + fmt.Printf(result) + //return fmt.Errorf(result) return nil } diff --git a/internal/pkg/timeslice/model.go b/internal/pkg/timeslice/model.go index 35dfb12..30c8173 100644 --- a/internal/pkg/timeslice/model.go +++ b/internal/pkg/timeslice/model.go @@ -1,6 +1,7 @@ package timeslice import ( + "encoding/json" "time" ) @@ -9,7 +10,12 @@ type Manager struct { EndTime time.Time // 结束时间 ProductNo string // 产品编号 GoNum int // 并发数 - TimeSliceHours int // 时间片"小时" + TimeSliceHours int64 // 时间片"小时" +} + +func (m *Manager) String() string { + b, _ := json.Marshal(m) + return string(b) } type Process struct { @@ -17,9 +23,19 @@ type Process struct { TaskCount int // 任务数 } +func (m *Process) String() string { + b, _ := json.Marshal(m) + return string(b) +} + type Task struct { Process *Process CurrentStartTime time.Time // 时间片开始时间 CurrentEndTime time.Time // 时间片结束时间 TaskID int // 任务ID } + +func (m *Task) String() string { + b, _ := json.Marshal(m) + return string(b) +} diff --git a/internal/service/script.go b/internal/service/script.go index 0a48089..5f05ccc 100644 --- a/internal/service/script.go +++ b/internal/service/script.go @@ -77,7 +77,7 @@ func (this *CmbService) TimeSliceQueryPush(ctx http.Context) error { return err } - var req *do.WechatQuery + var req *do.RdsWechatQuery if err = json.Unmarshal(bodyBytes, &req); err != nil { return err } From d348f247fed91a8762fe46b9c0b04b856fdd5433 Mon Sep 17 00:00:00 2001 From: ziming Date: Thu, 12 Jun 2025 09:37:25 +0800 Subject: [PATCH 13/36] timeSliceQueryPush --- internal/biz/timeslicequery/execute.go | 12 ++++++------ internal/biz/timeslicequery/mq.go | 16 ++++++++-------- internal/pkg/script/script_test.go | 3 +-- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/internal/biz/timeslicequery/execute.go b/internal/biz/timeslicequery/execute.go index b33acdb..3133b69 100644 --- a/internal/biz/timeslicequery/execute.go +++ b/internal/biz/timeslicequery/execute.go @@ -17,11 +17,11 @@ func (v *Query) execute(ctx context.Context, req *timeslice.Manager) error { taskCount, err := timeslice.NewManager(v.callbackFunc).Run(ctx, req) if err != nil { - log.Errorf("%s到%s,发生错误:%v", managerStartStr, managerEndStr, err) + log.Errorf("%s到%s,微信券查询处理失败:%v", managerStartStr, managerEndStr, err) } - fmt.Printf("%s到%s,总任务数:%d\n", managerStartStr, managerEndStr, taskCount) - log.Warnf("%s到%s,总任务数:%d", managerStartStr, managerEndStr, taskCount) + fmt.Printf("%s到%s,微信券查询处理,总任务数:%d\n", managerStartStr, managerEndStr, taskCount) + log.Warnf("%s到%s,微信券查询处理,总任务数:%d", managerStartStr, managerEndStr, taskCount) return nil } @@ -61,10 +61,10 @@ func (v *Query) callbackFunc(ctx context.Context, req *timeslice.Task) error { "open_id": order.Account, "err": err.Error(), } - log.Errorf("%s到%s,taskId:%d,发生错误:%+v", startTimeStr, endTimeStr, req.TaskID, logFields) + log.Errorf("%s到%s,taskId:%d,微信券查询处理发生错误:%+v", startTimeStr, endTimeStr, req.TaskID, logFields) if errNum > 20 { - return fmt.Errorf("%s到%s,第%d个任务，已经连续发生20次错误%+v", startTimeStr, endTimeStr, req.TaskID, logFields) + return fmt.Errorf("%s到%s,第%d个任务，微信券查询处理,已经连续发生20次错误%+v", startTimeStr, endTimeStr, req.TaskID, logFields) } } @@ -81,7 +81,7 @@ func (v *Query) callbackFunc(ctx context.Context, req *timeslice.Task) error { "notifyNum": notifyNum, "duration": end.Sub(start).String(), } - log.Warnf("%s到%s,taskId:%d,处理完毕:%+v", startTimeStr, endTimeStr, req.TaskID, logFields) + log.Warnf("%s到%s,taskId:%d,微信券查询处理,处理完毕:%+v", startTimeStr, endTimeStr, req.TaskID, logFields) return err } diff --git a/internal/biz/timeslicequery/mq.go b/internal/biz/timeslicequery/mq.go index 3fb2fcc..24086c2 100644 --- a/internal/biz/timeslicequery/mq.go +++ b/internal/biz/timeslicequery/mq.go @@ -81,7 +81,7 @@ func (v *Query) getManager(msg string) (*timeslice.Manager, error) { return nil, err } - m := ×lice.Manager{ + manager := ×lice.Manager{ StartTime: start, EndTime: end, ProductNo: req.ProductNo, @@ -90,14 +90,14 @@ func (v *Query) getManager(msg string) (*timeslice.Manager, error) { } if req.GoNum > 0 { - m.GoNum = req.GoNum + manager.GoNum = req.GoNum } if req.TimeSliceHours > 0 { - m.TimeSliceHours = req.TimeSliceHours + manager.TimeSliceHours = req.TimeSliceHours } - return m, nil + return manager, nil } func (v *Query) Consumer(ctx context.Context, msg string) error { @@ -114,8 +114,8 @@ func (v *Query) Consumer(ctx context.Context, msg string) error { executeStart := time.Now() executeStartStr := executeStart.Format(time.DateTime) - log.Warnf("微信券查询处理开始:%s,manager:%s", executeStartStr, reqStr) - fmt.Printf("微信券查询处理开始:%s,manager:%s", executeStartStr, reqStr) + log.Warnf("微信券查询处理开始:%s,msg:%s,manager%s", executeStartStr, msg, reqStr) + fmt.Printf("微信券查询处理开始:%s,msg:%s,manager%s", executeStartStr, msg, reqStr) if err = v.execute(ctx, req); err != nil { log.Errorf("微信券查询处理失败:%s,msg:%s,err:%v", executeStartStr, msg, err) @@ -123,8 +123,8 @@ func (v *Query) Consumer(ctx context.Context, msg string) error { } executeEnd := time.Now() - log.Warnf("微信券查询处理耗时:%s,结束时间%s,manager:%s", executeEnd.Sub(executeStart).String(), executeEnd.String(), reqStr) - fmt.Printf("微信券查询处理耗时:%s,结束时间%s,manager:%s", executeEnd.Sub(executeStart).String(), executeEnd.String(), reqStr) + log.Warnf("微信券查询处理耗时:%s,结束时间%s,manager%s", executeEnd.Sub(executeStart).String(), executeEnd.String(), reqStr) + fmt.Printf("微信券查询处理耗时:%s,结束时间%s,manager%s", executeEnd.Sub(executeStart).String(), executeEnd.String(), reqStr) return nil } diff --git a/internal/pkg/script/script_test.go b/internal/pkg/script/script_test.go index 1b35f8d..19c7850 100644 --- a/internal/pkg/script/script_test.go +++ b/internal/pkg/script/script_test.go @@ -14,7 +14,7 @@ func Test_script(t *testing.T) { return } - endTime, err := time.Parse(time.DateTime, "2025-05-01 10:00:00") + endTime, err := time.Parse(time.DateTime, "2025-05-01 5:00:00") if err != nil { t.Error(err) return @@ -26,7 +26,6 @@ func Test_script(t *testing.T) { requestBody := map[string]any{ "go_num": 2, // 并发数量 "time_slice_hours": 2, // 时间间隔 - "order_no": "", "product_no": "", "start_time": startTime.Format(time.DateTime), "end_time": endTime.Format(time.DateTime), From b366cadb9ddc0213914c7aecd264161d1cd65297 Mon Sep 17 00:00:00 2001 From: ziming Date: Thu, 12 Jun 2025 10:17:55 +0800 Subject: [PATCH 14/36] timeSliceQueryPush --- internal/pkg/script/script.go | 72 +++++++++++++++++++++++++----- internal/pkg/script/script_test.go | 39 +++------------- 2 files changed, 68 insertions(+), 43 deletions(-) diff --git a/internal/pkg/script/script.go b/internal/pkg/script/script.go index b6258b6..ca79b75 100644 --- a/internal/pkg/script/script.go +++ b/internal/pkg/script/script.go @@ -2,6 +2,7 @@ package script import ( "bytes" + "encoding/json" "fmt" "io" "net/http" @@ -10,25 +11,42 @@ import ( const ( URL = "http://127.0.0.1:15000/voucher/timeSliceQueryPush" - DEV_URL = "http://open.cszfan.com/voucher/cmb/timeSliceQueryPush" - PRO_URL = "https://voucher.86698.cn/voucher/cmb/timeSliceQueryPush" + DEV_URL = "http://open.cszfan.com/voucher/timeSliceQueryPush" + PRO_URL = "https://voucher.86698.cn/voucher/timeSliceQueryPush" ) const ( - SINGLE_URL = "http://127.0.0.1:15000/voucher/pushWechatQuery" - DEV_SINGLE_URL = "http://open.cszfan.com/voucher/cmb/pushWechatQuery" - PRO_SINGLE_URL = "https://voucher.86698.cn/voucher/cmb/pushWechatQuery" + SINGLE_URL = "http://127.0.0.1:15000//voucher/pushWechatQuery" + DEV_SINGLE_URL = "http://open.cszfan.com//voucher/pushWechatQuery" + PRO_SINGLE_URL = "https://voucher.86698.cn//voucher/pushWechatQuery" ) -func script(startTime, endTime time.Time, duration time.Duration, body []byte, URL string) error { +func timeSliceQueryPush(startTime, endTime time.Time, duration time.Duration, requestURL string) error { // 每指定间隔时间发送一次请求 for t := startTime; t.Before(endTime); t = t.Add(duration) { end := t.Add(duration) // 计算每次请求的结束时间 + // 创建请求体 + requestBody := map[string]any{ + "go_num": 2, // 并发数量 + "time_slice_hours": 1, // 时间间隔 + "product_no": "", + "start_time": t.Format(time.DateTime), + "end_time": end.Format(time.DateTime), + } + + // 将请求体转换为 JSON 格式 + bodyBytes, err := json.Marshal(requestBody) + if err != nil { + return err + } + + fmt.Printf("body:%s\n", string(bodyBytes)) + // 发送请求 - if err := sendRequest(t, end, body, URL); err != nil { - fmt.Printf("Error sending request: %v\n", err) + if err2 := sendRequest(bodyBytes, requestURL); err2 != nil { + fmt.Printf("Error sending request: %v\n", err2) } // 等待一段时间后再发送下一个请求 @@ -38,9 +56,43 @@ func script(startTime, endTime time.Time, duration time.Duration, body []byte, U return nil } -func sendRequest(startTime, endTime time.Time, body []byte, URL string) error { +func pushWechatQuery(startTime, endTime time.Time, duration time.Duration, requestURL string) error { - resp, err := http.Post(URL, "application/json", bytes.NewBuffer(body)) + // 每指定间隔时间发送一次请求 + for t := startTime; t.Before(endTime); t = t.Add(duration) { + end := t.Add(duration) // 计算每次请求的结束时间 + + // 创建请求体 + requestBody := map[string]any{ + "order_no": "", + "product_no": "", + "start_time": t.Format(time.DateTime), + "end_time": end.Format(time.DateTime), + } + + // 将请求体转换为 JSON 格式 + bodyBytes, err := json.Marshal(requestBody) + if err != nil { + return err + } + + fmt.Printf("body:%s\n", string(bodyBytes)) + + // 发送请求 + if err2 := sendRequest(bodyBytes, requestURL); err2 != nil { + fmt.Printf("Error sending request: %v\n", err2) + } + + // 等待一段时间后再发送下一个请求 + time.Sleep(1 * time.Second) // 可以根据需要调整间隔时间 + } + + return nil +} + +func sendRequest(body []byte, requestURL string) error { + + resp, err := http.Post(requestURL, "application/json", bytes.NewBuffer(body)) if err != nil { return fmt.Errorf("failed to send POST request: %v", err) } diff --git a/internal/pkg/script/script_test.go b/internal/pkg/script/script_test.go index 19c7850..55b86d2 100644 --- a/internal/pkg/script/script_test.go +++ b/internal/pkg/script/script_test.go @@ -1,7 +1,6 @@ package script import ( - "encoding/json" "testing" "time" ) @@ -14,7 +13,7 @@ func Test_script(t *testing.T) { return } - endTime, err := time.Parse(time.DateTime, "2025-05-01 5:00:00") + endTime, err := time.Parse(time.DateTime, "2025-05-01 2:00:03") if err != nil { t.Error(err) return @@ -22,23 +21,10 @@ func Test_script(t *testing.T) { duration := 5 * time.Hour - // 创建请求体 - requestBody := map[string]any{ - "go_num": 2, // 并发数量 - "time_slice_hours": 2, // 时间间隔 - "product_no": "", - "start_time": startTime.Format(time.DateTime), - "end_time": endTime.Format(time.DateTime), - } + //requestUrl := URL + requestUrl := DEV_URL - // 将请求体转换为 JSON 格式 - bodyBytes, err := json.Marshal(requestBody) - if err != nil { - t.Error(err) - return - } - - if err = script(startTime, endTime, duration, bodyBytes, URL); err != nil { + if err = timeSliceQueryPush(startTime, endTime, duration, requestUrl); err != nil { t.Error(err) } } @@ -59,22 +45,9 @@ func Test_script2(t *testing.T) { duration := 50 * time.Hour - // 创建请求体 - requestBody := map[string]any{ - "order_no": "", - "product_no": "", - "start_time": startTime.Format(time.DateTime), - "end_time": endTime.Format(time.DateTime), - } + requestUrl := SINGLE_URL - // 将请求体转换为 JSON 格式 - bodyBytes, err := json.Marshal(requestBody) - if err != nil { - t.Error(err) - return - } - - if err = script(startTime, endTime, duration, bodyBytes, SINGLE_URL); err != nil { + if err = pushWechatQuery(startTime, endTime, duration, requestUrl); err != nil { t.Error(err) } } From 4847e4eced2b7883b4b543066c887c380d67dfac Mon Sep 17 00:00:00 2001 From: ziming Date: Thu, 12 Jun 2025 10:23:58 +0800 Subject: [PATCH 15/36] timeSliceQueryPush --- internal/biz/timeslicequery/execute.go | 10 +++++----- internal/pkg/script/script.go | 4 ++-- internal/pkg/script/script_test.go | 2 +- internal/service/script.go | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/internal/biz/timeslicequery/execute.go b/internal/biz/timeslicequery/execute.go index 3133b69..0037332 100644 --- a/internal/biz/timeslicequery/execute.go +++ b/internal/biz/timeslicequery/execute.go @@ -17,11 +17,11 @@ func (v *Query) execute(ctx context.Context, req *timeslice.Manager) error { taskCount, err := timeslice.NewManager(v.callbackFunc).Run(ctx, req) if err != nil { - log.Errorf("%s到%s,微信券查询处理失败:%v", managerStartStr, managerEndStr, err) + log.Errorf("微信券查询处理,%s到%s,失败:%v", managerStartStr, managerEndStr, err) } fmt.Printf("%s到%s,微信券查询处理,总任务数:%d\n", managerStartStr, managerEndStr, taskCount) - log.Warnf("%s到%s,微信券查询处理,总任务数:%d", managerStartStr, managerEndStr, taskCount) + log.Warnf("微信券查询处理,%s到%s,总任务数:%d", managerStartStr, managerEndStr, taskCount) return nil } @@ -61,10 +61,10 @@ func (v *Query) callbackFunc(ctx context.Context, req *timeslice.Task) error { "open_id": order.Account, "err": err.Error(), } - log.Errorf("%s到%s,taskId:%d,微信券查询处理发生错误:%+v", startTimeStr, endTimeStr, req.TaskID, logFields) + log.Errorf("微信券查询处理,%s到%s,taskId:%d,错误:%+v", startTimeStr, endTimeStr, req.TaskID, logFields) if errNum > 20 { - return fmt.Errorf("%s到%s,第%d个任务，微信券查询处理,已经连续发生20次错误%+v", startTimeStr, endTimeStr, req.TaskID, logFields) + return fmt.Errorf("微信券查询处理,%s到%s,第%d个任务,已经连续发生20次错误%+v", startTimeStr, endTimeStr, req.TaskID, logFields) } } @@ -81,7 +81,7 @@ func (v *Query) callbackFunc(ctx context.Context, req *timeslice.Task) error { "notifyNum": notifyNum, "duration": end.Sub(start).String(), } - log.Warnf("%s到%s,taskId:%d,微信券查询处理,处理完毕:%+v", startTimeStr, endTimeStr, req.TaskID, logFields) + log.Warnf("微信券查询处理,%s到%s,taskId:%d,处理完毕:%+v", startTimeStr, endTimeStr, req.TaskID, logFields) return err } diff --git a/internal/pkg/script/script.go b/internal/pkg/script/script.go index ca79b75..9774a02 100644 --- a/internal/pkg/script/script.go +++ b/internal/pkg/script/script.go @@ -76,7 +76,7 @@ func pushWechatQuery(startTime, endTime time.Time, duration time.Duration, reque return err } - fmt.Printf("body:%s\n", string(bodyBytes)) + fmt.Printf("requestBody:%s\n", string(bodyBytes)) // 发送请求 if err2 := sendRequest(bodyBytes, requestURL); err2 != nil { @@ -104,7 +104,7 @@ func sendRequest(body []byte, requestURL string) error { } if resp.StatusCode == http.StatusOK { - fmt.Printf("Request sent successfully,body:%s", string(bodyBytes)) + fmt.Printf("responsBody:%s", string(bodyBytes)) } return fmt.Errorf("failed with status code: %d", resp.StatusCode) diff --git a/internal/pkg/script/script_test.go b/internal/pkg/script/script_test.go index 55b86d2..c5f7bee 100644 --- a/internal/pkg/script/script_test.go +++ b/internal/pkg/script/script_test.go @@ -43,7 +43,7 @@ func Test_script2(t *testing.T) { return } - duration := 50 * time.Hour + duration := 1 * time.Hour requestUrl := SINGLE_URL diff --git a/internal/service/script.go b/internal/service/script.go index 5f05ccc..c1cb2f0 100644 --- a/internal/service/script.go +++ b/internal/service/script.go @@ -90,12 +90,12 @@ func (this *CmbService) TimeSliceQueryPush(ctx http.Context) error { return fmt.Errorf("start_time or end_time is empty") } - reps, err := this.timeSliceQuery.Push(ctx, req) + _, err = this.timeSliceQuery.Push(ctx, req) if err != nil { return err } - return ctx.JSON(http2.StatusOK, reps) + return ctx.JSON(http2.StatusOK, req) } func (this *CmbService) PushWechatRetry(ctx http.Context) error { From 540d873eacb470bbbf88111f6d7c9350259add9c Mon Sep 17 00:00:00 2001 From: ziming Date: Thu, 12 Jun 2025 10:31:57 +0800 Subject: [PATCH 16/36] timeSliceQueryPush --- internal/pkg/script/script.go | 8 ++++++++ internal/pkg/script/script_test.go | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/internal/pkg/script/script.go b/internal/pkg/script/script.go index 9774a02..6bc68ce 100644 --- a/internal/pkg/script/script.go +++ b/internal/pkg/script/script.go @@ -25,7 +25,11 @@ func timeSliceQueryPush(startTime, endTime time.Time, duration time.Duration, re // 每指定间隔时间发送一次请求 for t := startTime; t.Before(endTime); t = t.Add(duration) { + end := t.Add(duration) // 计算每次请求的结束时间 + if end.After(endTime) { + end = endTime + } // 创建请求体 requestBody := map[string]any{ @@ -60,7 +64,11 @@ func pushWechatQuery(startTime, endTime time.Time, duration time.Duration, reque // 每指定间隔时间发送一次请求 for t := startTime; t.Before(endTime); t = t.Add(duration) { + end := t.Add(duration) // 计算每次请求的结束时间 + if end.After(endTime) { + end = endTime + } // 创建请求体 requestBody := map[string]any{ diff --git a/internal/pkg/script/script_test.go b/internal/pkg/script/script_test.go index c5f7bee..8fa5d22 100644 --- a/internal/pkg/script/script_test.go +++ b/internal/pkg/script/script_test.go @@ -13,7 +13,7 @@ func Test_script(t *testing.T) { return } - endTime, err := time.Parse(time.DateTime, "2025-05-01 2:00:03") + endTime, err := time.Parse(time.DateTime, "2025-05-01 02:00:03") if err != nil { t.Error(err) return From 48d03c16cee20f2377094794ee05b5f7af3d953a Mon Sep 17 00:00:00 2001 From: ziming Date: Thu, 12 Jun 2025 10:57:20 +0800 Subject: [PATCH 17/36] timeSliceQueryPush --- internal/biz/bo/order_bo.go | 1 - internal/biz/cron_notice.go | 29 +++++++- internal/data/repoimpl/order.go | 1 - internal/pkg/script/script_test.go | 104 +++++++++++++++++++++++++++++ internal/service/script.go | 4 ++ 5 files changed, 136 insertions(+), 3 deletions(-) diff --git a/internal/biz/bo/order_bo.go b/internal/biz/bo/order_bo.go index 753c706..7643f14 100644 --- a/internal/biz/bo/order_bo.go +++ b/internal/biz/bo/order_bo.go @@ -42,7 +42,6 @@ type OrderCreateReqBo struct { } type FindInBatchesUseBo struct { - Type vo.OrderType StartTime *time.Time EndTime *time.Time } diff --git a/internal/biz/cron_notice.go b/internal/biz/cron_notice.go index 605cb34..1bcb91c 100644 --- a/internal/biz/cron_notice.go +++ b/internal/biz/cron_notice.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/go-kratos/kratos/v2/log" "github.com/redis/go-redis/v9" + "golang.org/x/sync/errgroup" "time" "voucher/internal/biz/bo" "voucher/internal/biz/vo" @@ -30,7 +31,6 @@ func (v *VoucherBiz) Notice(ctx context.Context) error { endTime := time.Date(noticeEndDay.Year(), noticeEndDay.Month(), noticeEndDay.Day(), 23, 59, 59, 0, noticeEndDay.Location()) req := &bo.FindInBatchesUseBo{ - Type: vo.OrderTypeCmb, StartTime: &startTime, EndTime: &endTime, } @@ -38,6 +38,33 @@ func (v *VoucherBiz) Notice(ctx context.Context) error { return v.ExecuteNotice(ctx, req) } +func (v *VoucherBiz) timeSliceQueryPush(ctx context.Context, startTime, endTime time.Time) error { + + duration := 1 * time.Hour + + eg := new(errgroup.Group) + eg.SetLimit(5) + + for start := startTime; start.Before(endTime); start = start.Add(duration) { + + end := start.Add(duration) // 计算每次请求的结束时间 + if end.After(endTime) { + end = endTime + } + + req := &bo.FindInBatchesUseBo{ + StartTime: &start, + EndTime: &end, + } + + eg.Go(func() error { + return v.ExecuteNotice(ctx, req) + }) + } + + return eg.Wait() // 仅返回第一个错误 +} + func (v *VoucherBiz) ExecuteNotice(ctx context.Context, req *bo.FindInBatchesUseBo) error { return v.OrderRepo.FindInBatches(ctx, req, func(ctx context.Context, rows []*bo.OrderBo) error { diff --git a/internal/data/repoimpl/order.go b/internal/data/repoimpl/order.go index 713dd57..eab59c3 100644 --- a/internal/data/repoimpl/order.go +++ b/internal/data/repoimpl/order.go @@ -69,7 +69,6 @@ func (p *OrderRepoImpl) FinFailByStockIdInBatches(ctx context.Context, batchNo s result := p.DB(ctx). Where("batch_no = ?", batchNo). Where("status = ?", vo.OrderStatusFail.GetValue()). - //Where("remark <> ?", "error: code = 500 reason = WechatFAIL message = 微信返回错误:该用户账号异常，无法领券。商家可联系微信支付或让用户联系微信支付客服处理。 metadat"). FindInBatches(&results, 100, func(tx *gorm.DB, batch int) error { return fun(ctx, p.ToBos(results)) }) diff --git a/internal/pkg/script/script_test.go b/internal/pkg/script/script_test.go index 8fa5d22..dfc90cb 100644 --- a/internal/pkg/script/script_test.go +++ b/internal/pkg/script/script_test.go @@ -1,8 +1,12 @@ package script import ( + "encoding/json" + "fmt" + "golang.org/x/sync/errgroup" "testing" "time" + "voucher/internal/biz/bo" ) func Test_script(t *testing.T) { @@ -51,3 +55,103 @@ func Test_script2(t *testing.T) { t.Error(err) } } + +func Test_moreTime(t *testing.T) { + + startTime, err := time.Parse(time.DateTime, "2025-05-31 00:00:00") + if err != nil { + t.Error(err) + return + } + + endTime, err := time.Parse(time.DateTime, "2025-05-31 10:00:00") + if err != nil { + t.Error(err) + return + } + + //duration := 240 * time.Hour + duration := 1 * time.Hour + + for start := startTime; start.Before(endTime); start = start.Add(duration) { + + end := start.Add(duration) // 计算每次请求的结束时间 + if end.After(endTime) { + end = endTime + } + + // 创建请求体 + requestBody := map[string]any{ + "start_time": start.Format(time.DateTime), + "end_time": end.Format(time.DateTime), + "go_num": 2, // 并发数量 + "time_slice_hours": 1, // 时间间隔 + "product_no": "", + } + + // 将请求体转换为 JSON 格式 + bodyBytes, err2 := json.Marshal(requestBody) + if err2 != nil { + t.Error(err) + return + } + + fmt.Printf("body:%s\n", string(bodyBytes)) + } +} + +func Test_goMoreTime(t *testing.T) { + + startTime, err := time.Parse(time.DateTime, "2025-05-01 00:00:00") + if err != nil { + t.Error(err) + return + } + + //endTime, err := time.Parse(time.DateTime, "2025-05-31 23:59:59") + endTime, err := time.Parse(time.DateTime, "2025-05-01 23:59:59") + if err != nil { + t.Error(err) + return + } + + duration := 1 * time.Hour + + eg := new(errgroup.Group) + eg.SetLimit(5) + + for start := startTime; start.Before(endTime); start = start.Add(duration) { + + end := start.Add(duration) // 计算每次请求的结束时间 + if end.After(endTime) { + end = endTime + } + + req := &bo.FindInBatchesUseBo{ + StartTime: &start, + EndTime: &end, + } + + // 将请求体转换为 JSON 格式 + reqStr, err2 := json.Marshal(req) + if err2 != nil { + t.Error(err) + return + } + + eg.Go(func() error { + // 任务逻辑... + time.Sleep(2 * time.Second) + return fmt.Errorf("任务失败") + }) + + fmt.Printf("%s\n", string(reqStr)) + } + + err = eg.Wait() // 仅返回第一个错误 + if err != nil { + fmt.Println(err) + } else { + fmt.Println("所有任务完成") + } +} diff --git a/internal/service/script.go b/internal/service/script.go index c1cb2f0..a4cf1c4 100644 --- a/internal/service/script.go +++ b/internal/service/script.go @@ -90,6 +90,10 @@ func (this *CmbService) TimeSliceQueryPush(ctx http.Context) error { return fmt.Errorf("start_time or end_time is empty") } + if req.GoNum > 10 { + return fmt.Errorf("协程数量不能大于10") + } + _, err = this.timeSliceQuery.Push(ctx, req) if err != nil { return err From 1b9a4fdd806f8babed6207d865a51de0eebecd54 Mon Sep 17 00:00:00 2001 From: ziming Date: Thu, 12 Jun 2025 10:58:52 +0800 Subject: [PATCH 18/36] timeSliceQueryPush --- internal/biz/cron_notice.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/internal/biz/cron_notice.go b/internal/biz/cron_notice.go index 1bcb91c..a63a424 100644 --- a/internal/biz/cron_notice.go +++ b/internal/biz/cron_notice.go @@ -30,15 +30,10 @@ func (v *VoucherBiz) Notice(ctx context.Context) error { // 获取昨天 23:59:59 的时间 endTime := time.Date(noticeEndDay.Year(), noticeEndDay.Month(), noticeEndDay.Day(), 23, 59, 59, 0, noticeEndDay.Location()) - req := &bo.FindInBatchesUseBo{ - StartTime: &startTime, - EndTime: &endTime, - } - - return v.ExecuteNotice(ctx, req) + return v.timeSliceQuery(ctx, startTime, endTime) } -func (v *VoucherBiz) timeSliceQueryPush(ctx context.Context, startTime, endTime time.Time) error { +func (v *VoucherBiz) timeSliceQuery(ctx context.Context, startTime, endTime time.Time) error { duration := 1 * time.Hour From 8996df027924ea7a4ffb395daf128fba4c33deb4 Mon Sep 17 00:00:00 2001 From: ziming Date: Thu, 12 Jun 2025 11:02:34 +0800 Subject: [PATCH 19/36] timeSliceQueryPush --- internal/biz/cron_notice.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/biz/cron_notice.go b/internal/biz/cron_notice.go index a63a424..cb99d9f 100644 --- a/internal/biz/cron_notice.go +++ b/internal/biz/cron_notice.go @@ -62,6 +62,8 @@ func (v *VoucherBiz) timeSliceQuery(ctx context.Context, startTime, endTime time func (v *VoucherBiz) ExecuteNotice(ctx context.Context, req *bo.FindInBatchesUseBo) error { + log.Warnf("订单定时通知:%+v", req) + return v.OrderRepo.FindInBatches(ctx, req, func(ctx context.Context, rows []*bo.OrderBo) error { for _, order := range rows { From 1d8862ab7bddb43dae8accf5d5919a1af98065d7 Mon Sep 17 00:00:00 2001 From: ziming Date: Thu, 12 Jun 2025 11:04:07 +0800 Subject: [PATCH 20/36] timeSliceQueryPush --- internal/biz/cron_notice.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/biz/cron_notice.go b/internal/biz/cron_notice.go index cb99d9f..224c397 100644 --- a/internal/biz/cron_notice.go +++ b/internal/biz/cron_notice.go @@ -21,13 +21,14 @@ func (v *VoucherBiz) Notice(ctx context.Context) error { now := time.Now() - // 获取七天前的日期 + // 获取 NoticeStartDays 天前的日期 noticeStartDay := now.AddDate(0, 0, int(-v.bc.Cmb.NoticeStartDays)) - // 获取七天前 00:00:00 的时间 + // 获取 NoticeStartDays 天前 00:00:00 的时间 startTime := time.Date(noticeStartDay.Year(), noticeStartDay.Month(), noticeStartDay.Day(), 0, 0, 0, 0, noticeStartDay.Location()) + // 获取 NoticeEndDays 天前的日期 noticeEndDay := now.AddDate(0, 0, int(-v.bc.Cmb.NoticeEndDays)) - // 获取昨天 23:59:59 的时间 + // 获取 NoticeEndDays 天 23:59:59 的时间 endTime := time.Date(noticeEndDay.Year(), noticeEndDay.Month(), noticeEndDay.Day(), 23, 59, 59, 0, noticeEndDay.Location()) return v.timeSliceQuery(ctx, startTime, endTime) From 92b319f6c3099cde0681687ad3469dac55e33617 Mon Sep 17 00:00:00 2001 From: ziming Date: Thu, 12 Jun 2025 11:35:09 +0800 Subject: [PATCH 21/36] timeSliceQueryPush --- internal/data/repoimpl/order.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/data/repoimpl/order.go b/internal/data/repoimpl/order.go index eab59c3..d43d094 100644 --- a/internal/data/repoimpl/order.go +++ b/internal/data/repoimpl/order.go @@ -277,6 +277,7 @@ func (p *OrderRepoImpl) Success(ctx context.Context, id uint64, voucherNo string Updates(model.Order{ Status: vo.OrderStatusSuccess.GetValue(), VoucherNo: voucherNo, + Remark: "成功", ReceiveSuccessTime: &now, UpdateTime: &now, }) @@ -298,6 +299,7 @@ func (p *OrderRepoImpl) Available(ctx context.Context, id uint64) error { }). Updates(model.Order{ Status: vo.OrderStatusSuccess.GetValue(), + Remark: "成功,领取时间重置", ReceiveSuccessTime: &now, // 领取成功时间重置 UpdateTime: &now, }) @@ -345,6 +347,7 @@ func (p *OrderRepoImpl) Used(ctx context.Context, id uint64) error { }). Updates(model.Order{ Status: vo.OrderStatusUse.GetValue(), + Remark: "核销", LastUseTime: &now, UpdateTime: &now, }) @@ -366,6 +369,7 @@ func (p *OrderRepoImpl) NotifyUsed(ctx context.Context, id uint64, transactionId Updates(model.Order{ Status: vo.OrderStatusUse.GetValue(), TransactionId: transactionId, + Remark: "微信回调核销", LastUseTime: &now, UpdateTime: &now, }) @@ -386,6 +390,7 @@ func (p *OrderRepoImpl) Expired(ctx context.Context, id uint64) error { }). Updates(model.Order{ Status: vo.OrderStatusExpired.GetValue(), + Remark: "过期", UpdateTime: &now, }) From bd18369dcad840d2adc9f555f8110b456f4eb3ba Mon Sep 17 00:00:00 2001 From: ziming Date: Thu, 12 Jun 2025 11:36:09 +0800 Subject: [PATCH 22/36] timeSliceQueryPush --- internal/data/repoimpl/order.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/data/repoimpl/order.go b/internal/data/repoimpl/order.go index d43d094..b9ec73d 100644 --- a/internal/data/repoimpl/order.go +++ b/internal/data/repoimpl/order.go @@ -299,7 +299,7 @@ func (p *OrderRepoImpl) Available(ctx context.Context, id uint64) error { }). Updates(model.Order{ Status: vo.OrderStatusSuccess.GetValue(), - Remark: "成功,领取时间重置", + Remark: "重置为成功，领取成功时间重置", ReceiveSuccessTime: &now, // 领取成功时间重置 UpdateTime: &now, }) From 4e479ffaa83cd37c915097fb94463c198b1003a6 Mon Sep 17 00:00:00 2001 From: ziming Date: Thu, 12 Jun 2025 13:43:00 +0800 Subject: [PATCH 23/36] timeSliceQueryPush --- internal/biz/cron_notice.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/internal/biz/cron_notice.go b/internal/biz/cron_notice.go index 224c397..6279e86 100644 --- a/internal/biz/cron_notice.go +++ b/internal/biz/cron_notice.go @@ -54,6 +54,20 @@ func (v *VoucherBiz) timeSliceQuery(ctx context.Context, startTime, endTime time } eg.Go(func() error { + + select { + case <-ctx.Done(): + return ctx.Err() + default: + // 继续执行 + } + + defer func() { + if err := recover(); err != nil { + log.Errorf("查询券订单状态发生错误:req:%+v,err:%v", err) + } + }() + return v.ExecuteNotice(ctx, req) }) } @@ -70,7 +84,7 @@ func (v *VoucherBiz) ExecuteNotice(ctx context.Context, req *bo.FindInBatchesUse for _, order := range rows { if err := v.notice(ctx, order); err != nil { - log.Errorf("招行查询券订单状态发生错误,orderNo:%s,err:%v", order.OrderNo, err) + log.Errorf("查询券订单状态发生错误,orderNo:%s,err:%v", order.OrderNo, err) } } From 32c3b6c7a8f0938e44936342eb7bd4771535474e Mon Sep 17 00:00:00 2001 From: ziming Date: Thu, 12 Jun 2025 14:21:55 +0800 Subject: [PATCH 24/36] timeSliceQueryPush --- internal/biz/cron_notice.go | 104 ++++++++++++++++++------------------ 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/internal/biz/cron_notice.go b/internal/biz/cron_notice.go index 6279e86..b605bb5 100644 --- a/internal/biz/cron_notice.go +++ b/internal/biz/cron_notice.go @@ -13,6 +13,53 @@ import ( "voucher/internal/pkg/lock" ) +func (v *VoucherBiz) isCanNotice(ctx context.Context) error { + + if v.bc.Cmb.NoticeStartDays == 0 { + return errors.New("订单定时通知,noticeStartDays eq 0") + } + + if v.bc.Cmb.NoticeEndDays == 0 { + return errors.New("订单定时通知,noticeEndDays eq 0") + } + + cache := vo.CmbBatchNoticeCacheKey.BuildCache([]string{""}) + + _, err := v.rdb.Rdb.Get(ctx, cache.Key).Result() + + if err == nil { + return fmt.Errorf("订单定时通知,notice 获取redis缓存存在，已被执行,本台服务不做执行") + } + + if err != redis.Nil { + return fmt.Errorf(fmt.Sprintf("订单定时通知,notice 获取redis缓存%s异常:%v", cache.Key, err)) + } + + c := vo.CmbBatchNoticeLockKey.BuildCache([]string{""}) + + return lock.NewMutex(v.rdb.Rdb, c.TTL).Lock(ctx, c.Key, func(ctx context.Context) error { + + // 二次获取，判定处理，以免获取锁后又执行了一次 + + cacheValue, err2 := v.rdb.Rdb.Get(ctx, cache.Key).Result() + + if err2 != nil && err2 != redis.Nil { + return fmt.Errorf(fmt.Sprintf("订单定时通知,notice 二次获取redis缓存%s异常:%v", cache.Key, err2)) + } + + if len(cacheValue) > 0 { + return fmt.Errorf("订单定时通知,notice 二次获取redis缓存存在，已被执行,本台服务不做执行") + } + + if err = v.rdb.Rdb.Set(ctx, cache.Key, fmt.Sprintf("%d_%d", v.bc.Cmb.NoticeStartDays, v.bc.Cmb.NoticeEndDays), c.TTL).Err(); err != nil { + return fmt.Errorf(fmt.Sprintf("notice 设置redis缓存%s异常:%v", cache.Key, err)) + } + + log.Warnf("订单定时通知,notice 获取redis缓存,不存在，开始处理") + return nil + }) +} + func (v *VoucherBiz) Notice(ctx context.Context) error { if err := v.isCanNotice(ctx); err != nil { @@ -36,7 +83,7 @@ func (v *VoucherBiz) Notice(ctx context.Context) error { func (v *VoucherBiz) timeSliceQuery(ctx context.Context, startTime, endTime time.Time) error { - duration := 1 * time.Hour + duration := 2 * time.Hour eg := new(errgroup.Group) eg.SetLimit(5) @@ -64,7 +111,7 @@ func (v *VoucherBiz) timeSliceQuery(ctx context.Context, startTime, endTime time defer func() { if err := recover(); err != nil { - log.Errorf("查询券订单状态发生错误:req:%+v,err:%v", err) + log.Errorf("订单定时通知,发生错误:req:%+v,err:%v", req, err) } }() @@ -84,7 +131,7 @@ func (v *VoucherBiz) ExecuteNotice(ctx context.Context, req *bo.FindInBatchesUse for _, order := range rows { if err := v.notice(ctx, order); err != nil { - log.Errorf("查询券订单状态发生错误,orderNo:%s,err:%v", order.OrderNo, err) + log.Errorf("订单定时通知,券订单状态查询处理发生错误,orderNo:%s,err:%v", order.OrderNo, err) } } @@ -94,53 +141,6 @@ func (v *VoucherBiz) ExecuteNotice(ctx context.Context, req *bo.FindInBatchesUse } -func (v *VoucherBiz) isCanNotice(ctx context.Context) error { - - if v.bc.Cmb.NoticeStartDays == 0 { - return errors.New("noticeStartDays eq 0") - } - - if v.bc.Cmb.NoticeEndDays == 0 { - return errors.New("noticeEndDays eq 0") - } - - cache := vo.CmbBatchNoticeCacheKey.BuildCache([]string{""}) - - _, err := v.rdb.Rdb.Get(ctx, cache.Key).Result() - - if err == nil { - return fmt.Errorf("notice 获取redis缓存存在，已被执行,本台服务不做执行") - } - - if err != redis.Nil { - return fmt.Errorf(fmt.Sprintf("notice 获取redis缓存%s异常:%v", cache.Key, err)) - } - - c := vo.CmbBatchNoticeLockKey.BuildCache([]string{""}) - - return lock.NewMutex(v.rdb.Rdb, c.TTL).Lock(ctx, c.Key, func(ctx context.Context) error { - - // 二次获取，判定处理，以免获取锁后又执行了一次 - - cacheValue, err2 := v.rdb.Rdb.Get(ctx, cache.Key).Result() - - if err2 != nil && err2 != redis.Nil { - return fmt.Errorf(fmt.Sprintf("notice 二次获取redis缓存%s异常:%v", cache.Key, err2)) - } - - if len(cacheValue) > 0 { - return fmt.Errorf("notice 二次获取redis缓存存在，已被执行,本台服务不做执行") - } - - if err = v.rdb.Rdb.Set(ctx, cache.Key, fmt.Sprintf("%d_%d", v.bc.Cmb.NoticeStartDays, v.bc.Cmb.NoticeEndDays), c.TTL).Err(); err != nil { - return fmt.Errorf(fmt.Sprintf("notice 设置redis缓存%s异常:%v", cache.Key, err)) - } - - log.Warnf("notice 获取redis缓存,不存在，开始处理") - return nil - }) -} - func (v *VoucherBiz) notice(ctx context.Context, order *bo.OrderBo) error { // 批量通知不做数据存储，量会很大 @@ -188,11 +188,11 @@ func (v *VoucherBiz) cmbNotice(ctx context.Context, order *bo.OrderBo, orderNoti reply, err := v.CmbMixRepo.Request(ctx, request, order.NotifyUrl) if err != nil { - return fmt.Errorf("orderNo:%s,outBizNo:%s,%s", order.OrderNo, order.OutBizNo, err.Error()) + return fmt.Errorf("订单定时通知,orderNo:%s,outBizNo:%s,%s", order.OrderNo, order.OutBizNo, err.Error()) } if reply.RespCode != vo.CmbResponseStatusSuccess.GetValue() { - return errors.New(reply.RespMsg) + return errors.New("订单定时通知,招行返回：" + reply.RespMsg) } return nil From 21bb6f53ebee646814b3c7dc5554a5cce966be76 Mon Sep 17 00:00:00 2001 From: ziming Date: Thu, 12 Jun 2025 14:30:49 +0800 Subject: [PATCH 25/36] timeSliceQueryPush --- internal/biz/cron_notice.go | 14 +++++++------- internal/biz/vo/order_notify_event.go | 2 +- internal/data/repoimpl/order.go | 4 ++-- internal/service/voucher.go | 5 +++-- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/internal/biz/cron_notice.go b/internal/biz/cron_notice.go index b605bb5..79a339b 100644 --- a/internal/biz/cron_notice.go +++ b/internal/biz/cron_notice.go @@ -93,6 +93,8 @@ func (v *VoucherBiz) timeSliceQuery(ctx context.Context, startTime, endTime time end := start.Add(duration) // 计算每次请求的结束时间 if end.After(endTime) { end = endTime + } else { + end = end.Add(-1 * time.Second) } req := &bo.FindInBatchesUseBo{ @@ -150,8 +152,7 @@ func (v *VoucherBiz) notice(ctx context.Context, order *bo.OrderBo) error { } if order.Status == status { - //log.Warnf("notice 券状态未改变:%s，忽略不处理,orderNo:%s", order.Status.GetText(), order.OrderNo) - return nil + return nil // 券状态未改变，忽略不处理 } event, err := status.GetOrderNotifyEvent() @@ -176,9 +177,8 @@ func (v *VoucherBiz) notice(ctx context.Context, order *bo.OrderBo) error { func (v *VoucherBiz) cmbNotice(ctx context.Context, order *bo.OrderBo, orderNotify *bo.OrderNotifyBo) error { - if !orderNotify.Event.CanNotify() { - //log.Warnf("notice 券状态:%s，忽略不通知,orderNo:%s", orderNotify.Event.GetText(), order.OrderNo) - return nil + if orderNotify.Event.CanNotify() { + return nil // 不可通知，忽略 } request, err := v.Cmb.NotifyRequest(ctx, order, orderNotify) @@ -188,11 +188,11 @@ func (v *VoucherBiz) cmbNotice(ctx context.Context, order *bo.OrderBo, orderNoti reply, err := v.CmbMixRepo.Request(ctx, request, order.NotifyUrl) if err != nil { - return fmt.Errorf("订单定时通知,orderNo:%s,outBizNo:%s,%s", order.OrderNo, order.OutBizNo, err.Error()) + return fmt.Errorf("订单定时通知,orderNo:%s,outBizNo:%s,err:%s", order.OrderNo, order.OutBizNo, err.Error()) } if reply.RespCode != vo.CmbResponseStatusSuccess.GetValue() { - return errors.New("订单定时通知,招行返回：" + reply.RespMsg) + return errors.New("订单定时通知,招行返回错误:" + reply.RespMsg) } return nil diff --git a/internal/biz/vo/order_notify_event.go b/internal/biz/vo/order_notify_event.go index db5fb45..84c6782 100644 --- a/internal/biz/vo/order_notify_event.go +++ b/internal/biz/vo/order_notify_event.go @@ -40,7 +40,7 @@ func (s OrderNotifyEvent) IsExpired() bool { } func (s OrderNotifyEvent) CanNotify() bool { - return s.IsSendDEd() || s.IsUsed() || s.IsExpired() + return s.IsSendDEd() || s.IsUsed() } var OrderNotifyEventMapCmbStatus = map[OrderNotifyEvent]CmbStatus{ diff --git a/internal/data/repoimpl/order.go b/internal/data/repoimpl/order.go index b9ec73d..916fc5a 100644 --- a/internal/data/repoimpl/order.go +++ b/internal/data/repoimpl/order.go @@ -98,13 +98,13 @@ func (p *OrderRepoImpl) FindIngInBatches(ctx context.Context, fun func(ctx conte return nil } -func (p *OrderRepoImpl) FindInBatches(ctx context.Context, w *bo.FindInBatchesUseBo, fun func(ctx context.Context, rows []*bo.OrderBo) error) error { +func (p *OrderRepoImpl) FindInBatches(ctx context.Context, req *bo.FindInBatchesUseBo, fun func(ctx context.Context, rows []*bo.OrderBo) error) error { var results = make([]*model.Order, 0) result := p.DB(ctx). Where("status IN (?)", []uint8{vo.OrderStatusSuccess.GetValue(), vo.OrderStatusUse.GetValue()}). - Where("receive_success_time BETWEEN ? AND ?", w.StartTime, w.EndTime). + Where("receive_success_time BETWEEN ? AND ?", req.StartTime, req.EndTime). FindInBatches(&results, 100, func(tx *gorm.DB, batch int) error { // tx.RowsAffected 提供当前批处理中记录的计数（the count of records in the current batch） // 'batch' 变量表示当前批号（the current batch number） diff --git a/internal/service/voucher.go b/internal/service/voucher.go index 83f1acb..50b1cfe 100644 --- a/internal/service/voucher.go +++ b/internal/service/voucher.go @@ -72,7 +72,9 @@ func (s *VoucherService) CronOrderNotice(ctx context.Context) error { } func (s *VoucherService) OrderNotice(ctx context.Context) { + start := time.Now() + log.Errorf("订单定时通知,执行开始: %v", start.Format(time.DateTime)) if err := s.VoucherBiz.Notice(ctx); err != nil { log.Errorf("订单定时通知,执行失败: %v", err) @@ -81,11 +83,10 @@ func (s *VoucherService) OrderNotice(ctx context.Context) { end := time.Now() elapsed := end.Sub(start) log.Warnf("订单定时通知,开始执行时间%s,执行结束时间%s,代码块执行耗时: %s", start.Format(time.DateTime), end.Format(time.DateTime), elapsed) - - return } func (j *VoucherService) GetNotifyRetryConfig() *mq.ConsumerConfig { + elm, ok := j.bc.RocketMQ.EventMap["notifyRetry"] if !ok { return nil From 015c62ed60128d05cfb5ba644fe17028ed39d38d Mon Sep 17 00:00:00 2001 From: ziming Date: Thu, 12 Jun 2025 14:30:58 +0800 Subject: [PATCH 26/36] timeSliceQueryPush --- internal/service/voucher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/service/voucher.go b/internal/service/voucher.go index 50b1cfe..6c05020 100644 --- a/internal/service/voucher.go +++ b/internal/service/voucher.go @@ -74,7 +74,7 @@ func (s *VoucherService) CronOrderNotice(ctx context.Context) error { func (s *VoucherService) OrderNotice(ctx context.Context) { start := time.Now() - log.Errorf("订单定时通知,执行开始: %v", start.Format(time.DateTime)) + log.Errorf("订单定时通知,执行开始: %s", start.Format(time.DateTime)) if err := s.VoucherBiz.Notice(ctx); err != nil { log.Errorf("订单定时通知,执行失败: %v", err) From af64c0eee72b489031d6419e096f7442b391cafd Mon Sep 17 00:00:00 2001 From: ziming Date: Thu, 12 Jun 2025 14:41:29 +0800 Subject: [PATCH 27/36] timeSliceQueryPush --- internal/biz/timeslicequery/execute.go | 2 +- internal/biz/timeslicequery/mq.go | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/biz/timeslicequery/execute.go b/internal/biz/timeslicequery/execute.go index 0037332..76534fe 100644 --- a/internal/biz/timeslicequery/execute.go +++ b/internal/biz/timeslicequery/execute.go @@ -20,7 +20,7 @@ func (v *Query) execute(ctx context.Context, req *timeslice.Manager) error { log.Errorf("微信券查询处理,%s到%s,失败:%v", managerStartStr, managerEndStr, err) } - fmt.Printf("%s到%s,微信券查询处理,总任务数:%d\n", managerStartStr, managerEndStr, taskCount) + fmt.Printf("微信券查询处理,%s到%s,总任务数:%d\n", managerStartStr, managerEndStr, taskCount) log.Warnf("微信券查询处理,%s到%s,总任务数:%d", managerStartStr, managerEndStr, taskCount) return nil diff --git a/internal/biz/timeslicequery/mq.go b/internal/biz/timeslicequery/mq.go index 24086c2..87dc580 100644 --- a/internal/biz/timeslicequery/mq.go +++ b/internal/biz/timeslicequery/mq.go @@ -114,17 +114,17 @@ func (v *Query) Consumer(ctx context.Context, msg string) error { executeStart := time.Now() executeStartStr := executeStart.Format(time.DateTime) - log.Warnf("微信券查询处理开始:%s,msg:%s,manager%s", executeStartStr, msg, reqStr) - fmt.Printf("微信券查询处理开始:%s,msg:%s,manager%s", executeStartStr, msg, reqStr) + log.Warnf("微信券查询处理,开始:%s,msg:%s,manager%s", executeStartStr, msg, reqStr) + fmt.Printf("微信券查询处理,开始:%s,msg:%s,manager%s", executeStartStr, msg, reqStr) if err = v.execute(ctx, req); err != nil { - log.Errorf("微信券查询处理失败:%s,msg:%s,err:%v", executeStartStr, msg, err) - return fmt.Errorf("微信券查询处理失败:%s,msg:%s,err:%v", executeStartStr, msg, err) + log.Errorf("微信券查询处理,失败:%s,msg:%s,err:%v", executeStartStr, msg, err) + return fmt.Errorf("微信券查询处理,失败:%s,msg:%s,err:%v", executeStartStr, msg, err) } executeEnd := time.Now() - log.Warnf("微信券查询处理耗时:%s,结束时间%s,manager%s", executeEnd.Sub(executeStart).String(), executeEnd.String(), reqStr) - fmt.Printf("微信券查询处理耗时:%s,结束时间%s,manager%s", executeEnd.Sub(executeStart).String(), executeEnd.String(), reqStr) + log.Warnf("微信券查询处理,耗时:%s,结束时间%s,manager%s", executeEnd.Sub(executeStart).String(), executeEnd.String(), reqStr) + fmt.Printf("微信券查询处理,耗时:%s,结束时间%s,manager%s", executeEnd.Sub(executeStart).String(), executeEnd.String(), reqStr) return nil } From f965bf8b7f39f9d1ab44f3176420b0736172eca9 Mon Sep 17 00:00:00 2001 From: ziming Date: Thu, 12 Jun 2025 14:46:11 +0800 Subject: [PATCH 28/36] timeSliceQueryPush --- internal/pkg/script/script_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/pkg/script/script_test.go b/internal/pkg/script/script_test.go index dfc90cb..391b932 100644 --- a/internal/pkg/script/script_test.go +++ b/internal/pkg/script/script_test.go @@ -9,7 +9,7 @@ import ( "voucher/internal/biz/bo" ) -func Test_script(t *testing.T) { +func Test_timeSliceQueryPush(t *testing.T) { startTime, err := time.Parse(time.DateTime, "2025-05-01 00:00:00") if err != nil { @@ -33,7 +33,7 @@ func Test_script(t *testing.T) { } } -func Test_script2(t *testing.T) { +func Test_pushWechatQuery(t *testing.T) { startTime, err := time.Parse(time.DateTime, "2025-05-31 00:00:00") if err != nil { From 6b2622b1a3fda1928edb62e92ef868a5c982849f Mon Sep 17 00:00:00 2001 From: ziming Date: Thu, 12 Jun 2025 15:08:08 +0800 Subject: [PATCH 29/36] timeSliceQueryPush --- internal/biz/wechat_notify.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/biz/wechat_notify.go b/internal/biz/wechat_notify.go index 8c288f4..25509df 100644 --- a/internal/biz/wechat_notify.go +++ b/internal/biz/wechat_notify.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "github.com/go-kratos/kratos/v2/log" "gorm.io/gorm" errPb "voucher/api/err" "voucher/internal/biz/bo" @@ -76,7 +75,7 @@ func (this *VoucherBiz) getOrder(ctx context.Context, req *bo.WechatVoucherNotif func (v *VoucherBiz) notifyUsed(ctx context.Context, order *bo.OrderBo, req *bo.WechatVoucherNotifyBo) error { if order.Status.IsUse() { - log.Warnf("券状态已是已使用，忽略不处理,orderNo:%s", order.OrderNo) + // 券状态已是已使用，忽略不处理 return nil } @@ -90,7 +89,7 @@ func (v *VoucherBiz) notifyUsed(ctx context.Context, order *bo.OrderBo, req *bo. func (v *VoucherBiz) available(ctx context.Context, order *bo.OrderBo) error { if order.Status.IsSuccess() { - log.Warnf("券状态已是可使用，忽略不处理,orderNo:%s", order.OrderNo) + // 券状态已是可使用，忽略不处理 return nil } @@ -104,7 +103,7 @@ func (v *VoucherBiz) available(ctx context.Context, order *bo.OrderBo) error { func (v *VoucherBiz) expired(ctx context.Context, order *bo.OrderBo) error { if order.Status.IsExpired() { - log.Warnf("券状态已是已过期，忽略不处理,orderNo:%s", order.OrderNo) + // 券状态已是已过期，忽略不处理 return nil } From c68c9968924949b21d839a2aafa81eaee57facad Mon Sep 17 00:00:00 2001 From: ziming Date: Thu, 12 Jun 2025 17:33:07 +0800 Subject: [PATCH 30/36] timeSliceQuery --- internal/biz/cron_notice.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/biz/cron_notice.go b/internal/biz/cron_notice.go index 79a339b..5f256e2 100644 --- a/internal/biz/cron_notice.go +++ b/internal/biz/cron_notice.go @@ -86,7 +86,7 @@ func (v *VoucherBiz) timeSliceQuery(ctx context.Context, startTime, endTime time duration := 2 * time.Hour eg := new(errgroup.Group) - eg.SetLimit(5) + eg.SetLimit(8) for start := startTime; start.Before(endTime); start = start.Add(duration) { From d72757e7384a45fbfe718a96c745aad215acfd7c Mon Sep 17 00:00:00 2001 From: ziming Date: Thu, 12 Jun 2025 17:45:09 +0800 Subject: [PATCH 31/36] timeSliceQuery --- internal/biz/cron_notice.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/biz/cron_notice.go b/internal/biz/cron_notice.go index 5f256e2..84d9d5f 100644 --- a/internal/biz/cron_notice.go +++ b/internal/biz/cron_notice.go @@ -86,7 +86,7 @@ func (v *VoucherBiz) timeSliceQuery(ctx context.Context, startTime, endTime time duration := 2 * time.Hour eg := new(errgroup.Group) - eg.SetLimit(8) + eg.SetLimit(10) for start := startTime; start.Before(endTime); start = start.Add(duration) { From 8adc2b176bb99b448298fe74210344460283618b Mon Sep 17 00:00:00 2001 From: ziming Date: Fri, 13 Jun 2025 09:17:44 +0800 Subject: [PATCH 32/36] timeSliceQuery --- internal/biz/timeslicequery/execute.go | 1 + internal/biz/wechat_query.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/biz/timeslicequery/execute.go b/internal/biz/timeslicequery/execute.go index 76534fe..d5e36b9 100644 --- a/internal/biz/timeslicequery/execute.go +++ b/internal/biz/timeslicequery/execute.go @@ -59,6 +59,7 @@ func (v *Query) callbackFunc(ctx context.Context, req *timeslice.Task) error { "order_no": order.OrderNo, "coupon_id": order.VoucherNo, "open_id": order.Account, + "stock_id": order.BatchNo, "err": err.Error(), } log.Errorf("微信券查询处理,%s到%s,taskId:%d,错误:%+v", startTimeStr, endTimeStr, req.TaskID, logFields) diff --git a/internal/biz/wechat_query.go b/internal/biz/wechat_query.go index f389912..8ea7ae9 100644 --- a/internal/biz/wechat_query.go +++ b/internal/biz/wechat_query.go @@ -79,8 +79,8 @@ func (v *VoucherBiz) WechatQuery(ctx context.Context, msg string) error { num += 1 if err := v.wechatQuery(ctx, order, ¬ifyNum); err != nil { - log.Errorf("微信查询券订单状态发生错误,msg:%s,orderNo:%s,couponId:%s,appId:%s,openId:%s,err:%v", - msg, order.OrderNo, order.VoucherNo, order.AppID, order.Account, err) + log.Errorf("微信查询券订单状态发生错误,msg:%s,orderNo:%s,couponId:%s,appId:%s,openId:%s,stockId:%s,err:%v", + msg, order.OrderNo, order.VoucherNo, order.AppID, order.Account, order.BatchNo, err) } } From dc152617f3f99485a79a876f03753f51ade3ec24 Mon Sep 17 00:00:00 2001 From: ziming Date: Fri, 13 Jun 2025 09:21:20 +0800 Subject: [PATCH 33/36] timeSliceQuery --- internal/biz/cron_notice.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/biz/cron_notice.go b/internal/biz/cron_notice.go index 84d9d5f..ccef660 100644 --- a/internal/biz/cron_notice.go +++ b/internal/biz/cron_notice.go @@ -133,7 +133,7 @@ func (v *VoucherBiz) ExecuteNotice(ctx context.Context, req *bo.FindInBatchesUse for _, order := range rows { if err := v.notice(ctx, order); err != nil { - log.Errorf("订单定时通知,券订单状态查询处理发生错误,orderNo:%s,err:%v", order.OrderNo, err) + log.Error(err) } } From 6329e6b37f88921deeda1f29bcc449b650747c47 Mon Sep 17 00:00:00 2001 From: ziming Date: Fri, 13 Jun 2025 09:26:04 +0800 Subject: [PATCH 34/36] timeSliceQuery --- internal/biz/cron_notice.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/biz/cron_notice.go b/internal/biz/cron_notice.go index ccef660..1b1aaac 100644 --- a/internal/biz/cron_notice.go +++ b/internal/biz/cron_notice.go @@ -83,6 +83,8 @@ func (v *VoucherBiz) Notice(ctx context.Context) error { func (v *VoucherBiz) timeSliceQuery(ctx context.Context, startTime, endTime time.Time) error { + log.Warnf("订单定时通知,开始处理,按每两个小时分片处理,范围:%s到%s", startTime.Format(time.DateTime), endTime.Format(time.DateTime)) + duration := 2 * time.Hour eg := new(errgroup.Group) From ca8f9955cb28cbd85d2be54efd7ba316b33d0a55 Mon Sep 17 00:00:00 2001 From: ziming Date: Fri, 13 Jun 2025 09:36:28 +0800 Subject: [PATCH 35/36] timeSliceQuery --- internal/biz/cron_notice.go | 29 ++++++++++++++++++++------ internal/biz/timeslicequery/execute.go | 2 +- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/internal/biz/cron_notice.go b/internal/biz/cron_notice.go index 1b1aaac..7456502 100644 --- a/internal/biz/cron_notice.go +++ b/internal/biz/cron_notice.go @@ -128,13 +128,19 @@ func (v *VoucherBiz) timeSliceQuery(ctx context.Context, startTime, endTime time func (v *VoucherBiz) ExecuteNotice(ctx context.Context, req *bo.FindInBatchesUseBo) error { - log.Warnf("订单定时通知:%+v", req) + start := time.Now() - return v.OrderRepo.FindInBatches(ctx, req, func(ctx context.Context, rows []*bo.OrderBo) error { + num := 0 + notifyNum := 0 + errNum := 0 + + err := v.OrderRepo.FindInBatches(ctx, req, func(ctx context.Context, rows []*bo.OrderBo) error { for _, order := range rows { - if err := v.notice(ctx, order); err != nil { + num += 1 + if err := v.notice(ctx, order, ¬ifyNum); err != nil { + errNum += 1 log.Error(err) } @@ -143,9 +149,18 @@ func (v *VoucherBiz) ExecuteNotice(ctx context.Context, req *bo.FindInBatchesUse return nil }) + logFields := map[string]interface{}{ + "searchTime": req.StartTime.Format(time.DateTime) + "到" + req.EndTime.Format(time.DateTime), + "num": num, + "notifyNum": notifyNum, + "elapsed": time.Now().Sub(start).String(), + } + log.Warnf("订单定时通知,%+v", logFields) + + return err } -func (v *VoucherBiz) notice(ctx context.Context, order *bo.OrderBo) error { +func (v *VoucherBiz) notice(ctx context.Context, order *bo.OrderBo, notifyNum *int) error { // 批量通知不做数据存储，量会很大 status, err := v.WechatCpnRepo.Query(ctx, order) @@ -170,19 +185,21 @@ func (v *VoucherBiz) notice(ctx context.Context, order *bo.OrderBo) error { Type: order.Type, } - if err = v.cmbNotice(ctx, order, orderNotify); err != nil { + if err = v.cmbNotice(ctx, order, orderNotify, notifyNum); err != nil { return err } return v.UpdateOrderStatus(ctx, order.ID, status) } -func (v *VoucherBiz) cmbNotice(ctx context.Context, order *bo.OrderBo, orderNotify *bo.OrderNotifyBo) error { +func (v *VoucherBiz) cmbNotice(ctx context.Context, order *bo.OrderBo, orderNotify *bo.OrderNotifyBo, notifyNum *int) error { if orderNotify.Event.CanNotify() { return nil // 不可通知，忽略 } + *notifyNum += 1 + request, err := v.Cmb.NotifyRequest(ctx, order, orderNotify) if err != nil { return err diff --git a/internal/biz/timeslicequery/execute.go b/internal/biz/timeslicequery/execute.go index d5e36b9..5f6e11d 100644 --- a/internal/biz/timeslicequery/execute.go +++ b/internal/biz/timeslicequery/execute.go @@ -80,7 +80,7 @@ func (v *Query) callbackFunc(ctx context.Context, req *timeslice.Task) error { "searchTime": currentStartTimeStr + "到" + currentEndTimeStr, "num": num, "notifyNum": notifyNum, - "duration": end.Sub(start).String(), + "elapsed": end.Sub(start).String(), } log.Warnf("微信券查询处理,%s到%s,taskId:%d,处理完毕:%+v", startTimeStr, endTimeStr, req.TaskID, logFields) From 5e43a7d6d2c35dfa7d5c8a8677ae024992945f5f Mon Sep 17 00:00:00 2001 From: ziming Date: Fri, 13 Jun 2025 09:46:39 +0800 Subject: [PATCH 36/36] timeSliceQuery --- internal/biz/cron_notice.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/biz/cron_notice.go b/internal/biz/cron_notice.go index 7456502..b59f2c3 100644 --- a/internal/biz/cron_notice.go +++ b/internal/biz/cron_notice.go @@ -194,7 +194,7 @@ func (v *VoucherBiz) notice(ctx context.Context, order *bo.OrderBo, notifyNum *i func (v *VoucherBiz) cmbNotice(ctx context.Context, order *bo.OrderBo, orderNotify *bo.OrderNotifyBo, notifyNum *int) error { - if orderNotify.Event.CanNotify() { + if !orderNotify.Event.CanNotify() { return nil // 不可通知，忽略 } @@ -207,7 +207,7 @@ func (v *VoucherBiz) cmbNotice(ctx context.Context, order *bo.OrderBo, orderNoti reply, err := v.CmbMixRepo.Request(ctx, request, order.NotifyUrl) if err != nil { - return fmt.Errorf("订单定时通知,orderNo:%s,outBizNo:%s,err:%s", order.OrderNo, order.OutBizNo, err.Error()) + return fmt.Errorf("订单定时通知,orderNo:%s,outBizNo:%s,stockId:%s,err:%s", order.OrderNo, order.OutBizNo, order.BatchNo, err.Error()) } if reply.RespCode != vo.CmbResponseStatusSuccess.GetValue() {