From 38bc172e13de6329db2c08a7a8bc06f3979f90e5 Mon Sep 17 00:00:00 2001 From: renzhiyuan <465386466@qq.com> Date: Fri, 21 Feb 2025 10:52:23 +0800 Subject: [PATCH] first push --- README.md | 235 +++++++++++++++++ create_excel.go | 44 ++++ func.go | 158 ++++++++++++ go.mod | 36 +++ import.go | 658 ++++++++++++++++++++++++++++++++++++++++++++++++ import_test.go | 17 ++ option.go | 72 ++++++ types.go | 80 ++++++ 8 files changed, 1300 insertions(+) create mode 100644 README.md create mode 100644 create_excel.go create mode 100644 func.go create mode 100644 go.mod create mode 100644 import.go create mode 100644 import_test.go create mode 100644 option.go create mode 100644 types.go diff --git a/README.md b/README.md new file mode 100644 index 0000000..bd67f6e --- /dev/null +++ b/README.md @@ -0,0 +1,235 @@ +异步excel导入 + + + +## 安装 + +```bash +$ go get -u gitea.cdlsxd.cn/self-tools/l_excel_import +``` + + +## 使用 +```go + importExcel, err := excel.NewImportExcel("goods_import", +excel.WithHeader([]string{"条码", "分类名称", "货品名称", "货品编号", "商品货号", "品牌", "单位", "规格参数", "货品说明", "保质期", "保质期单位", "链接", "货品图片", "电商销售价格", "销售价", "供应商报价", "税率", "默认供应商", "默认存放仓库", "备注", "长", "宽", "高", "重量"}), +excel.WithTrimFiled([]string{"分类名称", "默认供应商", "默认存放仓库", "货品编号", "商品货号"}), +excel.WithFiledRegex(map[string]*excel.Regex{ +"货品编号": { +Rule: "[\\u4e00-\\u9fa5]", +MatchBool: true, +Desc: "不能包含中文", +}, +}), +excel.WithSpeedMod(true), +excel.WithRowPicHandle(func(pic *excel.Pic) (url string, err2 error) { +ossConf := s.c.GetOss() +businessMap, exist := ossConf.BusinessMap["goods"] +if !exist { +return "", errors.New("oss配置不存在") +} +//"physicalGoodsSystems/images/goods" +url = s.GoodsBiz.OssPathFile(businessMap.Folder, pic.PicInfos.Format.AltText, pic.PicInfos.Extension) +// 判断endpoint字段中是否存在internal的字段，如果存在则替换为internal +urlEndpoint := businessMap.Endpoint +if strings.Contains(businessMap.Endpoint, "-internal") { +urlEndpoint = strings.Replace(businessMap.Endpoint, "-internal", "", 1) +} +return fmt.Sprintf("https://%s.%s/%s", businessMap.Bucket, urlEndpoint, url), oss.NewOss(ossConf.AccessKeyId, ossConf.AccessKeySecret).UploadBase64Byte( +ctx, +businessMap.Endpoint, +businessMap.Bucket, +url, +pic.PicInfos.File) +}), +excel.WithDeleteOssHandle(func(ObjectName []string) (err error) { +ossConf := s.c.GetOss() +businessMap, exist := ossConf.BusinessMap["goods"] +if !exist { +return errors.New("oss配置不存在") +} +return oss.NewOss(ossConf.AccessKeyId, ossConf.AccessKeySecret).DeleteBatch(ctx, businessMap.Endpoint, businessMap.Bucket, ObjectName) +}), +excel.WithGetFileObject(func(fileObjectUrl string) (body io.ReadCloser, err error) { +ossConf := s.c.GetOss() +businessMap, exist := ossConf.BusinessMap["goods"] +if !exist { +return nil, errors.New("oss配置不存在") +} +return oss.NewOss(ossConf.AccessKeyId, ossConf.AccessKeySecret).GetObject(ctx, businessMap.Endpoint, businessMap.Bucket, fileObjectUrl) +}), +).Init(ctx, func(excel *excel.ImportExcel, rows []map[string]string) { +var sellByData int +//创建子上下文，防止父ctx取消导致无法请求rpc +subCtx, cancel := context.WithCancel(context.Background()) +//协程结束后释放掉子上下文 +defer cancel() +for k, v := range rows { +var ( +supId int32 +WareHouseId int32 +CateId int32 +webPrice float64 +price float64 +costPrice float64 +) +//这里用中文作key是为了方便，如果用英文key或者数字作key，需要做映射关系，而且还要考虑key可能是任意类型，而且开发的时候出现excel插入删除也很麻烦 +//分类 +if len(v["分类名称"]) > 30 { +excel.AddErr("条码过长，请检查", k+1, v) +continue +} +if v["分类名称"] != "" { +if _, exist := cateMap[v["分类名称"]]; !exist { +excel.AddErr("商品分类不存在", k+1, v) +continue +} +CateId = cateMap[v["分类名称"]].Id +} +//供应商 +if v["默认供应商"] != "" { +if _, exist := supplierMap[v["默认供应商"]]; !exist { +excel.AddErr("供应商不存在", k+1, v) +continue +} +supId = supplierMap[v["默认供应商"]].Id +} +//仓库 +if v["默认存放仓库"] != "" { +if _, exist := wareHouseMap[v["默认存放仓库"]]; !exist { +excel.AddErr("仓库不存在", k+1, v) +continue +} +WareHouseId = int32(wareHouseMap[v["默认存放仓库"]].Id) +} +//电商价 +if v["电商销售价格"] != "" { +webPrice, err = strconv.ParseFloat(v["电商销售价格"], 64) +if err != nil { +excel.AddErr("电商销售价格格式不正确", k+1, v) +continue +} +} +if v["销售价"] != "" { +price, err = strconv.ParseFloat(v["销售价"], 64) +if err != nil { +excel.AddErr("销售价格式不正确", k+1, v) +continue +} +} +if v["供应商报价"] != "" { +costPrice, err = strconv.ParseFloat(v["供应商报价"], 64) +if err != nil { +excel.AddErr("供应商报价格式不正确", k+1, v) +continue +} +} + +//保质期 +if v["保质期"] == "" { +sellByData = 0 +} else { +sellByData, err = strconv.Atoi(v["保质期"]) +if err != nil { +excel.AddErr("保质期格式不正确", k+1, v) +continue +} +} + +//税率 +tax := util.RemovePercentSignsWithBuilder(v["税率"]) +taxRate, err := strconv.ParseFloat(tax, 64) +if err != nil { +excel.AddErr("税率格式不正确", k+1, v) +continue +} +//"条码", "分类名称", "货品名称", "货品编号", "品牌", "单位", "规格参数", "货品说明", "保质期", "保质期单位", "链接", "货品图片", "电商销售价格", "代发含税运价格", "税率", "默认供应商", "默认存放仓库", "备注"} +goodsInfo := &pb.AddGoodsReqs{ +//商品标题 +Title: v["货品名称"], +//商品品牌 +Brand: v["品牌"], +//商品简介、卖点 +Introduction: v["货品说明"], + +//商品编码 +GoodsNum: v["货品编号"], +//商品货号 +GoodsCode: v["商品货号"], +//商品条形码 +GoodsBarCode: v["条码"], +//是否组合商品 +IsComposeGoods: 2, +//市场价，单位分 +Price: float32(price), +//单位 +Unit: v["单位"], +//保质期 +SellByDate: int32(sellByData), +//保质期单位 +SellByDateUnit: v["保质期单位"], +//外部平台链接 +ExternalUrl: v["链接"], +//电商平台价格 +ExternalPrice: float32(webPrice), +//销售价 +SalesPrice: float32(price), +//税率 +TaxRate: float32(taxRate), +//商品参数 +GoodsAttributes: v["规格参数"], +//商品说明 +GoodsIllustration: v["货品说明"], +//备注 +Remark: v["备注"], +Status: pojo.STATUS_ENABLE, +IsHot: 2, +} + +goodsAdd, err := goods.NewGoodsBiz(types.ToTmplConf(s.c)).Add(subCtx, goodsInfo) +if err != nil { +excel.AddErr(fmt.Sprintf("添加商品失败：%s", err), k+1, v) +continue +} +if CateId != 0 { +_, err = goods.NewGoodsCateGoryRelationBiz(types.ToTmplConf(s.c)).Add(subCtx, &api.AddGoodsCategoryRelationReqs{ +GoodsId: goodsAdd.Id, +CategoryIds: []int32{CateId}, +}) +if err != nil { +excel.AddErr(fmt.Sprintf("添加商品分类失败：%s", err), k+1, v) +continue +} +} +if WareHouseId != 0 && supId != 0 && costPrice != 0 { +_, err = goods.NewGoodsSupplierRelationBiz(types.ToTmplConf(s.c)).Add(subCtx, &pb.AddGoodsSupplierRelationReqs{ +SupplierId: supId, +GoodsId: goodsAdd.Id, +WarehouseId: WareHouseId, +IsDefaultWarehouse: pojo.IS_DEFAULT_WAREHOUSE, +SupplierGoodsPrice: float32(costPrice), +Sort: 1, +}) +if err != nil { +excel.AddErr(fmt.Sprintf("添加供应商商品关系失败：%s", err), k+1, v) +continue +} +} +if v["货品图片"] != "" { //商品图片 +for key, media := range strings.Split(v["货品图片"], ",") { +_, _err := goods.NewGoodsMediaBiz(types.ToTmplConf(s.c)).Add(subCtx, &pb.AddGoodsMediaReqs{ +GoodsId: goodsAdd.Id, +Url: media, +Sort: int32(key + 1), +Type: pojo.GOODS_MEDIA_TYPE_IMAGE, +}) +if _err != nil { +excel.AddErr(fmt.Sprintf("添加商品图片失败：%s", err), k+1, v) +continue +} +} +} +excel.Next() +} +}) +``` \ No newline at end of file diff --git a/create_excel.go b/create_excel.go new file mode 100644 index 0000000..7022dfd --- /dev/null +++ b/create_excel.go @@ -0,0 +1,44 @@ +package excel_import + +import "github.com/xuri/excelize/v2" + +type CreateExcel struct { + rowIndex int + file *excelize.File + sw *excelize.StreamWriter + Rows [][]interface{} `json:"rows"` + Header []interface{} `json:"header"` + Path string `json:"path"` + FileName string `json:"file_name"` + ErrFileUrls []string `json:"err_file_urls"` +} + +func (c *CreateExcel) Init() (err error) { + c.file = excelize.NewFile() + c.sw, err = c.file.NewStreamWriter("Sheet1") + err = c.WriteHeader() + return err +} + +func (c *CreateExcel) WriteHeader() error { + return c.Write(c.Header) +} + +func (c *CreateExcel) Write(values []interface{}) error { + + cell, err := excelize.CoordinatesToCellName(1, c.rowIndex+1) + if err != nil { + return err + } + err = c.sw.SetRow(cell, values) + if err != nil { + return err + } + + c.rowIndex++ + return nil +} + +func (c *CreateExcel) Save() error { + return c.file.SaveAs(c.Path + c.FileName) +} diff --git a/func.go b/func.go new file mode 100644 index 0000000..9ff92db --- /dev/null +++ b/func.go @@ -0,0 +1,158 @@ +package excel_import + +import ( + "fmt" + "github.com/bytedance/sonic" + "os" + "regexp" + "sort" + "strings" + "time" +) + +func importLogPath(jobName string) (string, error) { + path, err := os.Getwd() + path = fmt.Sprintf("%s/%s/%s", path, "log/import", jobName) + err = CheckDir(path) + return path, err +} + +func ImportLogPathByOrder(jobName string) (string, error) { + path, err := os.Getwd() + path = fmt.Sprintf("%s/%s/%s/excel/", path, "log/import", jobName) + err = CheckDir(path) + return path, err +} + +func tempFile(jobName string) string { + path, _ := os.Getwd() + path = fmt.Sprintf("%s/%s/%s.xlsx", path, "docs/import_temp", jobName) + + return path +} + +func CheckDir(path string) error { + // 判断目录是否存在 + if _, err := os.Stat(path); os.IsNotExist(err) { + // 如果目录不存在，则创建它 + err = os.MkdirAll(path, os.ModePerm) + if err != nil { + return err + } + } else if err != nil { + // 如果Stat返回了其他错误（比如权限问题） + return err + } + return nil +} + +func SortFileWithStatus(dir string) []FileInfoStatus { + + // 获取目录中的文件信息 + d, _ := os.Open(dir) + defer d.Close() + files, _ := d.ReadDir(0) + + var fileInfoList []FileInfoStatus + + // 填充切片 + for _, file := range files { + fileName := file.Name() + fileInfo, _ := file.Info() + + bytes, _ := os.ReadFile(dir + "/" + fileName) + var info Task + _ = sonic.Unmarshal(bytes, &info) + times, _ := time.Parse(time.DateTime, info.Ctime) + fileInfoList = append(fileInfoList, FileInfoStatus{FileInfo: fileInfo, Status: info.Status, Time: times}) + } + + // 根据修改时间对切片进行排序 + + sort.Slice(fileInfoList, func(i, j int) bool { + return fileInfoList[i].Time.After(fileInfoList[j].Time) + }) + return fileInfoList +} + +// SortHeader 根据SortSlice排序, +// 这里这样设计主要是考虑在实际开发中可能会频繁的出现excel表里面插入或者删除字段，导致之后的导入数据的具体v[n]也需要跟着改的情况 +// HeaderMap决定了v[n]和excel表里面的字段名之间的映射关系 +// SortSlice决定了导入导出数据的顺序，里面的元素可以是HeaderMap里面的key，也可以是HeaderMap里面的value，这个取决于HeaderMap是否为nil +// HeaderMap可以为nil，如果为nil，则SortSlice里面的元素就是excel表里面的字段名，如果HeaderMap不为nil，则SortSlice里面的元素则是经过HeaderMap映射之后的值 +// 严格来说这里的SortSlice应该是一个interface{}类型的切片，这里使用了string是为了方便 +//***最愚蠢的事情就是为了强迫症而强迫症 +//func SortHeader(header *Header) []string { +// if header.HeaderMap == nil { +// return header.SortSlice +// } +// var sortSlice []string +// for _, v := range header.SortSlice { +// if _, exist := header.HeaderMap[v]; exist { +// sortSlice = append(sortSlice, header.HeaderMap[v]) +// } +// } +// return sortSlice +//} + +func ExchangeRows(oldRows [][]string, setHeader []string) (rows []map[string]string) { + oldRowsHeader := oldRows[0] + oldRowsMap := make(map[string]int, len(oldRowsHeader)) + for index, header := range oldRowsHeader { + oldRowsMap[header] = index + } + for _, oldRow := range oldRows { + newRow := make(map[string]string, len(setHeader)) + lenOldRow := len(oldRow) + for _, header := range setHeader { + point, exist := oldRowsMap[header] + if !exist || point >= lenOldRow { + newRow[header] = "" + } else { + newRow[header] = oldRow[point] + } + } + rows = append(rows, newRow) + } + return rows +} + +func ExchangeRowWithMap(oldRow map[string]string, setHeader []string) (rows []string) { + + for _, header := range setHeader { + rows = append(rows, oldRow[header]) + } + return rows +} + +func Ter[T any](cond bool, a, b T) T { + if cond { + return a + } + return b +} + +func RegexMatch(str string, pattern string) bool { + matched, err := regexp.MatchString(pattern, str) + if err != nil { + return false + } + return matched +} + +// IsExcelFormat 检查是否为 Excel 相关格式 +func IsExcelFormat(fileObjectUrl string) bool { + // 支持的扩展名 + allowedExtensions := []string{".xls", ".xlsx", ".csv"} + + // 转为小写，避免大小写问题 + fileObjectUrl = strings.ToLower(fileObjectUrl) + + // 遍历匹配扩展名 + for _, ext := range allowedExtensions { + if strings.HasSuffix(fileObjectUrl, ext) { + return true + } + } + return false +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..481c1e4 --- /dev/null +++ b/go.mod @@ -0,0 +1,36 @@ +module gitea.cdlsxd.cn/self-tools/l_excel_import + +go 1.23.6 + +require ( + github.com/bytedance/sonic v1.12.9 + github.com/go-kratos/kratos/v2 v2.8.3 + github.com/xuri/excelize/v2 v2.9.0 +) + +require ( + github.com/bytedance/sonic/loader v0.2.2 // indirect + github.com/cloudwego/base64x v0.1.5 // indirect + github.com/go-kratos/aegis v0.2.0 // indirect + github.com/go-playground/form/v4 v4.2.0 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/uuid v1.4.0 // indirect + github.com/gorilla/mux v1.8.1 // indirect + github.com/klauspost/cpuid/v2 v2.0.9 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/richardlehane/mscfb v1.0.4 // indirect + github.com/richardlehane/msoleps v1.0.4 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/xuri/efp v0.0.0-20240408161823-9ad904a10d6d // indirect + github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 // indirect + golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect + golang.org/x/crypto v0.28.0 // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/text v0.19.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 // indirect + google.golang.org/grpc v1.61.1 // indirect + google.golang.org/protobuf v1.33.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/import.go b/import.go new file mode 100644 index 0000000..839936f --- /dev/null +++ b/import.go @@ -0,0 +1,658 @@ +package excel_import + +import ( + "context" + "fmt" + "github.com/bytedance/sonic" + "github.com/go-kratos/kratos/v2/log" + "github.com/go-kratos/kratos/v2/transport/http" + "github.com/xuri/excelize/v2" + "io" + "math" + urlnet "net/url" + "os" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" +) + +type ImportExcel struct { + Ctx context.Context + basePath string + task *Task + errRowsExporter *CreateExcel + rowCount int + importRowsCount int32 + RowPicCells []string //图片所在单元格 + PicSaveHandle func(pic *Pic) (url string, err error) //存储图片文件 + DeleteOssHandle func(ObjectName []string) (err error) //删除OSS图片 + GetFileObject func(fileObjectUrl string) (body io.ReadCloser, err error) + Rows []map[string]string + TrimFiled []string //需要进行去掉空格操作的字段 + RegexFiledMap map[string]*Regex + JobName string //任务类型，区分不同业务 + TaskId string //自动生成 + SpeedMod bool //是否开启加速 + SliceLen uint //加速模式下，并行切片长度,默认为100 + Header []string //头部标题对标 + HandleFunc func(excel *ImportExcel, rows []map[string]string) + FileObjectUrl string + DownloadUrl string +} + +type checkOption struct { + TrimCheck bool + RegexCheck bool +} + +func NewImportExcel(jobName string, opts ...Option) *ImportExcel { + Import := &ImportExcel{ + + JobName: jobName, + errRowsExporter: &CreateExcel{}, //错误行导出 + task: &Task{}, //任务 + } + for _, opt := range opts { + opt(Import) // 应用选项 + } + return Import +} + +func (i *ImportExcel) Init(ctx http.Context, handleFunc func(excel *ImportExcel, row []map[string]string)) (*ImportExcel, error) { + i.Ctx = ctx + basePath, err := importLogPath(i.JobName) + i.basePath = basePath + if err != nil { + return nil, err + } + err = CheckDir(i.taskFile()) + if err != nil { + return nil, err + } + err = CheckDir(i.importLogFile()) + if err != nil { + return nil, err + } + err = CheckDir(i.errExcelPath()) + if err != nil { + return nil, err + } + i.HandleFunc = handleFunc + i.TaskId = i.createTaskId() + err = i.GetRows(ctx.Request()) + if err != nil { + return nil, err + } + i.errRowsExporter.FileName = fmt.Sprintf("%s.xlsx", i.TaskId) + i.errRowsExporter.Path = i.errExcelPath() + + return i, nil +} + +func (i *ImportExcel) Run() (err error) { + //创建任务 + i.task = &Task{ + TaskId: i.TaskId, + Ctime: time.Now().Format(time.DateTime), + Status: TaskStatusInit, + } + err = i.updateTask() + if err != nil { + return err + } + i.rowCount = len(i.Rows) - 1 + err = i.filedCheck() + if err != nil { + return err + } + if i.SpeedMod { + if i.SliceLen == 0 { + i.SliceLen = 100 + } + //从第二行开始读取 + for j := 1; j <= len(i.Rows[1:]); j += int(i.SliceLen) { + if j+int(i.SliceLen) > len(i.Rows) { + i.SliceLen = uint(len(i.Rows) - j) + } + //****HandleFunc需要创建子上下文来防止ctx被取消导致上下文丢失**** ChildCtx,cancel:=context.WithCancel(context.BackGround()) defer cancel() + go i.HandleFunc(i, i.Rows[j:j+int(i.SliceLen)]) + } + } else { + i.HandleFunc(i, i.Rows[1:]) + } + i.task.Status = TaskStatusFinish + i.updateTask() + return nil +} + +func (i *ImportExcel) filedCheck() (err error) { + var ( + checkOptions checkOption + errMsg []string + ) + if len(i.TrimFiled) >= 0 { + checkOptions.TrimCheck = true + } + if len(i.RegexFiledMap) >= 0 { + checkOptions.RegexCheck = true + } + + //去掉空格 + trimKeysMap := make(map[string]struct{}) + for _, key := range i.TrimFiled { + trimKeysMap[key] = struct{}{} + } + trimValue := func(value string) string { + return strings.TrimSpace(value) + } + + for line, dataMap := range i.Rows[1:] { + for key, value := range dataMap { + //去掉空格 + if checkOptions.TrimCheck { + if _, exists := trimKeysMap[key]; exists { + value = trimValue(value) + } + } + //正则匹配 + if checkOptions.RegexCheck { + if _, exists := i.RegexFiledMap[key]; exists { + match := RegexMatch(i.RegexFiledMap[key].Rule, value) + if match == i.RegexFiledMap[key].MatchBool { + errMsg = append(errMsg, fmt.Sprintf("第%d行,字段:%s,值:%s,格式错误:%s", line+2, key, value, i.RegexFiledMap[key].Desc)) + } + } + } + dataMap[key] = value + } + } + if len(errMsg) > 0 { + return fmt.Errorf(strings.Join(errMsg, "\n")) + } + return +} + +func (i *ImportExcel) GetRows(request *http.Request) (err error) { + err = i.getRowsFromHttp(request) + if err != nil { + return err + } + if i.Rows == nil || len(i.Rows) == 0 { + return fmt.Errorf("未获取到导入数据或导入数据为空") + } + + return nil +} + +func (i *ImportExcel) GetTaskInfo(taskId string) (*TaskResp, error) { + var ( + info *Task + ) + i.TaskId = taskId + basePath, err := importLogPath(i.JobName) + i.basePath = basePath + if err != nil { + return nil, err + } + taskInfo, _ := os.ReadFile(i.taskFile()) + _ = sonic.Unmarshal(taskInfo, &info) + if info == nil { + // 兼容，提供默认值，类似：{"task_id":"1734576235708222000","process":100,"ctime":"2024-12-19 10:44:03","ftime":"2024-12-19 10:44:03","status":2} + info = &Task{ + TaskId: taskId, + Process: 0, + Status: TaskStatusInit, + Ctime: time.Now().Format(time.DateTime), + Ftime: time.Now().Format(time.DateTime), + } + } + if info.Process >= 95 { + info.Process = 100 + } + + return &TaskResp{ + Task: info, + ImportErrLog: i.importLog(), + }, nil +} + +func (i *ImportExcel) importLog() []*ImportLog { + var logs = []*ImportLog{} + _, err := os.Stat(i.importLogFile()) + if err != nil { + return logs + } + taskLog, err := os.ReadFile(i.importLogFile()) + if err != nil { + return logs + } + logList := strings.Split(string(taskLog), "\n") + for _, v := range logList { + if v == "" { + continue + } + var log *ImportLog + err = sonic.Unmarshal([]byte(v), &log) + if err != nil { + return logs + } + logs = append(logs, log) + } + return logs +} + +func (i *ImportExcel) GetExcel(taskId string) string { + i.TaskId = taskId + basePath, _ := importLogPath(i.JobName) + i.basePath = basePath + return i.errExcelFile() +} + +func (i *ImportExcel) DownloadExcel(ctx http.Context) (err error) { + taskId := ctx.Query().Get("task_id") + addr := i.GetExcel(taskId) + _, exist := os.Stat(addr) + if exist != nil { + return fmt.Errorf("文件不存在") + } + + file, err := os.Open(addr) + if err != nil { + return err + } + defer file.Close() + payload, err := io.ReadAll(file) + if err != nil { + return err + } + // 设置HTTP响应头 + // 打开为预览 + //ctx.Response().Header().Set("Content-Type", "image/png") + // 打开为下载 + ctx.Response().Header().Set("Content-Type", "application/octet-stream") + ctx.Response().Header().Set("Content-Disposition", "attachment; filename="+taskId+".xlsx") + // 将结果写入 + _, err = ctx.Response().Write(payload) + return +} + +func (i *ImportExcel) DownloadExcelTemp(ctx http.Context) (err error) { + + file, err := os.Open(i.DownloadUrl) + if err != nil { + return err + } + defer file.Close() + payload, err := io.ReadAll(file) + if err != nil { + return err + } + // 设置HTTP响应头 + // 打开为预览 + //ctx.Response().Header().Set("Content-Type", "image/png") + // 打开为下载 + ctx.Response().Header().Set("Content-Type", "application/octet-stream") + ctx.Response().Header().Set("Content-Disposition", "attachment; filename="+i.JobName+".xlsx") + // 将结果写入 + _, err = ctx.Response().Write(payload) + return +} + +func (i *ImportExcel) TaskInfo(ctx http.Context) (err error) { + taskId := ctx.Query().Get("task_id") + importExcelInfo, err := i.GetTaskInfo(taskId) + if err != nil { + return err + } + response := make(map[string]interface{}, 1) + + response["data"] = importExcelInfo + return ctx.Result(200, response) +} + +func (i *ImportExcel) TaskHis(ctx http.Context) (err error) { + var ( + data []map[string]interface{} + res ResPage + ) + + page := ctx.Query().Get("page") + num := ctx.Query().Get("limit") + + path := i.taskPath() + + entries := SortFileWithStatus(path) + count := len(entries) + pageInt, _ := strconv.ParseInt(page, 10, 64) + numInt, _ := strconv.ParseInt(num, 10, 64) + begin := (pageInt - 1) * numInt + entEnd := begin + numInt + if count < int(entEnd) { + entEnd = int64(count) + } + entries = entries[begin:entEnd] + for _, entry := range entries { + if entry.FileInfo == nil { + break + } + info := make(map[string]interface{}) + if entry.IsDir() { + continue + } + info["task_id"] = entry.Name() + file := fmt.Sprintf("%s/%s", path, entry.Name()) + bytes, _ := os.ReadFile(file) + _ = sonic.Unmarshal(bytes, &info) + fileOs, _ := os.Stat(file) + info["update_time"] = fileOs.ModTime().Format(time.DateTime) + data = append(data, info) + } + res = ResPage{ + Page: int(pageInt), + Limit: int(numInt), + Total: count, + Data: data, + LastPage: int(math.Ceil(float64(count) / float64(numInt))), + } + response := make(map[string]interface{}, 1) + + response["data"] = res + return ctx.Result(200, response) +} + +func (i *ImportExcel) getRowsFromHttp(r *http.Request) (err error) { + // 定义一个file文件 + var file io.ReadCloser + // 获取上传的文件 + // 获取post请求的参数 + fileObjectUrl := r.FormValue("fileObjectUrl") + if i.GetFileObject != nil && fileObjectUrl != "" { + // 判断fileObjectUrl 是否为excel相关得格式 + if !IsExcelFormat(fileObjectUrl) { + return fmt.Errorf("文件格式错误, 不是excel") + } + file, err = i.GetFileObject(fileObjectUrl) + if err != nil { + return err + } + // 记录excel地址，用于删除文件 + i.FileObjectUrl = fileObjectUrl + } else { + file, _, err = r.FormFile("file") + speed_mode := r.PostForm.Get("speed_mode") + if strings.EqualFold(speed_mode, "true") { + i.SpeedMod = true + } + if err != nil { + return fmt.Errorf("未找到导入文件: %w", err) + } + } + defer file.Close() + + // 解析Excel文件 + f, err := excelize.OpenReader(file) + if err != nil { + return fmt.Errorf("解析excel文件失败: %w", err) + } + defer f.Close() + // 获取第一个工作表的名称 + sheetNames := f.GetSheetList() + if len(sheetNames) == 0 { + return fmt.Errorf("无效的excel，未获取到对应的sheet") + } + rows, err := f.GetRows(sheetNames[0]) + if err != nil { + return fmt.Errorf("excel内未找到数据: %w", err) + } + + if i.PicSaveHandle != nil { + i.RowPicCells, err = f.GetPictureCells(sheetNames[0]) + if err != nil { + return fmt.Errorf("excel内图片获取失败: %w", err) + } + // 遍历所有的图片 + var picList []*Pic + for _, cell := range i.RowPicCells { + if cell == "N1" { + continue + } + pics, _err := f.GetPictures(sheetNames[0], cell) + if _err != nil { + return fmt.Errorf("excel内未找到图片: %w", _err) + } + if len(pics) > 0 { + for _, v := range pics { + if v.File != nil { + picList = append(picList, &Pic{ + PicInfos: v, + Cell: cell, + }) + } + } + } + } + fmt.Printf("开始上传OSS，时间：%s \n", time.Now().Format(time.DateTime)) + if len(picList) > 0 { + var ( + cellUrlMap = make(map[string][]string) + wg = sync.WaitGroup{} + mu sync.Mutex + ) + wg.Add(len(picList)) + for _, pic := range picList { + go func() { + defer wg.Done() + defer func() { + if r := recover(); r != nil { + fmt.Printf("图片上传失败，cell:%s,url:%s,err:%s", pic.Cell, pic.Url, r) + } + }() + url, _err := i.PicSaveHandle(pic) + mu.Lock() + if _err != nil { + log.Error(fmt.Sprintf("图片上传失败，cell:%s,url:%s,err:%s", pic.Cell, pic.Url, _err.Error())) + } + cellUrlMap[pic.Cell] = append(cellUrlMap[pic.Cell], url) + mu.Unlock() + }() + } + wg.Wait() + fmt.Printf("结束上传OSS，时间：%s \n", time.Now().Format(time.DateTime)) + if len(cellUrlMap) > 0 { + for cell, cellPic := range cellUrlMap { + err = f.SetCellValue(sheetNames[0], cell, strings.Join(cellPic, ",")) + if err != nil { + continue + } + } + } + } + fmt.Printf("os--暂无图片，时间：%s \n", time.Now().Format(time.DateTime)) + //从新获取一次 + rows, err = f.GetRows(sheetNames[0]) + if err != nil { + return fmt.Errorf("excel内未找到数据: %w", err) + } + } + // 获取所有行 + + if len(rows) == 0 { + return fmt.Errorf("无效的excel，未获取到对应的记录数据") + } + + if len(i.Header) > 0 || i.Header != nil { + + i.Rows = ExchangeRows(rows, i.Header) + } + + return err +} + +func (i *ImportExcel) getRowsFromFile(filePath string) (err error) { + + return err +} + +func (i *ImportExcel) updateTask() error { + + file, err := os.OpenFile(i.taskFile(), os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0766) + if err != nil { + return err + } + defer file.Close() + + taskInfo, err := sonic.Marshal(i.task) + if err != nil { + return err + } + jsonInfo := string(taskInfo) + _, err = file.WriteString(jsonInfo) + if err != nil { + return err + } + return nil +} +func (i *ImportExcel) AddErr(failReason string, key int, row map[string]string) { + + _ = i.updateImportLog(&ImportLog{ + FailReason: failReason, + Line: key, + }) + + var interfaces []interface{} + rowSort := ExchangeRowWithMap(row, i.Header) + for _, v := range rowSort { + interfaces = append(interfaces, v) + } + interfaces = append(interfaces, failReason) + i.errRowsExporter.Rows = append(i.errRowsExporter.Rows, interfaces) + i.errRowsExporter.ErrFileUrls = append(i.errRowsExporter.ErrFileUrls, row["货品图片"]) + i.Next() + + return +} + +func (i *ImportExcel) Next() (err error) { + atomic.AddInt32(&i.importRowsCount, 1) + i.task.Process = int(i.importRowsCount * 100 / int32(i.rowCount)) + i.updateTask() + if int(i.importRowsCount) == i.rowCount { + i.task.Process = 100 + i.task.Status = TaskStatusCreateFailExcel + err = i.updateTask() + if err != nil { + return + } + if len(i.errRowsExporter.Rows) > 0 { + headerInterfaces := make([]interface{}, len(i.Header)+1) + for index, header := range i.Header { + headerInterfaces[index] = header + } + headerInterfaces[len(headerInterfaces)-1] = "失败原因" + i.errRowsExporter.Header = headerInterfaces + err = i.errRowsExporter.Init() + if err != nil { + return + } + for _, v := range i.errRowsExporter.Rows { + _ = i.errRowsExporter.Write(v) + } + i.errRowsExporter.Save() + } + // 删除已经上传到OSS的图片 + if len(i.errRowsExporter.ErrFileUrls) > 0 { + var errFileObjectName []string + for _, url := range i.errRowsExporter.ErrFileUrls { + if url == "" { + continue + } + ossUrl := strings.Split(url, ",") + for _, v := range ossUrl { + parsedURL, err := urlnet.Parse(v) + if err != nil { + fmt.Printf("解析URL失败: %v\n", err) + continue + } + // 去掉path前面得/ + parsedURL.Path = strings.TrimPrefix(parsedURL.Path, "/") + errFileObjectName = append(errFileObjectName, parsedURL.Path) + } + } + // 批量删除OSS文件 + if len(errFileObjectName) > 0 { + if i.DeleteOssHandle != nil { + err = i.DeleteOssHandle(errFileObjectName) + if err != nil { + return + } + } + } + } + + if i.FileObjectUrl != "" { + // 批量删除OSS文件 + if i.DeleteOssHandle != nil { + err = i.DeleteOssHandle([]string{i.FileObjectUrl}) + if err != nil { + return + } + } + } + } + i.task.Status = TaskStatusRunning + i.task.Ftime = time.Now().Format(time.DateTime) + i.updateTask() + return nil +} + +func (i *ImportExcel) CreateFailExcel() { + i.importRowsCount++ +} + +func (i *ImportExcel) updateImportLog(importLog *ImportLog) error { + file, err := os.OpenFile(i.importLogFile(), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0766) + if err != nil { + return err + } + defer file.Close() + logInfo, err := sonic.Marshal(importLog) + if err != nil { + return err + } + jsonInfo := string(logInfo) + _, err = file.WriteString(jsonInfo + "\n") + if err != nil { + return err + } + return nil +} + +func (i *ImportExcel) taskFile() string { + return i.basePath + "/task/" + i.TaskId +} + +func (i *ImportExcel) taskPath() string { + if i.basePath == "" { + basePath, _ := importLogPath(i.JobName) + i.basePath = basePath + } + return i.basePath + "/task/" +} + +func (i *ImportExcel) importLogFile() string { + return i.basePath + "/import/" + i.TaskId +} + +func (i *ImportExcel) errExcelPath() string { + return i.basePath + "/excel/" +} + +func (i *ImportExcel) errExcelFile() string { + return i.errExcelPath() + i.TaskId + ".xlsx" +} + +func (i *ImportExcel) createTaskId() string { + return fmt.Sprintf("%d", time.Now().UnixNano()) +} diff --git a/import_test.go b/import_test.go new file mode 100644 index 0000000..18d80a6 --- /dev/null +++ b/import_test.go @@ -0,0 +1,17 @@ +package excel_import + +import ( + "fmt" + "testing" +) + +func TestCut(t *testing.T) { + data := [][]string{{"a", "b", "c"}, {"d", "e", "f"}, {"d", "e", "f"}, {"d", "e", "f"}, {"d", "e", "f"}} + + fmt.Println(data[1:]) +} + +func TestTaskInfo(t *testing.T) { + NewImportExcel("goods_import").GetTaskInfo("1727160398921095702") + +} diff --git a/option.go b/option.go new file mode 100644 index 0000000..4a0cbe1 --- /dev/null +++ b/option.go @@ -0,0 +1,72 @@ +package excel_import + +import "io" + +type ( + Option func(*ImportExcel) + Regex struct { + Rule string //匹配正则 + MatchBool bool //匹配结果为true抛错还是匹配为false抛错 + Desc string //抛错内容 + } +) + +func WithSpeedMod(speedMod bool) Option { + return func(s *ImportExcel) { + s.SpeedMod = speedMod + } +} + +func WithSliceLen(sliceLen uint) Option { + return func(s *ImportExcel) { + s.SliceLen = sliceLen + } +} + +func WithHeader(header []string) Option { + return func(s *ImportExcel) { + s.Header = header + } +} + +func WithTaskId(taskId string) Option { + return func(s *ImportExcel) { + s.TaskId = taskId + } +} + +func WithRowPicHandle(picSaveHandle func(pic *Pic) (url string, err error)) Option { + return func(s *ImportExcel) { + s.PicSaveHandle = picSaveHandle + } +} + +func WithDeleteOssHandle(deleteOssHandle func(ObjectName []string) (err error)) Option { + return func(s *ImportExcel) { + s.DeleteOssHandle = deleteOssHandle + } +} + +func WithGetFileObject(getFileObject func(fileObjectUrl string) (body io.ReadCloser, err error)) Option { + return func(s *ImportExcel) { + s.GetFileObject = getFileObject + } +} + +func WithTrimFiled(trimFiledMap []string) Option { + return func(s *ImportExcel) { + s.TrimFiled = trimFiledMap + } +} + +func WithFiledRegex(regexFiledMap map[string]*Regex) Option { + return func(s *ImportExcel) { + s.RegexFiledMap = regexFiledMap + } +} + +func WithExcelDownLoadUrl(url string) Option { + return func(s *ImportExcel) { + s.DownloadUrl = url + } +} diff --git a/types.go b/types.go new file mode 100644 index 0000000..26b1ab3 --- /dev/null +++ b/types.go @@ -0,0 +1,80 @@ +package excel_import + +import ( + "github.com/xuri/excelize/v2" + "io/fs" + "time" +) + +type ( + Task struct { + TaskId string `json:"task_id"` + Process int `json:"process"` + Ctime string `json:"ctime"` + Ftime string `json:"ftime"` + Status int `json:"status"` + } + + ImportLog struct { + FailReason string `json:"fail_reason"` + Line int `json:"line"` + } + + RowInfo struct { + Row []string + } + + TaskResp struct { + Task *Task `json:"task"` + ImportErrLog []*ImportLog `json:"log"` + } + + FileInfoStatus struct { + fs.FileInfo + Status int + Time time.Time + } + + ResPage struct { + Page int `json:"current_page"` + Limit int `json:"per_page"` + Total int `json:"total"` + LastPage int `json:"last_page"` + Data []map[string]interface{} `json:"data"` + } + + PicList struct { + PicInfos []excelize.Picture + Url []string + } + + Pic struct { + PicInfos excelize.Picture + Url string + Cell string + } + + //Header struct { + // HeaderMap map[string]string + // SortSlice []string + //} +) + +const ( + TaskStatusInit = iota + 1 // 初始化 + TaskStatusRunning + TaskStatusCreateFailExcel + TaskStatusFinish +) + +const ( + Rows_From_Request = iota + 1 + Rows_From_File + Rows_From_Rows +) + +var fromWays = map[int]string{ + Rows_From_Request: "request", + Rows_From_File: "file", + Rows_From_Rows: "rows", +}