csv.go

@@ -10,22 +10,11 @@ import (
 	"strings"
 )
 
-func getCsvInfo(dir string) (dataMap []map[string]string, title []string, files []string) {
+func getCsvInfo(dir string) (dataMap []map[string]string, title []string) {
 	fp := fmt.Sprintf("%s/%s/", dir, "csvfile")
-	if _, err := os.Stat(fp); os.IsNotExist(err) {
-		// 文件夹不存在，创建它
-		err := os.Mkdir(fp, 0755) // 0755 是常见的目录权限
-		if err != nil {
-			fatal("无法创建目录 %s: %v", fp, err)
-		}
-	}
 	entries, err := os.ReadDir(fp)
 	if err != nil {
-		fatal("获取文件目录失败 %s: %v", fp, err)
+		panic("当前目录下未找到csvfile文件夹，请自行创建")
-	}
-	if len(entries) == 0 {
-		warning("请将需导入文件放入%s目录下", fp)
-		exit()
 	}
 	var csvFiles []string
 	for _, entry := range entries {
@@ -36,46 +25,18 @@ func getCsvInfo(dir string) (dataMap []map[string]string, title []string, files
 	for {
 		prompt := promptui.Select{
 			Label: "请选择需要导入的csv文件: ",
-			Items: append([]string{"全选", "导入非`已导入_`开头的文件"}, csvFiles...),
+			Items: csvFiles,
-			Size:  len(csvFiles) + 2,
+			Size:  len(csvFiles),
 		}
 		_, f, err := prompt.Run()
 		if err != nil {
 			warning("Prompt failed %v\n", err)
 			continue
 		}
-		f = strings.TrimSpace(f)
+		fp = fmt.Sprintf("%s/%s", fp, f)
-
-		switch f {
-		case "全选":
-			for _, s := range csvFiles {
-				files = append(files, fmt.Sprintf("%s%s", fp, s))
-			}
-
-		case "导入非`已导入_`开头的文件":
-			for _, s := range csvFiles {
-				if !strings.HasPrefix(s, "已导入_") {
-					files = append(files, fmt.Sprintf("%s%s", fp, s))
-				}
-			}
-		default:
-			files = append(files, fmt.Sprintf("%s%s", fp, f))
-		}
 		break
 	}
-	if len(files) == 0 {
+	return ReadCsvData(fp)
-		warning("未找到符合条件的文件")
-		finish(dir)
-	}
-
-	for _, v := range files {
-		data, fileTitle := ReadCsvData(v)
-		if title == nil {
-			title = fileTitle
-		}
-		dataMap = append(dataMap, data...)
-	}
-	return dataMap, title, files
 }
 
 func ReadCsvData(fp string) (dataMap []map[string]string, title []string) {

func.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"fmt"
-
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -25,13 +24,11 @@ func saveData(dataChan chan []map[string]string, set *Set, title []string, dir s
 	var (
 		wg      sync.WaitGroup
 		current int64
-		errCount atomic.Int32
 	)
 
 	lent := len(dataChan)
 	wg.Add(lent)
 	for v := range dataChan {
-		v := v
 		go func() {
 			defer wg.Done()
 			defer func() {
@@ -47,92 +44,13 @@ func saveData(dataChan chan []map[string]string, set *Set, title []string, dir s
 				}
 			}()
 
-			switch set.Op {
-			case Add:
-				add(title, v, &errCount)
-			case OverWrite:
-				table, err := db.DB()
-				if err != nil {
-					warning(err.Error())
-				}
-				defer table.Close()
-				for _, item := range v {
-					var (
-						dataRaw strings.Builder
-						judge   strings.Builder
-					)
-
-					is_start := true
-					dataRaw.WriteString(fmt.Sprintf("SELECT %s From %s WHERE ", fmt.Sprintf("`%s`", strings.Join(set.OverWriteJudColumns, "`,`")), c.Table))
-					for _, t := range set.OverWriteJudColumns {
-						if strings.Contains(item[t], "'") {
-							item[t] = strings.ReplaceAll(item[t], "'", "`")
-						}
-						if strings.Contains(item[t], `"`) {
-							item[t] = strings.ReplaceAll(item[t], `"`, "`")
-						}
-						if is_start {
-							judge.WriteString(fmt.Sprintf("%s=%s", t, item[t]))
-							is_start = false
-							continue
-						}
-						judge.WriteString(fmt.Sprintf(" AND %s=%s", t, item[t]))
-					}
-					dataRaw.WriteString(judge.String())
-					dataRaw.WriteString(" LIMIT 1")
-					raw := dataRaw.String()
-
-					result, _err := table.Query(raw)
-					defer result.Close()
-					if _err != nil {
-						warning(fmt.Sprintf(" 查询失败: %v", _err))
-						return
-					}
-					var adds []map[string]string
-					if result.Next() {
-						var (
-							updateRaw       strings.Builder
-							updateStartLock = false
-						)
-						updateRaw.WriteString(fmt.Sprintf("Update %s Set ", c.Table))
-						for key, value := range item {
-							if !updateStartLock {
-								updateRaw.WriteString(fmt.Sprintf(" %s=%s", key, value))
-							}
-							updateRaw.WriteString(fmt.Sprintf(",%s=%s", key, value))
-						}
-						updateRaw.WriteString(judge.String())
-						update := updateRaw.String()
-						result := db.Exec(update)
-						if result.Error != nil {
-							errCount.Add(1)
-							warning(fmt.Sprintf("failed to insert user: %v", result.Error))
-							return
-						}
-					} else {
-						adds = append(adds, item)
-					}
-					if len(adds) > 0 {
-						add(title, adds, &errCount)
-					}
-
-				}
-			default:
-
-			}
-		}()
-	}
-	wg.Wait()
-	success("导入完毕.....失败：%d", errCount.Load())
-
-	finish(dir)
-}
-
-func add(title []string, v []map[string]string, errCount *atomic.Int32) {
 			var (
 				err   error
 				query strings.Builder
 			)
+
+			switch set.Op {
+			case Add:
 				query.WriteString(fmt.Sprintf("INSERT INTO %s (`%s`) VALUES ", c.Table, strings.Join(title, "`,`")))
 				is_start := true
 				for _, item := range v {
@@ -160,13 +78,38 @@ func add(title []string, v []map[string]string, errCount *atomic.Int32) {
 				raw := query.String()
 				result := db.Exec(raw)
 				if result.Error != nil {
-		errCount.Add(1)
 					warning(fmt.Sprintf("failed to insert user: %v", result.Error))
 					return
 				}
 				if err != nil {
-		errCount.Add(1)
 					warning("数据保存失败:%v", err)
 					return
 				}
+
+			case OverWrite:
+				for _, item := range v {
+					var dataRaw strings.Builder
+					is_start := true
+					dataRaw.WriteString(fmt.Sprintf("SELECT %s From %s WHERE ", strings.Join(title, "`,`"), c.Table))
+					for _, t := range set.OverWriteJudColumns {
+						if strings.Contains(item[t], "'") {
+							item[t] = strings.ReplaceAll(item[t], "'", "`")
+						}
+						if strings.Contains(item[t], `"`) {
+							item[t] = strings.ReplaceAll(item[t], `"`, "`")
+						}
+						if !is_start {
+							dataRaw.WriteString(fmt.Sprintf("'%s',", item[t]))
+						}
+						dataRaw.WriteString(fmt.Sprintf("'%s',", item[t]))
+					}
+				}
+			default:
+				query.WriteString(";")
+			}
+		}()
+	}
+	wg.Wait()
+	success("导入完毕.....")
+	finish(dir)
 }

import_csv_data

@@ -0,0 +1 @@
+s1QM2t7N1NQtPksWJpIjU5swReLzUGryqueba8bh/Yj0YQB0zMpxxT/kwjE09zq2ZBcC4TvUr4SHL6WbcT3OT991BATzhZj5s1LJ9oo1ZWFObSZUHhGYiUJ+y92ynZYGMP2kHMrrfNgHkGVpjvcUo8mIbDIJAW6m9XUSYBwdYUZoecXe35g8F71l11ipDdfMDWmOH3DZzy9tzBM=

main.go

@@ -7,7 +7,6 @@ import (
 	"gorm.io/gorm"
 	"os"
 	"strings"
-	"time"
 )
 
 var (
@@ -37,8 +36,6 @@ var (
 	db *gorm.DB
 )
 
-var Cut int = 1
-
 func main() {
 	log("正在读取配置。。。")
 	dir, err := os.Getwd()
@@ -57,11 +54,10 @@ func main() {
 
 }
 func do(dir string) {
-	csvData, title, files := getCsvInfo(dir)
+	csvData, title := getCsvInfo(dir)
 	set := setOp(title)
-	cutChannel := cutData(csvData, Cut)
+	cutChannel := cutData(csvData, 100)
 	saveData(cutChannel, set, title, dir)
-	overFiles(files)
 	finish(dir)
 	select {}
 }
@@ -145,29 +141,9 @@ func cConf(fp string) *Conf {
 }
 
 func exit() {
-	waitTime := 3
-	warning("程序即将再%d秒后退出", waitTime)
-	for {
-		if waitTime == 0 {
 	os.Exit(1)
-		}
-		time.Sleep(1 * time.Second)
-		warning("%d", waitTime)
-		waitTime--
-	}
-
 }
 
 func finish(dir string) {
 	do(dir)
 }
-
-func overFiles(files []string) {
-	for _, v := range files {
-		err := os.Rename(v, fmt.Sprintf("已导入_%s", v))
-		if err != nil {
-			warning("重命名文件时出错: %v\n", err)
-			exit()
-		}
-	}
-}

op.go

@@ -26,7 +26,7 @@ func setOp(title []string) *Set {
 	for {
 		prompt := promptui.Select{
 			Label: "请选择需要执行的操作: ",
-			Items: []string{"新增"}, // "覆盖(有则改，无则增)"
+			Items: []string{"新增"}, //, "覆盖(有则改，无则增)"
 		}
 		_, r, err := prompt.Run()
 		if err != nil {