From f42f667764397060a252648332d7c6c1a3dafc0d Mon Sep 17 00:00:00 2001 From: renzhiyuan <465386466@qq.com> Date: Tue, 27 May 2025 15:37:42 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0CSV=E5=AF=BC=E5=85=A5?= =?UTF-8?q?=E5=B7=A5=E5=85=B7=E5=8F=8A=E6=95=B0=E6=8D=AE=E5=BA=93=E6=93=8D?= =?UTF-8?q?=E4=BD=9C=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ans.go | 40 ++++++++++++ crypt.go | 87 +++++++++++++++++++++++++ csv.go | 166 ++++++++++++++++++++++++++++++++++++++++++++++++ db_operate.go | 81 +++++++++++++++++++++++ func.go | 115 +++++++++++++++++++++++++++++++++ gen.sh | 19 ++++++ go.mod | 18 ++++++ import_csv_data | 1 + main.go | 149 +++++++++++++++++++++++++++++++++++++++++++ op.go | 77 ++++++++++++++++++++++ test.go | 8 +++ 11 files changed, 761 insertions(+) create mode 100644 ans.go create mode 100644 crypt.go create mode 100644 csv.go create mode 100644 db_operate.go create mode 100644 func.go create mode 100755 gen.sh create mode 100644 go.mod create mode 100644 import_csv_data create mode 100644 main.go create mode 100644 op.go create mode 100644 test.go diff --git a/ans.go b/ans.go new file mode 100644 index 0000000..5c302fd --- /dev/null +++ b/ans.go @@ -0,0 +1,40 @@ +package main + +import "fmt" + +type ANS string + +const ( + reset ANS = "\033[0m" + red ANS = "\033[31m" + green ANS = "\033[32m" + yellow ANS = "\033[33m" + blue ANS = "\033[34m" + purple ANS = "\033[35m" + cyan ANS = "\033[36m" + white ANS = "\033[37m" +) + +func success(content string, arg ...interface{}) { + fmt.Printf("%s%s%s\n", green, fmt.Sprintf(content, arg...), reset) +} + +func fatal(content string, arg ...interface{}) { + panic(fmt.Sprintf("%s%s%s\n", red, fmt.Sprintf(content, arg...))) +} + +func warning(content string, arg ...interface{}) { + fmt.Printf("%s%s%s\n", yellow, fmt.Sprintf(content, arg...), reset) +} + +func log(content string, arg ...interface{}) { + fmt.Printf("%s\n", fmt.Sprintf(content, arg...)) +} + +func input(content string, arg ...interface{}) { + fmt.Printf("\n%s%s%s", cyan, fmt.Sprintf(content, arg...), reset) +} + +func tip(content string, arg ...interface{}) { + fmt.Printf("\n*%s%s%s", purple, fmt.Sprintf(content, arg...), reset) +} diff --git a/crypt.go b/crypt.go new file mode 100644 index 0000000..33c6170 --- /dev/null +++ b/crypt.go @@ -0,0 +1,87 @@ +package main + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/base64" + "errors" + "io" +) + +func padToLength(s string, length int, padChar rune) string { + if len(s) >= length { + return s + } + // 计算需要填充的字符数 + padding := make([]rune, length-len(s)) + for i := range padding { + padding[i] = padChar + } + // 将原字符串和填充字符组合起来 + return s + string(padding) +} + +// 加密函数 +func encrypt(key []byte, text string) (string, error) { + plaintext := []byte(text) + + // 创建一个新的cipher.Block + block, err := aes.NewCipher(key) + if err != nil { + return "", err + } + + // 创建一个新的GCM模式的加密器 + gcm, err := cipher.NewGCM(block) + if err != nil { + return "", err + } + + // 创建一个nonce（随机数） + nonce := make([]byte, gcm.NonceSize()) + if _, err = io.ReadFull(rand.Reader, nonce); err != nil { + return "", err + } + + // 加密数据 + ciphertext := gcm.Seal(nonce, nonce, plaintext, nil) + + // 返回Base64编码的加密字符串 + return base64.StdEncoding.EncodeToString(ciphertext), nil +} + +// 解密函数 +func decrypt(key []byte, encryptedText string) (string, error) { + // 解码Base64字符串 + ciphertext, err := base64.StdEncoding.DecodeString(encryptedText) + if err != nil { + return "", err + } + + // 创建一个新的cipher.Block + block, err := aes.NewCipher(key) + if err != nil { + return "", err + } + + // 创建一个新的GCM模式的解密器 + gcm, err := cipher.NewGCM(block) + if err != nil { + return "", err + } + + // 分离nonce和实际加密数据 + nonceSize := gcm.NonceSize() + if len(ciphertext) < nonceSize { + return "", errors.New("ciphertext too short") + } + + nonce, ciphertext := ciphertext[:nonceSize], ciphertext[nonceSize:] + plaintext, err := gcm.Open(nil, nonce, ciphertext, nil) + if err != nil { + return "", err + } + + return string(plaintext), nil +} diff --git a/csv.go b/csv.go new file mode 100644 index 0000000..4ac6631 --- /dev/null +++ b/csv.go @@ -0,0 +1,166 @@ +package main + +import ( + "encoding/csv" + "fmt" + "github.com/manifoldco/promptui" + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/transform" + "os" + "strings" +) + +func getCsvInfo(dir string) (dataMap []map[string]string, title []string) { + fp := fmt.Sprintf("%s/%s/", dir, "csvfile") + entries, err := os.ReadDir(fp) + if err != nil { + panic("当前目录下未找到csvfile文件夹，请自行创建") + } + var csvFiles []string + for _, entry := range entries { + if !entry.IsDir() && strings.HasSuffix(strings.ToLower(entry.Name()), ".csv") { + csvFiles = append(csvFiles, entry.Name()) + } + } + for { + prompt := promptui.Select{ + Label: "请选择需要导入的csv文件: ", + Items: csvFiles, + Size: len(csvFiles), + } + _, f, err := prompt.Run() + if err != nil { + warning("Prompt failed %v\n", err) + continue + } + fp = fmt.Sprintf("%s/%s", fp, f) + break + } + return ReadCsvData(fp) +} + +func ReadCsvData(fp string) (dataMap []map[string]string, title []string) { + log("正在读取CSV文件，请稍等...") + // 打开CSV文件 + file, err := os.Open(fp) + if err != nil { + fatal("无法打开文件: %v", err) + } + defer file.Close() + decoder := simplifiedchinese.GBK.NewDecoder() + // 创建CSV读取器 + reader := csv.NewReader(transform.NewReader(file, decoder)) + // 可选：设置更宽松的解析选项 + reader.LazyQuotes = true // 允许非引号包裹的字段中包含引号 + // 读取所有记录 + records, err := reader.ReadAll() + if err != nil { + fatal("读取CSV记录出错: %v", err) + } + if len(records) < 2 { + fatal("CSV文件至少需要两行，第一行是标题，第二行是数据") + } + // 创建map来存储数据 + dataMap = make([]map[string]string, len(records)) + title = records[0] + for i := 1; i < len(records); i++ { + + dataMap[i-1] = make(map[string]string) + for j := 0; j < len(records[i]); j++ { // 假设第一列是唯一标识符 + dataMap[i-1][title[j]] = records[i][j] + } + } + + log("文件读取完毕") + return +} + +func desc(video *Video) { + var d strings.Builder + tip("描述信息，请输入一段描述文字，描述文字越多，生成视频越精彩。") + input("请设置描述信息: ") + text, _ := reader.ReadString('\n') + d.WriteString(fmt.Sprintf("%s ", strings.TrimSpace(text))) + d.WriteString("--watermark ") + d.WriteString("false ") + for { + prompt := promptui.Select{ + Label: "分辨率: ", + Items: []string{"480p", "720p"}, + } + _, r, err := prompt.Run() + if err != nil { + warning("Prompt failed %v\n", err) + continue + } + d.WriteString("--resolution ") + d.WriteString(r) + d.WriteString(" ") + break + } + + for { + prompt := promptui.Select{ + Label: "视频时长(秒): ", + Items: []string{"5", "10"}, + } + _, r, err := prompt.Run() + if err != nil { + warning("Prompt failed %v\n", err) + continue + } + d.WriteString("--dur ") + d.WriteString(r) + d.WriteString(" ") + break + } + tip("adaptive：根据所上传图片的比例，自动选择最合适的宽高比。") + for { + prompt := promptui.Select{ + Label: "视频的宽高比例: ", + Items: []string{"adaptive", "16:9", "9:16", "4:3", "3:4", "21:9", "9:21", "1:1"}, + } + _, r, err := prompt.Run() + if err != nil { + warning("Prompt failed %v\n", err) + continue + } + d.WriteString("--ratio ") + d.WriteString(r) + d.WriteString(" ") + break + } + + for { + prompt := promptui.Select{ + Label: "帧率: ", + Items: []string{"16", "24"}, + } + _, r, err := prompt.Run() + if err != nil { + warning("Prompt failed %v\n", err) + continue + } + d.WriteString("--framepersecond ") + d.WriteString(r) + d.WriteString(" ") + break + } + + for { + prompt := promptui.Select{ + Label: "是否固定摄像头: ", + Items: []string{"false", "true"}, + } + _, r, err := prompt.Run() + if err != nil { + warning("Prompt failed %v\n", err) + continue + } + d.WriteString("--camerafixed ") + d.WriteString(r) + d.WriteString(" ") + break + } + video.Text = d.String() +} diff --git a/db_operate.go b/db_operate.go new file mode 100644 index 0000000..3880540 --- /dev/null +++ b/db_operate.go @@ -0,0 +1,81 @@ +package main + +import ( + "database/sql" + "errors" + "gorm.io/driver/mysql" + "gorm.io/gorm" +) + +var DbConnPools = make(map[string]*gorm.DB) + +func setDb() (sql *sql.DB, err error) { + db, err = InstanceDb(c.Dns) + if err != nil { + fatal("数据库连接失败:%v", err) + } + + return db.DB() +} + +func InstanceDb(dns string) (*gorm.DB, error) { + var ( + dbConn *gorm.DB + ok bool + ) + if dbConn, ok = DbConnPools[dns]; ok { + sqlDB, _ := dbConn.DB() + connTest := sqlDB.Ping() + if connTest == nil { + return nil, nil + } + } + + db, err := NewDb(&DbConfig{ + DriverName: "mysql", + Dns: dns, + Debug: true, + }) + + if err != nil { + return nil, errors.New("s链接失败") + } + + return db, nil +} + +type DbConfig struct { + DriverName string + Dns string + Debug bool + MaxIdleConns int +} + +func NewDb(config *DbConfig, opt ...gorm.Option) (gorm *gorm.DB, err error) { + gorm, err = config.Conn(opt...) + return gorm, err +} + +func (m *DbConfig) Conn(opt ...gorm.Option) (*gorm.DB, error) { + dial := DriverMysql(m.Dns) + + db, err := gorm.Open( + dial, + opt..., + ) + + if err != nil { + return nil, err + } + sqlDB, _ := db.DB() + if m.Debug { + db = db.Debug() + } + sqlDB.SetMaxIdleConns(3) // 设置最大空闲连接数 + sqlDB.SetMaxOpenConns(10) // 设置最大打开连接数 + return db, nil +} + +func DriverMysql(dns string) gorm.Dialector { + return mysql.Open(dns + "") +} diff --git a/func.go b/func.go new file mode 100644 index 0000000..90fba1d --- /dev/null +++ b/func.go @@ -0,0 +1,115 @@ +package main + +import ( + "fmt" + "strings" + "sync" + "sync/atomic" +) + +func cutData(data []map[string]string, cut int) chan []map[string]string { + lent := (len(data) / cut) + 1 + batchChan := make(chan []map[string]string, lent) + for i := 0; i < len(data); i += cut { + if i+cut > len(data) { + batchChan <- data[i:] + continue + } + batchChan <- data[i : i+cut] + } + return batchChan +} + +func saveData(dataChan chan []map[string]string, set *Set, title []string, dir string) { + var ( + wg sync.WaitGroup + current int64 + ) + + lent := len(dataChan) + wg.Add(lent) + for v := range dataChan { + go func() { + defer wg.Done() + defer func() { + if e := recover(); e != nil { + warning(fmt.Sprintf("协程弹出")) + } + }() + defer func() { + atomic.AddInt64(¤t, 1) + log("当前进度:%d/%d", current, lent) + if current == int64(lent) { + close(dataChan) + } + }() + + var ( + err error + query strings.Builder + ) + + switch set.Op { + case Add: + query.WriteString(fmt.Sprintf("INSERT INTO %s (`%s`) VALUES ", c.Table, strings.Join(title, "`,`"))) + is_start := true + for _, item := range v { + var dataRaw strings.Builder + if !is_start { + dataRaw.WriteString(",") + } else { + is_start = false + } + dataRaw.WriteString(" (") + for _, t := range title { + if strings.Contains(item[t], "'") { + item[t] = strings.ReplaceAll(item[t], "'", "`") + } + if strings.Contains(item[t], `"`) { + item[t] = strings.ReplaceAll(item[t], `"`, "`") + } + dataRaw.WriteString(fmt.Sprintf("'%s',", item[t])) + } + str := dataRaw.String() + str = str[:len(str)-1] + query.WriteString(str + ")") + } + query.WriteString(";") + raw := query.String() + result := db.Exec(raw) + if result.Error != nil { + warning(fmt.Sprintf("failed to insert user: %v", result.Error)) + return + } + if err != nil { + warning("数据保存失败:%v", err) + return + } + + case OverWrite: + for _, item := range v { + var dataRaw strings.Builder + is_start := true + dataRaw.WriteString(fmt.Sprintf("SELECT %s From %s WHERE ", strings.Join(title, "`,`"), c.Table)) + for _, t := range set.OverWriteJudColumns { + if strings.Contains(item[t], "'") { + item[t] = strings.ReplaceAll(item[t], "'", "`") + } + if strings.Contains(item[t], `"`) { + item[t] = strings.ReplaceAll(item[t], `"`, "`") + } + if !is_start { + dataRaw.WriteString(fmt.Sprintf("'%s',", item[t])) + } + dataRaw.WriteString(fmt.Sprintf("'%s',", item[t])) + } + } + default: + query.WriteString(";") + } + }() + } + wg.Wait() + success("导入完毕.....") + finish(dir) +} diff --git a/gen.sh b/gen.sh new file mode 100755 index 0000000..f45c3dc --- /dev/null +++ b/gen.sh @@ -0,0 +1,19 @@ +#!/bin/bash + + # 使用方法： + # ./genModel.sh usercenter user + # ./genModel.sh usercenter user_auth + # 再将./genModel下的文件剪切到对应服务的model目录里面，记得改package + + + #生成的表名 + tables=$1 + #表生成的genmodel目录 + modeldir=./ + + # 数据库配置 + + + + +gentool --dsn "liucaijun:Sglcj@3374295@tcp(rm-bp15r7sd5q9q47sqy4o.mysql.rds.aliyuncs.com:3306)/datacenter" -outPath ${modeldir} -onlyModel -modelPkgName "model" -tables ${tables} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..d1e641e --- /dev/null +++ b/go.mod @@ -0,0 +1,18 @@ +module ex_pic_to_video + +go 1.23.7 + +require ( + github.com/manifoldco/promptui v0.9.0 + golang.org/x/text v0.25.0 + gorm.io/driver/mysql v1.5.7 + gorm.io/gorm v1.26.1 +) + +require ( + github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect + github.com/go-sql-driver/mysql v1.7.0 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + golang.org/x/sys v0.30.0 // indirect +) diff --git a/import_csv_data b/import_csv_data new file mode 100644 index 0000000..ac6f3ce --- /dev/null +++ b/import_csv_data @@ -0,0 +1 @@ +s1QM2t7N1NQtPksWJpIjU5swReLzUGryqueba8bh/Yj0YQB0zMpxxT/kwjE09zq2ZBcC4TvUr4SHL6WbcT3OT991BATzhZj5s1LJ9oo1ZWFObSZUHhGYiUJ+y92ynZYGMP2kHMrrfNgHkGVpjvcUo8mIbDIJAW6m9XUSYBwdYUZoecXe35g8F71l11ipDdfMDWmOH3DZzy9tzBM= \ No newline at end of file diff --git a/main.go b/main.go new file mode 100644 index 0000000..5a84566 --- /dev/null +++ b/main.go @@ -0,0 +1,149 @@ +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "gorm.io/gorm" + "os" + "strings" +) + +var ( + reader = bufio.NewReader(os.Stdin) +) + +var statusMap = map[string]string{ + "queued": "排队中", + "running": "任务运行中", + "cancelled": "取消任务", + "succeeded": "任务成功", + "failed": "任务失败", +} + +type Conf struct { + Dns string `json:"dns"` + Table string `json:"table"` +} + +type Video struct { + Url string `json:"url"` + Text string `json:"text"` +} + +var ( + c *Conf + db *gorm.DB +) + +func main() { + log("正在读取配置。。。") + dir, err := os.Getwd() + if err != nil { + panic("获取当前路径失败") + } + c = getConf(dir) + success("读取配置成功！请务必保证网络通畅") + sqlc, err := setDb() + if err != nil { + fatal("数据库连接失败1:%v", err) + } + defer sqlc.Close() + + do(dir) + +} +func do(dir string) { + csvData, title := getCsvInfo(dir) + set := setOp(title) + cutChannel := cutData(csvData, 100) + saveData(cutChannel, set, title, dir) + finish(dir) + select {} +} + +func getConf(dir string) *Conf { + + fp := fmt.Sprintf("%s/%s", dir, "import_csv_data") + file, err := os.Open(fp) + if err != nil { + if !os.IsNotExist(err) { + os.Remove(fp) + } + return cConf(fp) + } + defer file.Close() + content, err := os.ReadFile(fp) + if err != nil { + fatal("获取配置信息失败") + exit() + } + + var ( + pwd string + conf = new(Conf) + ) + for { + input("请输入访问密码: ") + pwd, _ = reader.ReadString('\n') + cjson, err := decrypt([]byte(padToLength(strings.TrimSpace(pwd), 16, ' ')), string(content)) + if err != nil { + warning("密码错误") + continue + } + err = json.Unmarshal([]byte(cjson), &conf) + if err != nil { + warning("配置读取失败") + } + break + } + return conf +} + +func cConf(fp string) *Conf { + conf := new(Conf) + log("进入初始化配置向导") + log("----------------------") + + input("请设置DNS: ") + k, _ := reader.ReadString('\n') + conf.Dns = strings.TrimSpace(k) + + input("请设置数据表: ") + t, _ := reader.ReadString('\n') + conf.Table = strings.TrimSpace(t) + + var ( + pwd string + ) + for { + input("请设置访问密码(密码长度请小于16位): ") + pwd, _ = reader.ReadString('\n') + if len(strings.TrimSpace(pwd)) > 16 { + warning("密码长度请小于16位") + continue + } + break + } + cjson, _ := json.Marshal(conf) + pad := padToLength(strings.TrimSpace(pwd), 16, ' ') + en, e := encrypt([]byte(pad), string(cjson)) + if e != nil { + fatal("设置失败，程序退出:%v", e) + exit() + } + if err := os.WriteFile(fp, []byte(en), 0666); err != nil { + fatal("写入失败，程序退出") + exit() + } + return conf + +} + +func exit() { + os.Exit(1) +} + +func finish(dir string) { + do(dir) +} diff --git a/op.go b/op.go new file mode 100644 index 0000000..aa9549f --- /dev/null +++ b/op.go @@ -0,0 +1,77 @@ +package main + +import ( + "github.com/manifoldco/promptui" +) + +type opType int32 + +const ( + Add opType = 1 + OverWrite opType = 2 +) + +var opIndex = map[string]opType{ + "新增": Add, + "覆盖(有则改，无则增)": OverWrite, +} + +type Set struct { + Op opType + OverWriteJudColumns []string +} + +func setOp(title []string) *Set { + var set = new(Set) + for { + prompt := promptui.Select{ + Label: "请选择需要执行的操作: ", + Items: []string{"新增"}, //, "覆盖(有则改，无则增)" + } + _, r, err := prompt.Run() + if err != nil { + warning("Prompt failed %v\n", err) + continue + } + set.Op = opIndex[r] + break + } + if set.Op == OverWrite { + selector := append(title, "继续") + selector[2] = "继续" + for { + r := columnSelect(selector) + if r != "继续" { + set.OverWriteJudColumns = append(set.OverWriteJudColumns, r) + selector = removeElement(selector, r) + continue + } + break + } + } + return set +} + +func columnSelect(selector []string) string { + prompt := promptui.Select{ + Label: "请选择判重字段（可以选择多个，`继续`选项表示选择完了）", + Items: selector, + Size: len(selector) + 1, + } + _, r, err := prompt.Run() + if err != nil { + fatal("Prompt failed %v\n", err) + + } + return r +} + +func removeElement(slice []string, element string) []string { + result := slice[:0] // 创建一个空切片，与原切片共享底层数组 + for _, v := range slice { + if v != element { + result = append(result, v) + } + } + return result +} diff --git a/test.go b/test.go new file mode 100644 index 0000000..e1711b6 --- /dev/null +++ b/test.go @@ -0,0 +1,8 @@ +package main + +func test() { + //key = "236ba4b6-9daa-4755-b22f-2fd274cd223a" + //modelEp = "doubao-seedance-1-0-lite-i2v-250428" + //desc = "女孩抱着狐狸，女孩睁开眼，温柔地看向镜头，狐狸友善地抱着，镜头缓缓拉出，女孩的头发被风吹动 --resolution 480p --dur 5 --camerafixed false --watermark false" + //img = "https://ark-project.tos-cn-beijing.volces.com/doc_image/i2v_foxrgirl.png" +}