golang csv导出多个文件,然后最终合并一个csv文件

导出多个文件，最终合并成一个csv文件

	package export

import (
	"bytes"
	"encoding/csv"
	"fmt"
	"log"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"testing"
	"time"
)

// https://bbs.huaweicloud.com/blogs/286321
var tFmt = "2006-01-02 15:04:05"

// 第二步模拟获取数据，然后批量导出多个文件，然后合并一个文件
func Test_export(t *testing.T) {
	content := readFile()
	fmt.Printf("%s - read file end......%d\n", time.Now().Format(tFmt), len(content))
	exportFile(content)
	time.Sleep(time.Millisecond * 200)
}

// 第一步测试数据
func Test_write(t *testing.T) {
	f, err := os.Create("test.csv")
	if err != nil {
		panic(err)
	}
	defer f.Close()
	f.WriteString("\xEF\xBB\xBF") // 写入UTF-8 BOM
	w := csv.NewWriter(f)
	str := []string{"xxx", "ewewe", "werwe", ""}
	for i := 1; i <= 100*10000; i++ {
		str[3] = strconv.Itoa(i)
		w.Write(str)
	}
	w.Flush()
}

func readFile() [][]string {
	//准备读取文件
	fileName := "test.csv"
	fmt.Printf("%s - read file......\n", time.Now().Format(tFmt))
	fs1, _ := os.Open(fileName)
	r1 := csv.NewReader(fs1)
	content, err := r1.ReadAll()
	if err != nil {
		log.Fatalf("can not readall, err is %+v", err)
	}
	return content
}
func exportFile(content [][]string) {
	fmt.Printf("%s - start export multiple  file......\n", time.Now().Format(tFmt))
	length := len(content)

	// 2. 并发循环导出，一次200,000 (20万数据) 到 1个文件，最终输出n个10万数据的文件
	count := 1
	pos := 0

	fileNameArr := make([]string, 0)
	for pos < length {
		maxPos := pos + 20*10000
		if maxPos > length {
			maxPos = length
		}
		temp := content[pos:maxPos]
		go wiriteTempFile(pos, "temp"+strconv.Itoa(count)+".csv", temp)
		fileNameArr = append(fileNameArr, "temp"+strconv.Itoa(count)+".csv")
		count++
		pos += 20 * 10000
	}

	// 3. 合并n个文件到1个文件

	var outBytes bytes.Buffer
	fileNameStr := strings.Join(fileNameArr, " ")
	cmdStr := fmt.Sprintf("cat %s > testall.csv", fileNameStr)
	cmd := exec.Command("sh", "-c", cmdStr)
	cmd.Stdout = &outBytes
	err := cmd.Run()
	if err != nil {
		log.Println(err)
		return
	}
	resStr := outBytes.String()
	log.Println(resStr, cmdStr)
	fmt.Printf("%s - export file success......\n", time.Now().Format(tFmt))
}

func wiriteTempFile(pos int, file string, data [][]string) {
	fmt.Printf("len === %d , pos=%d \n", len(data), pos)
	f, err := os.Create(file)
	if err != nil {
		panic(err)
	}
	defer f.Close()
	f.WriteString("\xEF\xBB\xBF") // 写入UTF-8 BOM
	w := csv.NewWriter(f)
	w.WriteAll(data)
	w.Flush()
}

// 创建文件夹
func MakeDir(dir string) error {
	if !FileIsExisted(dir) {
		if err := os.MkdirAll(dir, 0777); err != nil { //os.ModePerm
			fmt.Println("MakeDir failed:", err)
			return err
		}
	}
	return nil
}

// 检测文件是否存在
func FileIsExisted(filename string) bool {
	existed := true
	if _, err := os.Stat(filename); os.IsNotExist(err) {
		existed = false
	}
	return existed
}