go 文件差量对比，并用天做区分文件夹(几个月的精减版)

package main


import (
    "encoding/json"
    "path/filepath"
    "compress/zlib"
    "encoding/binary"
    "crypto/md5"
    "io/ioutil"
    "strings"
    "os/exec"
    "strconv"
    "runtime"
    "bytes"
    "bufio"
    "path"
    "time" 
    "sync" 
    "fmt"
    "io"
    "os"
)


type Config struct {
    OnDataPath string
    OnDataSubPath string   // 子级目录
    GoDataPath string
    GoServerVersionPath string // 版本文件放到服务端目录
    ClientInitialCsvPath   string   // 客户预加载文件传位置
    CsvFilesPath string
    ClientSwfPath string     // swf目录
    IsCommonFefresh string  // 是否必定普刷新 


}


type Item struct {
    MD5 string
    Version string
    Path string
}


var zibSuffix = ".zip"    // 压缩后缀
var version_name="version.txt"      // 版本文件名字
var versionPath = ""    // 版本路径
var initial_name = "initial.txt"    // 客户预加载文件名字
var initialPath = ""    // 客户预加载文件传位置


var config Config
var items map[string] *Item
var currDateVersion string    // 当前日期版本
var currDateTime string       // 当前日期
var minCurrDateVersion string // 最小日期版本
var isCommonFefresh = 0     // 0:svn 增量更新， 1:单进程全部更新，2:修改多进程处理 


var waitgroup sync.WaitGroup  // 记录进程处理结束


func main() {
    var initTime = time.Now().Format("2006-01-02 15:04:05")
    var initTimestamp = time.Now().Unix()


    items = make(map[string] *Item)
    currDateTime = time.Now().Format("20060102")
    var FlaConfigPath = "./flaConfig.json"
    if !checkFileIsExist(FlaConfigPath) {
        fmt.Println("当前目录缺少文件: flaConfig.json")
        return 
     }
     if !checkFileIsExist("./nil.xx") {
        fmt.Println("缺少文件nil.xx")
        return 
     } 
    readConfig(FlaConfigPath)
    initialPath = config.ClientInitialCsvPath
    var endTime = time.Now().Format("2006-01-02 15:04:05")
    var TimeNum = time.Now().Unix() - initTimestamp
    fmt.Println("readConfig: ", TimeNum , "秒\r\n范围:\t", initTime, " >> ",endTime)


    readCsv(config.CsvFilesPath)
    endTime = time.Now().Format("2006-01-02 15:04:05")
    TimeNum = time.Now().Unix() - initTimestamp
    fmt.Println("readCsv: ", TimeNum , "秒\r\n范围:\t", initTime, " >> ",endTime)
    if isCommonFefresh == 0{
        if config.IsCommonFefresh == "true" {
            isCommonFefresh=2
        } else if config.IsCommonFefresh == "1" {
            isCommonFefresh=1
        }
    }
    readFiles(config.ClientSwfPath, config.OnDataPath, config.OnDataSubPath, config.GoDataPath + currDateVersion + "/")
    endTime = time.Now().Format("2006-01-02 15:04:05")
    TimeNum = time.Now().Unix() - initTimestamp
    fmt.Println("readFiles: ", TimeNum , "秒\r\n范围:\t", initTime, " >> ",endTime)
    writeNewCsv(config.CsvFilesPath, config.GoServerVersionPath)
    endTime = time.Now().Format("2006-01-02 15:04:05")
    TimeNum = time.Now().Unix() - initTimestamp
    fmt.Println("运行时间时间: ", TimeNum , "秒\r\n范围:\t", initTime, " >> ",endTime)
}




// 读取 上次 该目录的文件信息
func readCsv(xx_files_fath string) {
    currDateVersion = currDateTime + "01"
    minCurrDateVersion = currDateVersion
    var currDir = string_remove(xx_files_fath) 
    versionPath = currDir + "/" + version_name


    initialCsvPath := initialPath + initial_name
    if checkFileIsExist(initialCsvPath) {
        var initialCsvBuffer bytes.Buffer
        initialCsvBytes, _ := ioutil.ReadFile(initialCsvPath)
        initialCsvLines := strings.Split(string(initialCsvBytes), "\r\n")
        var removeNum = 2
        // var totalInitialLen := len(initialCsvLines) - removeNum 
        var totalInitialLen = 0
        // initialCsvBuffer.WriteString(strconv.Itoa(totalInitialLen))
        for i_num1, initialValue1 := range initialCsvLines {
            if removeNum <= i_num1 {
                if "" != initialValue1 {
                    totalInitialLen ++ 
                }
            }
        }


        bytesBuffer := bytes.NewBuffer([]byte{})  
        binary.Write(bytesBuffer, binary.BigEndian, uint32(totalInitialLen))  
        initialCsvBuffer.Write(bytesBuffer.Bytes())
         for i_num, initialValue := range initialCsvLines {
            if removeNum <= i_num {
                if "" != initialValue {
                    initialLine := strings.Split(initialValue, ",")
                    initialBytes, _ := ioutil.ReadFile(initialPath + initialLine[1])


                    bytesBuffer = bytes.NewBuffer([]byte{})  
                    binary.Write(bytesBuffer, binary.BigEndian, uint32(len(initialLine[1]))) 
                    initialCsvBuffer.Write(bytesBuffer.Bytes())
                        // buf := bytes.NewBufferString(initialLine[1])
                        // initialCsvBuffer.Write(buf.Bytes())
                    initialCsvBuffer.WriteString(initialLine[1])
                    bytesBuffer = bytes.NewBuffer([]byte{})  
                    binary.Write(bytesBuffer, binary.BigEndian, uint32(len(initialBytes)))
                    initialCsvBuffer.Write(bytesBuffer.Bytes())
                    initialCsvBuffer.Write(initialBytes)
                }
            }
        }
        ioutil.WriteFile(initialPath + "gamedata.xx", []byte(initialCsvBuffer.String()), 0666)
     }




    if !checkFileIsExist(xx_files_fath) {
        isCommonFefresh=1
        return
    }


    if checkFileIsExist(versionPath) {
        versionBytes, _ := ioutil.ReadFile(versionPath)
        versionLines := strings.Split(string(versionBytes), "\r\n")
        var currDate1 = ""
        for _, versionValue := range versionLines {
          if "" != versionValue {
            line := strings.Split(versionValue, "\t")
            currDate1 = line[0]
            }
        }
        if currDate1 != "" {
            currDate2 := strings.Split(currDate1, ":")
            currDate3 :=currDate2[1]
            if currDate3[:8] == currDateTime {
                currDateVersion = currDateTime + change_num(currDate3[8:])
            }
        }
     }


    xx_bytes, _ := ioutil.ReadFile(xx_files_fath)
    lines := strings.Split(string(xx_bytes), "\r\n")
    for _, value := range lines {
        if "" != value {
            line := strings.Split(value, ",")
            if minCurrDateVersion > line[1] {
                minCurrDateVersion = line[1]
            }
            items[line[2]] = &Item{line[0], line[1], line[2]}
        }
    }
    fmt.Println("minCurrDateVersion  最小日期版本 ",minCurrDateVersion)
}


// 写出新的 目录的文件信息
func writeNewCsv(filesListCsvPath string, goServerVersionPath string) {
   // var versionFile *os.File
    var currDir = string_remove(filesListCsvPath) 
    var clientCsvPath = currDir + "/" + currDateVersion + ".xx" 
    var filesListBuff bytes.Buffer
    filesListBuff.WriteString("文件的MD5值,版本,路径\r\n")
    var clientCsvBuffer bytes.Buffer
    clientCsvBuffer.WriteString("//路径,版本\r\nFilePath,Version\r\n")
    for _, value := range items {
        if minCurrDateVersion < value.Version{        
            clientCsvBuffer.WriteString(value.Path)
            clientCsvBuffer.WriteString(",")
            clientCsvBuffer.WriteString(value.Version)
            clientCsvBuffer.WriteString("\r\n")
        }


        filesListBuff.WriteString(value.MD5)
        filesListBuff.WriteString(",")
        filesListBuff.WriteString(value.Version)
        filesListBuff.WriteString(",")
        filesListBuff.WriteString(value.Path)
        filesListBuff.WriteString("\r\n")
    }
    ioutil.WriteFile(clientCsvPath, []byte(clientCsvBuffer.String()), 0666)
    ioutil.WriteFile(filesListCsvPath, []byte(filesListBuff.String()), 0666)
    compressorFile(clientCsvPath, clientCsvPath)
    ioutil.WriteFile(versionPath, []byte("当前最新版本:" + currDateVersion + "\t操作时间:" + time.Now().Format("2006-01-02 15:04:05")+"\r\n"), 0666)
    ioutil.WriteFile(goServerVersionPath + version_name, []byte(currDateVersion), 0666)
}


// 比对该目录下的文件是否相同
func readFiles(SwfPath string, onDir string, onSubPath, dest_dir string) {
    if err1 := os.MkdirAll(dest_dir, 0666); err1 != nil{
        println(err1)
        fmt.Println("文件不存在")
    }
    if isCommonFefresh == 1 {
        fmt.Println("》》》混合进程普通更新  子级目录设置无效")
        commonRendFile(onDir, dest_dir)
    } else if isCommonFefresh == 2 {
      fmt.Println( "》》》多进程普通更新 ")
      commonRendFile2(onDir, onSubPath, dest_dir)
    } else{
        fmt.Println("》》》svn差量更新")
        svnRendFile(onDir, dest_dir)
    }
    
    CopyFile(SwfPath, dest_dir + currDateVersion + ".swf")
}




// 正常方式
func commonRendFile(path string, dest_dir string) {
    var len1= len(path)
    runtime.GOMAXPROCS(runtime.NumCPU())
    var calcTime = time.Now().Unix()
    var toTime = time.Now().Unix()
    var toNum = 0
    var topNum = 0
    var minNum = 9999999
    var totleNum = 0
    var calcNumber = 1
    var calc_count = 0      // 计算每次处理的数数


    err := filepath.Walk(path, func(path string, file os.FileInfo, err error) error {
        if ( file == nil ) {return err}
        if file.IsDir() {return nil}
        if strings.Index(path, ".svn") != -1 { return nil}          // 过滤svn文件 
        toTime = time.Now().Unix()
        if (toTime - calcTime) > 10 {
            if topNum < toNum {
              topNum = toNum
            }
            if toNum < minNum && 0 < toNum {
              minNum = toNum
            }
            totleNum += toNum
            fmt.Println(calcNumber ,"个 10s 处理了:", toNum, " 》》条数据, 处理了数据:", totleNum)
            calcNumber ++ 
            calcTime = toTime
            toNum = 0
        }
        toNum ++
        // copy到别一个目录中  
        var onPath = strings.Replace(path, "\\", "/", -1)                // println(nextPath1)
        var nextPath = string([] byte(onPath) [len1:]) 
        item, ok := items[nextPath]
        md5 := readMD5(path)
        if ok {
            if md5 != item.MD5 {
                calc_count ++ 
                if calc_count % 5 == 1{
                    loop_md5(onPath, dest_dir + nextPath)
                }else {
                    waitgroup.Add(1) //每创建一个goroutine，就把任务队列中任务的数量+1
                    go loop_md5_2(onPath, dest_dir + nextPath, true)
                }
              item.MD5 = md5
              item.Version = currDateVersion  
            }
        }else{
            calc_count ++ 
            if calc_count % 5 == 1{
                loop_md5(onPath, dest_dir + nextPath)
            }else {
                waitgroup.Add(1) //每创建一个goroutine，就把任务队列中任务的数量+1
                go loop_md5_2(onPath, dest_dir + nextPath, true)
            }
            items[nextPath] = &Item{md5, currDateVersion, nextPath}
        }
        return nil
    })
    
    if err != nil {
        fmt.Printf("error: %v\n", err)
    }
    fmt.Println("混合进程 10s处理 最高", topNum, " 》》最低", minNum, " 》》平均", totleNum/calcNumber, " 》》操作文件数", calc_count)
    fmt.Println("混合进程方式等待结束 .......")
    waitgroup.Wait() //Wait()这里会发生阻塞，直到队列中所有的任务结束就会解除阻塞
    fmt.Println("混合进程方式结束")
}


// 多进程方式
func commonRendFile2(path string, onSubPath string, dest_dir string) {
    var len1= len(path)
    runtime.GOMAXPROCS(runtime.NumCPU())


    var calcTime = time.Now().Unix()
    var toTime = time.Now().Unix()
    var toNum = 0
    var topNum = 0
    var minNum = 9999999
    var totleNum = 0
    var calcNumber = 1
    err := filepath.Walk(path + onSubPath, func(path string, file os.FileInfo, err error) error {
        if ( file == nil ) {return err}
        if file.IsDir() {return nil}
        if strings.Index(path, ".svn") != -1 { return nil}          // 过滤svn文件 
        toTime = time.Now().Unix()
        if (toTime - calcTime) > 10 {
            if topNum < toNum {
              topNum = toNum
            }
            if toNum < minNum && 0 < toNum {
              minNum = toNum
            }
            totleNum += toNum
            fmt.Println(calcNumber ,"个 10s 处理了:", toNum, " 》》条数据, 处理了数据:", totleNum)
            calcNumber ++ 
            calcTime = toTime
            toNum = 0
        }
        toNum ++
        // copy到别一个目录中  
        var onPath = strings.Replace(path, "\\", "/", -1)                // println(nextPath1)
        var nextPath = string([] byte(onPath) [len1:])
        item, ok := items[nextPath]
        md5 := readMD5(path)
        if ok {
            if md5 != item.MD5 {
              waitgroup.Add(1) //每创建一个goroutine，就把任务队列中任务的数量+1
              go loop_md5_2(onPath, dest_dir + nextPath, true)
              item.MD5 = md5
              item.Version = currDateVersion  
            }
        }else {
            waitgroup.Add(1) //每创建一个goroutine，就把任务队列中任务的数量+1
            go loop_md5_2(onPath, dest_dir + nextPath, true)
            items[nextPath] = &Item{md5, currDateVersion, nextPath}
        }
        return nil
    })
    if err != nil {
        fmt.Printf("error: %v\n", err)
    }
    fmt.Println("多进程 10s处理 最高", topNum, " 》》最低", minNum, " 》》平均", totleNum/calcNumber)
    fmt.Println("多进程方式等待结束 .......")
    waitgroup.Wait() //Wait()这里会发生阻塞，直到队列中所有的任务结束就会解除阻塞
    fmt.Println("多进程方式结束")
}


// svn 差量方式
func svnRendFile(Dir string, dest_dir string) {
    cmd := exec.Command("/bin/sh", "-c", `svn st | awk '{if ($1 == "?" || $1 == "A" || $1 == "M") {print $2} }' | xargs -l`)
    cmd.Dir = Dir
    stdout, err := cmd.StdoutPipe()
    if err != nil {
        fmt.Println("StdoutPipe: " + err.Error())
        return
    }
    if err != nil {
        fmt.Println(err)
        return 
    }
    cmd.Start()
    reader := bufio.NewReader(stdout)
    //实时循环读取输出流中的一行内容
    for {
        line, err2 := reader.ReadString('\n')
        if err2 != nil || io.EOF == err2 {
            break
        }
        nextPath := strings.Replace(line, "\n", "", -1)
        if "" != nextPath {
        DirPath := Dir + nextPath
        IsDir := pathIsDir1(DirPath)
        if IsDir { continue}      // 过滤目录
        if strings.Index(nextPath, ".svn") != -1 { continue}          // 过滤svn文件 
         item, ok := items[nextPath]
         md5 := readMD5(DirPath)
            if ok {
                if md5 != item.MD5 {
                    waitgroup.Add(1) //每创建一个goroutine，就把任务队列中任务的数量+1
                    go loop_md5_2(DirPath, dest_dir + nextPath, true)
                    item.MD5 = md5
                    item.Version = currDateVersion
                 // }   
                }
            }else {
                waitgroup.Add(1) //每创建一个goroutine，就把任务队列中任务的数量+1
                go loop_md5_2(DirPath, dest_dir + nextPath, true)
                items[nextPath] = &Item{md5, currDateVersion, nextPath}
            }
        }
    }
    fmt.Println("svn 差量方式等待结束 .......")
    waitgroup.Wait() //Wait()这里会发生阻塞，直到队列中所有的任务结束就会解除阻塞
    fmt.Println("svn 差量方式结束")
}
func loop_md5(onPath string, toPath string){
    loop_md5_2(onPath, toPath, false)
    
}
func loop_md5_2(onPath string, toPath string, IsDouble bool)(err error) {
    var pathDir = string_remove(toPath)
    if !checkFileIsExist(pathDir) {
        if err := os.MkdirAll(pathDir, 0777); err != nil{
            println(err)
            fmt.Println("目录不可创建")
        }
    }
    if err = CopyFile(onPath, toPath);  err != nil{
        println(err)
    }
    fileSuffix := path.Ext(toPath) //获取文件后缀   要压缩的文件格式
    if fileSuffix == ".xx" ||
    fileSuffix == ".xml" ||
    fileSuffix == ".ini" ||
    fileSuffix == ".json"||
    fileSuffix == ".txt" {  
        compressorFile(onPath, toPath)
    }
    if IsDouble {
        waitgroup.Done() //任务完成，将任务队列中的任务数量-1，其实.Done就是.Add(-1) 
    }
    return nil
}


func change_num(num_str string)(string) {
    num, err := strconv.Atoi(num_str)
    if err != nil {
        num = 0
    }
    num += 1
    
    if 0 < num && num < 10{
        return "0" + strconv.Itoa(num)
    }else{
        return strconv.Itoa(num)
    }
}


func change_uint64_str(len int)(string) {
    return strconv.FormatUint(uint64(len), 10)
}


// uint32 转成bytes
func uint32_to_bytes(len uint32)([] byte) {
    bytesBuffer := bytes.NewBuffer([]byte{}) 
    binary.Write(bytesBuffer, binary.BigEndian, len) 
    return bytesBuffer.Bytes()
}


// 读取该文件的MD5
func readMD5(path string) string {
 file, inerr := os.Open(path)
    if inerr != nil {
         return ""
    }
    defer file.Close()
    md5hash := md5.New()
    io.Copy(md5hash, file)
    return fmt.Sprintf("%x", md5hash.Sum(nil))
   
}


// 读取配置
func readConfig(path string) {
    bytes, _ := ioutil.ReadFile(path)
    err := json.Unmarshal(bytes, &config)
    if err != nil {  
        fmt.Println("error in translating,", err.Error())
        return
    }
}


// 读取文件成二进制
func ReadFileByte(filePth string) ([]byte, error) {
    f, err := os.Open(filePth)
        if err != nil {
            return nil, err
    }
    return ioutil.ReadAll(f)
 }




// 复制文件 
func CopyFile(source string, dest string) (err error) {
     sourcefile, err := os.Open(source)
     if err != nil {
         return err
     }
     defer sourcefile.Close()
     destfile, err := os.Create(dest)
     defer destfile.Close()
     _, err = io.Copy(destfile, sourcefile)
     if err == nil {
         sourceinfo, err := os.Stat(source)
         if err != nil {
             err = os.Chmod(dest, sourceinfo.Mode())
         }
     }
     return
 }


// 复制目录
func CopyDir(source string, dest string) (err error) {
     // get properties of source dir
     sourceinfo, err := os.Stat(source)
     if err != nil {
         return err
     }
     // create dest dir
      err = os.MkdirAll(dest, sourceinfo.Mode())
     if err != nil {
         return err
     }
     directory, _ := os.Open(source)
     objects, err := directory.Readdir(-1)
     for _, obj := range objects {
         sourcefilepointer := source + "/" + obj.Name()
         destinationfilepointer := dest + "/" + obj.Name()
         if obj.IsDir() {
             // create sub-directories - recursively
             err = CopyDir(sourcefilepointer, destinationfilepointer)
             if err != nil {
                 fmt.Println(err)
             }
         } else {
             // perform copy
             err = CopyFile(sourcefilepointer, destinationfilepointer)
             if err != nil {
                 fmt.Println(err)
             }
         }
     }
     return
 }


// 压缩文件 onPath: 要压缩的文件 ，，压缩到目标位置文件的位置
func compressorFile(onPath string, toPath string) {
    // fmt.Printf("compressorFile: %s\n", path1)
    bytes, err := ioutil.ReadFile(onPath)
    if err != nil {
        fmt.Println("compressorFile err:", err)
    }
    // var fileSuffix string
    fileSuffix := path.Ext(toPath) //获取文件后缀


    path2 := strings.TrimSuffix(toPath, fileSuffix)// 获取路径和文件名
    // fmt.Printf("compressorFile: %s\n", path2)
    err = ioutil.WriteFile(path2 + zibSuffix, compressor(bytes), 0666)
    if err != nil {
        fmt.Println("compressorFile err:", err)
    }
}


// zlib 压缩
func compressor(output []byte) []byte {
    var buf bytes.Buffer
    compressor := zlib.NewWriter(&buf)
    n, err := compressor.Write(output)
    if err != nil {
        fmt.Println("compressor error n,err:", n, err)
    }
    compressor.Close()
    return buf.Bytes()
}




// zlib 解压缩   ---------------------------------------------------------------
func _uncompressorFileOnTest(path1 string) {
    fileSuffix := path.Ext(path1) //获取文件后缀
    pathFileSuffix := path.Ext(path1) //获取文件后缀
    path2 := strings.TrimSuffix(path1, pathFileSuffix)// 获取路径和文件名
    uncompressorFileOn( path2 + zibSuffix, path1, fileSuffix)
}
// zlib 解压缩   onPath:压缩文件路径.z    toPath:目标文件路径 加后缀
func uncompressorFileOnTest(onPath string, toPath string) {
    toPathFileSuffix := path.Ext(toPath) //获取文件后缀
    uncompressorFileOn(onPath, toPath, toPathFileSuffix)
}


func uncompressorFileOn(onPath string, toPath string, toFileSuffix string) {
    fileSuffix := path.Ext(onPath) //获取文件后缀
    if fileSuffix == zibSuffix {
        uncompressorFile(onPath, toPath, toFileSuffix)
    }
}
//  onPath：为压缩文件 xxx.z
func uncompressorFile(onPath string, toPath string, toFileSuffix string) {
    onFile := path.Base(onPath) //获取文件名带后缀
    toPathDir := string_remove(toPath) + "/"


    bytes,err := ioutil.ReadFile(onPath)
    if err != nil {
        fmt.Println("compressorFile err:", err)
    }
    newFileName := strings.Replace(onFile, zibSuffix, toFileSuffix, -1)
    uncompressor(toPathDir + newFileName, bytes)
}


func uncompressor(path2 string, output []byte) []byte {    
    b := bytes.NewReader(output)
    r, err := zlib.NewReader(b)
    if err != nil {
        fmt.Println("uncompressor error n,err:", r, err)
    }


    destfile, err := os.Create(path2)
    defer destfile.Close()
    io.Copy(destfile, r)
    r.Close()
    return nil
}




/** 
 * 判断文件是否存在  存在返回 true 不存在返回false
 */
func checkFileIsExist(filename string) (bool) {
    var exist = true;
    if _, err := os.Stat(filename); os.IsNotExist(err) {
       exist = false;
    }
    return exist;
}


// 判断路径 是否是目录
func pathIsDir1(Path string)(bool) {
  var exist = true;
  fi, err := os.Stat(Path)
  if err == nil && !fi.IsDir() {
      exist = false
  }
 
  return exist
}


// 获得文件夹路径        去除路径文件部分
func string_remove(str string)(file string){
    return path.Dir(str)
}