go 文件夹之间差量(最新精减版)_go 文件一致性比较-CSDN博客

本文链接：https://blog.csdn.net/wangzhufei/article/details/79881870

/********************************************************
* 比对文件目录版本
* 增加：1)是否压缩，2)差量文件生json/csv;3)增加差量文件目录放于资源目录下;4)删除没用的svn对比
* Date: 2018-04-10
*******************************************************/

package main

import (
"encoding/json"
"path/filepath"
"compress/zlib"
"encoding/binary"
"crypto/md5"
"io/ioutil"
"strings"
"strconv"
"runtime"
"bytes"
"path"
"time"
"sync"
"fmt"
"io"
"os"
)

type Config struct {
OnDataPath string // 源目录
GoDataPath string // 目标目录
GoServerVersionPath string // 版本文件放到服务端目录
ClientInitialCsvPath string // 客户预加载文件传位置
CsvFilesPath string // 版本csv文件位置
ClientSwfPath string // 运行swf文件目录
IsZip bool // 是否压缩
ClinetOutFileType string // 客户端导出文件类型
IsCommonFefresh bool // true:单进程全部更新，false:修改多进程处理

}

type Item struct {
MD5 string
Version string
Path string
}

var zibSuffix = ".zf" // 压缩后缀
var version_name="version.txt" // 版本文件名字
var versionPath = "" // 版本路径
var initial_name = "initial_csv.csv" // 客户预加载文件名字
var initialPath = "" // 客户预加载文件传位置
var goCsvFilesPath = "filelist/" // 版本csv文件位置

var config Config
var items map[string] *Item
var currDateVersion string // 当前日期版本
var currDateTime string // 当前日期
var minCurrDateVersion string // 最小日期版本
var isCommonFefresh = true // true:单进程全部更新，false:修改多进程处理
var isZip = false // 是否压缩
var clinetOutFileType = "csv" // 客户端导出文件类型

var waitgroup sync.WaitGroup // 记录进程处理结束

func main() {
var initTime = time.Now().Format("2006-01-02 15:04:05")
var initTimestamp = time.Now().Unix()

items = make(map[string] *Item)
currDateTime = time.Now().Format("20060102")
var FlaConfigPath = "./filebranchConfig.json"
if !checkFileIsExist(FlaConfigPath) {
fmt.Println("当前目录缺少文件: filebranchConfig.json")
return
}
if !checkFileIsExist("./nil.csv") {
fmt.Println("缺少文件nil.csv")
return
}
readConfig(FlaConfigPath)

isZip = config.IsZip
clinetOutFileType = config.ClinetOutFileType
initialPath = config.ClientInitialCsvPath
isCommonFefresh = config.IsCommonFefresh
var endTime = time.Now().Format("2006-01-02 15:04:05")
var TimeNum = time.Now().Unix() - initTimestamp
fmt.Println("readConfig: ", TimeNum , "秒\r\n范围:\t", initTime, " >> ",endTime)

readCsv(config.CsvFilesPath)
endTime = time.Now().Format("2006-01-02 15:04:05")
TimeNum = time.Now().Unix() - initTimestamp
fmt.Println("readCsv: ", TimeNum , "秒\r\n范围:\t", initTime, " >> ",endTime)
readFiles(config.ClientSwfPath, config.OnDataPath, config.GoDataPath + currDateVersion + "/")
endTime = time.Now().Format("2006-01-02 15:04:05")
TimeNum = time.Now().Unix() - initTimestamp
fmt.Println("readFiles: ", TimeNum , "秒\r\n范围:\t", initTime, " >> ",endTime)
writeNewCsv(config.CsvFilesPath, config.GoDataPath + goCsvFilesPath,config.GoServerVersionPath)
endTime = time.Now().Format("2006-01-02 15:04:05")
TimeNum = time.Now().Unix() - initTimestamp
fmt.Println("运行时间时间: ", TimeNum , "秒\r\n范围:\t", initTime, " >> ",endTime)
}

// 读取上次该目录的文件信息
func readCsv(csv_files_fath string) {

currDateVersion = currDateTime + "01"
minCurrDateVersion = currDateVersion
var currDir = string_remove(csv_files_fath)
versionPath = currDir + "/" + version_name

initialCsvPath := initialPath + initial_name
if checkFileIsExist(initialCsvPath) {
var initialCsvBuffer bytes.Buffer
initialCsvBytes, _ := ioutil.ReadFile(initialCsvPath)
initialCsvLines := strings.Split(string(initialCsvBytes), "\r\n")
var removeNum = 2
// var totalInitialLen := len(initialCsvLines) - removeNum
var totalInitialLen = 0
// initialCsvBuffer.WriteString(strconv.Itoa(totalInitialLen))
for i_num1, initialValue1 := range initialCsvLines {
if removeNum <= i_num1 {
if "" != initialValue1 {
totalInitialLen ++
}
}
}

bytesBuffer := bytes.NewBuffer([]byte{})
binary.Write(bytesBuffer, binary.BigEndian, uint32(totalInitialLen))
initialCsvBuffer.Write(bytesBuffer.Bytes())
for i_num, initialValue := range initialCsvLines {
if removeNum <= i_num {
if "" != initialValue {
initialLine := strings.Split(initialValue, ",")
initialBytes, _ := ioutil.ReadFile(initialPath + initialLine[1])

bytesBuffer = bytes.NewBuffer([]byte{})
binary.Write(bytesBuffer, binary.BigEndian, uint32(len(initialLine[1])))
initialCsvBuffer.Write(bytesBuffer.Bytes())
// buf := bytes.NewBufferString(initialLine[1])
// initialCsvBuffer.Write(buf.Bytes())
initialCsvBuffer.WriteString(initialLine[1])
bytesBuffer = bytes.NewBuffer([]byte{})
binary.Write(bytesBuffer, binary.BigEndian, uint32(len(initialBytes)))
initialCsvBuffer.Write(bytesBuffer.Bytes())
initialCsvBuffer.Write(initialBytes)
}
}
}
ioutil.WriteFile(initialPath + "gamedata.csv", []byte(initialCsvBuffer.String()), 0666)
}

if !checkFileIsExist(csv_files_fath) {
isCommonFefresh=true
return
}

if checkFileIsExist(versionPath) {
versionBytes, _ := ioutil.ReadFile(versionPath)
versionLines := strings.Split(string(versionBytes), "\r\n")
var currDate1 = ""
for _, versionValue := range versionLines {
if "" != versionValue {
line := strings.Split(versionValue, "\t")
currDate1 = line[0]
}
}
if currDate1 != "" {
currDate2 := strings.Split(currDate1, ":")
currDate3 :=currDate2[1]
if currDate3[:8] == currDateTime {
currDateVersion = currDateTime + change_num(currDate3[8:])
}
}
}

csv_bytes, _ := ioutil.ReadFile(csv_files_fath)
lines := strings.Split(string(csv_bytes), "\r\n")
for _, value := range lines {
if "" != value {
line := strings.Split(value, ",")
if minCurrDateVersion > line[1] {
minCurrDateVersion = line[1]
}
items[line[2]] = &Item{line[0], line[1], line[2]}
}
}
fmt.Println("minCurrDateVersion 最小日期版本 ",minCurrDateVersion)

}

// 写出新的目录的文件信息
func writeNewCsv(filesListCsvPath string, goCsvFilesPath string, goServerVersionPath string) {
// var versionFile *os.File
var currDir = string_remove(goCsvFilesPath)
var clientCsvPath = currDir + "/" + currDateVersion + "." + clinetOutFileType
var filesListBuff bytes.Buffer
filesListBuff.WriteString("文件的MD5值,版本,路径\r\n")
var clientCsvBuffer bytes.Buffer
if clinetOutFileType == "json" {
clientCsvBuffer.WriteString("{\r\n")
} else if clinetOutFileType == "csv" {
clientCsvBuffer.WriteString("//路径,版本\r\nFilePath,Version\r\n")
}
var clientJsonCount = 0 // 记录条数
for _, value := range items {
if minCurrDateVersion < value.Version{
if clinetOutFileType == "json" {
if clientJsonCount > 0 {
clientCsvBuffer.WriteString(",\r\n\t\"")
} else{
clientCsvBuffer.WriteString("\t\"")
}
clientCsvBuffer.WriteString(value.Path)
clientCsvBuffer.WriteString("\":")
clientCsvBuffer.WriteString(value.Version)
clientJsonCount++

} else if clinetOutFileType == "csv" {
clientCsvBuffer.WriteString(value.Path)
clientCsvBuffer.WriteString(",")
clientCsvBuffer.WriteString(value.Version)
clientCsvBuffer.WriteString("\r\n")
}
}

filesListBuff.WriteString(value.MD5)
filesListBuff.WriteString(",")
filesListBuff.WriteString(value.Version)
filesListBuff.WriteString(",")
filesListBuff.WriteString(value.Path)
filesListBuff.WriteString("\r\n")
}
if !checkFileIsExist(currDir) {
if err := os.MkdirAll(currDir, 0777); err != nil{
println(err)
fmt.Println("版本目录不可创建")
}
}
if clinetOutFileType == "json" {
clientCsvBuffer.WriteString("\r\n}")
}
ioutil.WriteFile(clientCsvPath, []byte(clientCsvBuffer.String()), 0666)
ioutil.WriteFile(filesListCsvPath, []byte(filesListBuff.String()), 0666)
if isZip == true {
compressorFile(clientCsvPath, clientCsvPath)
}
ioutil.WriteFile(versionPath, []byte("当前最新版本:" + currDateVersion + "\t操作时间:" + time.Now().Format("2006-01-02 15:04:05")+"\t初始版本:"+ minCurrDateVersion + "\r\n"), 0666)
ioutil.WriteFile(goServerVersionPath + version_name, []byte(currDateVersion + ":"+minCurrDateVersion), 0666)
}

// 比对该目录下的文件是否相同
func readFiles(SwfPath string, onDir string, dest_dir string) {
if err1 := os.MkdirAll(dest_dir, 0666); err1 != nil{
println(err1)
fmt.Println("文件不存在")
}
if isCommonFefresh == true {
fmt.Println("》》》混合进程普通更新")
commonRendFile(onDir, dest_dir)
} else {
fmt.Println( "》》》多进程普通更新 ")
commonRendFile2(onDir, dest_dir)
}

swfPathSuffix := path.Ext(SwfPath) //获取运行文件后缀
CopyFile(SwfPath, dest_dir + currDateVersion + swfPathSuffix)
}

// 正常方式
func commonRendFile(path string, dest_dir string) {
var len1= len(path)
runtime.GOMAXPROCS(runtime.NumCPU())
var calcTime = time.Now().Unix()
var toTime = time.Now().Unix()
var toNum = 0
var topNum = 0
var minNum = 9999999
var totleNum = 0
var calcNumber = 1
var calc_count = 0 // 计算每次处理的数数

err := filepath.Walk(path, func(path string, file os.FileInfo, err error) error {
if ( file == nil ) {return err}
if file.IsDir() {return nil}
if strings.Index(path, ".svn") != -1 { return nil} // 过滤svn文件
toTime = time.Now().Unix()
if (toTime - calcTime) > 10 {
if topNum < toNum {
topNum = toNum
}
if toNum < minNum && 0 < toNum {
minNum = toNum
}
totleNum += toNum
fmt.Println(calcNumber ,"个 10s 处理了:", toNum, " 》》条数据, 处理了数据:", totleNum)
calcNumber ++
calcTime = toTime
toNum = 0
}
toNum ++
// copy到别一个目录中
var onPath = strings.Replace(path, "\\", "/", -1) // println(nextPath1)
var nextPath = string([] byte(onPath) [len1:])
item, ok := items[nextPath]
md5 := readMD5(path)
if ok {
if md5 != item.MD5 {
calc_count ++
if calc_count % 5 == 1{
loop_md5(onPath, dest_dir + nextPath)
}else {
waitgroup.Add(1) //每创建一个goroutine，就把任务队列中任务的数量+1
go loop_md5_2(onPath, dest_dir + nextPath, true)
}
item.MD5 = md5
item.Version = currDateVersion
}
}else{
calc_count ++
if calc_count % 5 == 1{
loop_md5(onPath, dest_dir + nextPath)
}else {
waitgroup.Add(1) //每创建一个goroutine，就把任务队列中任务的数量+1
go loop_md5_2(onPath, dest_dir + nextPath, true)
}
items[nextPath] = &Item{md5, currDateVersion, nextPath}
}
return nil
})

if err != nil {
fmt.Printf("error: %v\n", err)
}
fmt.Println("混合进程 10s处理最高", topNum, " 》》最低", minNum, " 》》平均", totleNum/calcNumber, " 》》操作文件数", calc_count)
fmt.Println("混合进程方式等待结束 .......")
waitgroup.Wait() //Wait()这里会发生阻塞，直到队列中所有的任务结束就会解除阻塞
fmt.Println("混合进程方式结束")
}

// 多进程方式
func commonRendFile2(path string, dest_dir string) {
var len1= len(path)
runtime.GOMAXPROCS(runtime.NumCPU())

var calcTime = time.Now().Unix()
var toTime = time.Now().Unix()
var toNum = 0
var topNum = 0
var minNum = 9999999
var totleNum = 0
var calcNumber = 1
err := filepath.Walk(path, func(path string, file os.FileInfo, err error) error {
if ( file == nil ) {return err}
if file.IsDir() {return nil}
if strings.Index(path, ".svn") != -1 { return nil} // 过滤svn文件
toTime = time.Now().Unix()
if (toTime - calcTime) > 10 {
if topNum < toNum {
topNum = toNum
}
if toNum < minNum && 0 < toNum {
minNum = toNum
}
totleNum += toNum
fmt.Println(calcNumber ,"个 10s 处理了:", toNum, " 》》条数据, 处理了数据:", totleNum)
calcNumber ++
calcTime = toTime
toNum = 0
}
toNum ++
// copy到别一个目录中
var onPath = strings.Replace(path, "\\", "/", -1) // println(nextPath1)
var nextPath = string([] byte(onPath) [len1:])
// num := 0
// fmt.Println(" >>nextPath:", nextPath)
item, ok := items[nextPath]
md5 := readMD5(path)
if ok {
if md5 != item.MD5 {
waitgroup.Add(1) //每创建一个goroutine，就把任务队列中任务的数量+1
go loop_md5_2(onPath, dest_dir + nextPath, true)
item.MD5 = md5
item.Version = currDateVersion
}
}else {
waitgroup.Add(1) //每创建一个goroutine，就把任务队列中任务的数量+1
go loop_md5_2(onPath, dest_dir + nextPath, true)
items[nextPath] = &Item{md5, currDateVersion, nextPath}
}
return nil
})
if err != nil {
fmt.Printf("error: %v\n", err)
}
fmt.Println("多进程 10s处理最高", topNum, " 》》最低", minNum, " 》》平均", totleNum/calcNumber)
fmt.Println("多进程方式等待结束 .......")
waitgroup.Wait() //Wait()这里会发生阻塞，直到队列中所有的任务结束就会解除阻塞
fmt.Println("多进程方式结束")
}

func loop_md5(onPath string, toPath string){
loop_md5_2(onPath, toPath, false)

}
func loop_md5_2(onPath string, toPath string, IsDouble bool)(err error) {
var pathDir = string_remove(toPath)
if !checkFileIsExist(pathDir) {
if err := os.MkdirAll(pathDir, 0777); err != nil{
println(err)
fmt.Println("目录不可创建")
}
}
if err = CopyFile(onPath, toPath); err != nil{
println(err)
}
if isZip == true {
fileSuffix := path.Ext(toPath) //获取文件后缀
if fileSuffix == ".csv" ||
fileSuffix == ".xml" ||
fileSuffix == ".ini" ||
fileSuffix == ".json"||
fileSuffix == ".txt" {
compressorFile(onPath, toPath)
}
}
if IsDouble {
waitgroup.Done() //任务完成，将任务队列中的任务数量-1，其实.Done就是.Add(-1)
}
return nil
}

func change_num(num_str string)(string) {
num, err := strconv.Atoi(num_str)
if err != nil {
num = 0
}
num += 1

if 0 < num && num < 10{
return "0" + strconv.Itoa(num)
}else{
return strconv.Itoa(num)
}
}

func change_uint64_str(len int)(string) {
return strconv.FormatUint(uint64(len), 10)
}

// uint32 转成bytes
func uint32_to_bytes(len uint32)([] byte) {
bytesBuffer := bytes.NewBuffer([]byte{})
binary.Write(bytesBuffer, binary.BigEndian, len)
return bytesBuffer.Bytes()
}

// 读取该文件的MD5
func readMD5(path string) string {
file, inerr := os.Open(path)
if inerr != nil {
return ""
}
defer file.Close()
md5hash := md5.New()
io.Copy(md5hash, file)
return fmt.Sprintf("%x", md5hash.Sum(nil))
}

// 读取配置
func readConfig(path string) {
bytes, _ := ioutil.ReadFile(path)
err := json.Unmarshal(bytes, &config)
if err != nil {
fmt.Println("error in translating,", err.Error())
return
}
}

// 读取文件成二进制
func ReadFileByte(filePth string) ([]byte, error) {
f, err := os.Open(filePth)
if err != nil {
return nil, err
}
return ioutil.ReadAll(f)
}

// 复制文件
func CopyFile(source string, dest string) (err error) {
sourcefile, err := os.Open(source)
if err != nil {
return err
}
defer sourcefile.Close()
destfile, err := os.Create(dest)
defer destfile.Close()
_, err = io.Copy(destfile, sourcefile)
if err == nil {
sourceinfo, err := os.Stat(source)
if err != nil {
err = os.Chmod(dest, sourceinfo.Mode())
}
}
return
}

// 复制目录
func CopyDir(source string, dest string) (err error) {
// get properties of source dir
sourceinfo, err := os.Stat(source)
if err != nil {
return err
}
// create dest dir
err = os.MkdirAll(dest, sourceinfo.Mode())
if err != nil {
return err
}
directory, _ := os.Open(source)
objects, err := directory.Readdir(-1)
for _, obj := range objects {
sourcefilepointer := source + "/" + obj.Name()
destinationfilepointer := dest + "/" + obj.Name()
if obj.IsDir() {
// create sub-directories - recursively
err = CopyDir(sourcefilepointer, destinationfilepointer)
if err != nil {
fmt.Println(err)
}
} else {
// perform copy
err = CopyFile(sourcefilepointer, destinationfilepointer)
if err != nil {
fmt.Println(err)
}
}
}
return
}

// 压缩文件 onPath: 要压缩的文件，，压缩到目标位置文件的位置
func compressorFile(onPath string, toPath string) {
bytes, err := ioutil.ReadFile(onPath)
if err != nil {
fmt.Println("compressorFile err:", err)
}
// var fileSuffix string
fileSuffix := path.Ext(toPath) //获取文件后缀

path2 := strings.TrimSuffix(toPath, fileSuffix)// 获取路径和文件名
// fmt.Printf("compressorFile: %s\n", path2)
err = ioutil.WriteFile(path2 + zibSuffix, compressor(bytes), 0666)
if err != nil {
fmt.Println("compressorFile err:", err)
}
}

// zlib 压缩
func compressor(output []byte) []byte {
var buf bytes.Buffer
compressor := zlib.NewWriter(&buf)
n, err := compressor.Write(output)
if err != nil {
fmt.Println("compressor error n,err:", n, err)
}
compressor.Close()
return buf.Bytes()
}

// zlib 解压缩 ---------------------------------------------------------------
func _uncompressorFileOnTest(path1 string) {
fileSuffix := path.Ext(path1) //获取文件后缀
pathFileSuffix := path.Ext(path1) //获取文件后缀
path2 := strings.TrimSuffix(path1, pathFileSuffix)// 获取路径和文件名
uncompressorFileOn( path2 + zibSuffix, path1, fileSuffix)
}
// zlib 解压缩 onPath:压缩文件路径.z toPath:目标文件路径加后缀
func uncompressorFileOnTest(onPath string, toPath string) {
toPathFileSuffix := path.Ext(toPath) //获取文件后缀
uncompressorFileOn(onPath, toPath, toPathFileSuffix)
}

func uncompressorFileOn(onPath string, toPath string, toFileSuffix string) {
fileSuffix := path.Ext(onPath) //获取文件后缀
if fileSuffix == zibSuffix {
uncompressorFile(onPath, toPath, toFileSuffix)
}
}
// onPath：为压缩文件 xxx.z
func uncompressorFile(onPath string, toPath string, toFileSuffix string) {
onFile := path.Base(onPath) //获取文件名带后缀
toPathDir := string_remove(toPath) + "/"

bytes,err := ioutil.ReadFile(onPath)
if err != nil {
fmt.Println("compressorFile err:", err)
}
newFileName := strings.Replace(onFile, zibSuffix, toFileSuffix, -1)
uncompressor(toPathDir + newFileName, bytes)
}

func uncompressor(path2 string, output []byte) []byte {
b := bytes.NewReader(output)
r, err := zlib.NewReader(b)
if err != nil {
fmt.Println("uncompressor error n,err:", r, err)
}

destfile, err := os.Create(path2)
defer destfile.Close()
io.Copy(destfile, r)
r.Close()
return nil
}

/**
* 判断文件是否存在存在返回 true 不存在返回false
*/
func checkFileIsExist(filename string) (bool) {
var exist = true;
if _, err := os.Stat(filename); os.IsNotExist(err) {
exist = false;
}
return exist;
}

// 判断路径是否是目录
func pathIsDir1(Path string)(bool) {
var exist = true;
fi, err := os.Stat(Path)
if err == nil && !fi.IsDir() {
exist = false
}
return exist
}

// 获得文件夹路径去除路径文件部分
func string_remove(str string)(file string){
return path.Dir(str)
}