golang 实现文件分块上传

分块上传主要逻辑如下：

1、计算文件总大小

2、定义每块文件的大小

3、计算出总共有多少块

4、读取每块文件，分批次上传

5、上传失败时，添加重试上传机制

涉及到的代码如下：

package utils

import (
	"NativeAndroidProxyQS865V2/logconfig"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"mime/multipart"
	"net/http"
	"os"
)

var fileId int32 = 0
var fileUrl string

/*
*
分块上传文件
*/
func UploadFile(fileName string, fileDir string, uploadUrl string) (string, bool) {

	logconfig.SugarLogger.Infof("begin UploadFile,fileName=%s,fileDir=%s,uploadUrl=%s", fileName, fileDir, uploadUrl)
	//const fileChunk = 4 << 20 // 4MB

	fileChunk := 10 * 1024 * 1024 //10MB
	uploadCount := 1 //上传次数

	//打开文件句柄操作
	file, err := os.Open(fileDir + fileName)
	if err != nil {
		fmt.Println("error opening file")
		logconfig.SugarLogger.Errorf("PostFile() opening file err:%v", err)
		return "", false
	}
	defer file.Close()

	fi, err := file.Stat()
	if err != nil {
		logconfig.SugarLogger.Errorf("PostFile() file.Stat err:%v", err)
		return "", false
	}

	//文件块数
	numChunks := int(math.Ceil(float64(fi.Size()) / float64(fileChunk)))

	//计算上传文件md5
	getMd5Cmd := fmt.Sprintf("md5sum %s | awk '{print $1}'", fileDir+fileName)
	fileMd5Value, _ := LocalCommand(getMd5Cmd, "PostFile")
	logconfig.SugarLogger.Infof("PostFile() file=%s,md5=%s", fileDir+fileName, fileMd5Value)

	//文件切割
	for i := 0; i < numChunks; i++ {
		uploadCount = i + 1
		bodyBuf := &bytes.Buffer{}
		bodyWriter := multipart.NewWriter(bodyBuf)

		chunkSize := int(math.Min(float64(fileChunk), float64(fi.Size()-int64(i*fileChunk))))

		//计算单次要上传的文件
		buf := make([]byte, chunkSize)
		_, err := file.ReadAt(buf, int64(i*fileChunk))
		if err != nil && err != io.EOF {
			panic(err)
		}

		//关键的一步操作,这里写入每次要上传的文件
		fileWriter, err := bodyWriter.CreateFormFile("file", fileName)
		if err != nil {
			logconfig.SugarLogger.Errorf("PostFile() CreateFormFile--file  err:%v", err)
			return "", false
		}

		_, err = io.Copy(fileWriter, bytes.NewReader(buf))
		if err != nil {
			logconfig.SugarLogger.Errorf("PostFile() CreateFormFile--file Copy err:%v", err)
			return "", false
		}

		currentBlockWriter, err := bodyWriter.CreateFormField("currentBlock")
		if err != nil {
			logconfig.SugarLogger.Errorf("PostFile() CreateFormFile--currentBlock  err:%v", err)
			return "", false
		}
		currentBlockValue, _ := json.Marshal(i + 1)
		io.Copy(currentBlockWriter, bytes.NewReader(currentBlockValue))

		totalBlockWriter, err := bodyWriter.CreateFormField("totalBlock")
		if err != nil {
			logconfig.SugarLogger.Errorf("PostFile() CreateFormFile--totalBlock  err:%v", err)
			return "", false
		}
		totalBlockValue, _ := json.Marshal(numChunks)
		io.Copy(totalBlockWriter, bytes.NewReader(totalBlockValue))

		totalSizeWriter, err := bodyWriter.CreateFormField("totalSize")
		if err != nil {
			logconfig.SugarLogger.Errorf("PostFile() CreateFormFile--totalSize  err:%v", err)
			return "", false
		}
		totalSizeValue, _ := json.Marshal(fi.Size())
		io.Copy(totalSizeWriter, bytes.NewReader(totalSizeValue))

		//上传类型 1：应用 2：文件
		typeWriter, err := bodyWriter.CreateFormField("type")
		if err != nil {
			logconfig.SugarLogger.Errorf("PostFile() CreateFormFile--type  err:%v", err)
			return "", false
		}
		typeValue, _ := json.Marshal(2)
		io.Copy(typeWriter, bytes.NewReader(typeValue))

		if fileId != 0 {
			idWriter, err := bodyWriter.CreateFormField("id")
			if err != nil {
				logconfig.SugarLogger.Errorf("PostFile() CreateFormFile--id  err:%v", err)
				return "", false
			}
			idValue, _ := json.Marshal(fileId)
			io.Copy(idWriter, bytes.NewReader(idValue))
		}

		bodyWriter.WriteField("fileName", fileName)

		bodyWriter.WriteField("md5", fileMd5Value)

		contentType := bodyWriter.FormDataContentType()
		bodyWriter.Close()

		logconfig.SugarLogger.Infof("PostFile() 开始上传第%d段", i+1)

        //上传失败重传机制
		isUploadSuccess, postErr := postFileForm(uploadUrl, contentType, bodyBuf)
		if isUploadSuccess == false || postErr != nil {
			for num := 1; num < 4; num++ {
				logconfig.SugarLogger.Errorf("PostFile() 开始上传第%d段--fail,第%d次重试", i+1, num)
				isUploadSuccess, postErr := postFileForm(uploadUrl, contentType, bodyBuf)
				if isUploadSuccess == true && postErr == nil {
					break
				} else {
					if num == 3 {
						return fileUrl, numChunks == uploadCount
					}
				}
			}
		}
	}

	return fileUrl, numChunks == uploadCount
}

//post上传文件

func postFileForm(uploadUrl string, contentType string, bodyBuf *bytes.Buffer) (bool, error) {
	resp, err := http.Post(uploadUrl, contentType, bodyBuf)
	if err != nil {
		logconfig.SugarLogger.Errorf("postFileForm() Post Err:%v", err)
		return false, err
	}

	defer resp.Body.Close()

	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		logconfig.SugarLogger.Errorf("postFileForm() ReadAll Err:%v", err)
		return false, err
	}

	logconfig.SugarLogger.Infof("resp.Status:%v,,,消息返回体%s", resp.Status, string(respBody))
    //解析返回的消息
	uploadFileCallback := &UploadFileCallback{}
	parseErr := json.Unmarshal(respBody, uploadFileCallback)
	if parseErr != nil {
		logconfig.SugarLogger.Errorf("postFileForm() parseErr:%v", parseErr)
		return false, parseErr
	}

	//0：成功、1：失败
	if uploadFileCallback.Status == 0 {
		fileId = uploadFileCallback.ResponseData.Id
		fileUrl = uploadFileCallback.ResponseData.Url
		logconfig.SugarLogger.Infof("postFileForm() fileId:%d", fileId)
		return true, nil
	} else {
		logconfig.SugarLogger.Errorf("postFileForm() uploadFileCallback.Status callback fail,status!=0")
		return false, nil
	}
}

type UploadFileCallback struct {
	Status       int16        `json:"status"`
	Msg          string       `json:"msg"`
	RetCode      int16        `json:"retCode"`
	ResponseData ResponseData `json:"data"`
}

type ResponseData struct {
	Id  int32  `json:"id"`
	Url string `json:"url"`
}

注：post表单上传涉及多种类型参数时,WriteField（）只能传递string类型值,其他类型值用CreateFormField()方式，使用方法参照上面代码