AliOss文件迁移到JDOss

因为项目需要把AliOss上的许多文件迁移到JDOss上，本来JDoss上有个工具可以实现迁移的，但是实际效果却不怎么好，很多文件都遗漏了，所以针对这种情况写了一个函数来判断缺少了哪些函数，缺少的从Ali下载到JD上，实现思路：因为是判断两个云上文件的差异，所以先根据prefix遍历Ali文件夹，将文件key作为key，size作为value缓存到内存中，在根据同样的prefix遍历JD云上的文件，同样缓存到map中，将Alimap的key取出一一和JD对应，JD没有的文件就从Ali下载，得到的文件流直接上传相同目录到JDOss，下面是具体代码的实现：

---

//将AliOss的文件拷贝至JDOss
func CopyAliOss2JDoss() {

	//列举文件使用的prefix
	bothverdsion := "2012"

	a := ListObjectinstance{}
	a.aliListmap = map[string]int64{}
	a.JDListmap = map[string]int64{}
	a.shrotmap = map[string]int64{}

	// 创建OSSClient实例。
	client, err := oss.New("xxxxxxxxxxxxxxx", "xxxxxxxxxxxxx", "xxxxxxxxxxxxxxxxx")
	if err != nil {
		fmt.Println("创建oss上传client Error:", err)
		return
	}
	// 获取存储空间。
	bucket, err := client.Bucket("xxxxx")
	if err != nil {
		fmt.Println("创建oss上传bucket Error:", err)
		//os.Exit(-1)
		return
	}
	//分页列举文件
	marker := oss.Marker("")
	prefix := oss.Prefix(bothverdsion)
	//fmt.Println("列举文件:", prefix)
	for {
		lsRes, err := bucket.ListObjects(marker, prefix)
		if err != nil {
			fmt.Println("从oss获取文件失败", err)
			return
		}
		for _, object := range lsRes.Objects {
			//下载文件到working目录
			//fmt.Println("key:", object.Key)
			a.aliListmap[object.Key] = object.Size
		}
		prefix = oss.Prefix(lsRes.Prefix)
		marker = oss.Marker(lsRes.NextMarker)
		if !lsRes.IsTruncated {
			break
		}
	}
	logs.Info(" 获得alimap的size:", len(a.aliListmap))

	JDbucket := "xxxxxx"
	JDPrefix := bothverdsion
	JDMarker := ""
	input := s3.ListObjectsInput{
		Bucket: &JDbucket,
		Prefix: &JDPrefix,
		Marker: &JDMarker,
	}
	Client := NewS3Client()
	for {
		output, err := Client.ListObjects(&input)
		if err != nil {
			logs.Error("获取JD失败:", err.Error())
			return
		}

		for _, value := range output.Contents {
			crtDir := *value.Key
			size := *value.Size
			a.JDListmap[crtDir] = size
		}
		input.Marker = output.NextMarker
		input.Prefix = output.Prefix
		if !*output.IsTruncated {
			break
		}
	}
	logs.Info(" 获得JDmap的size:", len(a.JDListmap))

	for k, v := range a.aliListmap {
		if jd, ok := a.JDListmap[k]; ok {
			if jd != v {
				logs.Info("ali的key:", k, "ali的size:", v, "JD的size:", jd)
			}
		} else {
			logs.Info("JDmap中没有找到:", k)
			a.shrotmap[k] = v
		}
	}

	accessKey := "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
	secretKey := "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"

	creds := credentials.NewStaticCredentials(accessKey, secretKey, "")
	config := &aws.Config{
		Region:      aws.String("cn-south-1"),
		Endpoint:    aws.String("xxxxxxxxxxx"),
		DisableSSL:  aws.Bool(false),
		Credentials: creds,
	}
	newSession, err := session.NewSession(config)

	uploader := s3manager.NewUploader(newSession)
	fmt.Println("---------JDshortmap szie:", len(a.shrotmap))
	for k, _ := range a.shrotmap {
		closer, err := bucket.GetObject(k)
		if err != nil {
			logs.Error("alioss下载文件失败:", err.Error())
			continue
		}
		_, err = uploader.Upload(&s3manager.UploadInput{
			Bucket: aws.String("xxxxxx"),
			Key:    aws.String(k),
			Body:   closer,
		})
		if err != nil {
			logs.Error("JD上传失败:", err.Error())
		} else {
			logs.Info("上传成功 key:", k)
		}
		if closer != nil {
			closer.Close()
		}
	}
}

//创建新的AWS S3客户端
func NewS3Client() *s3.S3 {
	accessKey := "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
	secretKey := "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"

	creds := credentials.NewStaticCredentials(accessKey, secretKey, "")
	config := &aws.Config{
		Region:      aws.String("cn-south-1"),
		Endpoint:    aws.String("xxxxxxxx"),
		DisableSSL:  aws.Bool(false),
		Credentials: creds,
	}
	client := s3.New(session.New(config))
	return client
}