go操作亚马逊S3教程
官网API地址：https://docs.aws.amazon.com/sdk-for-go/api/service/s3/
前置：S3介绍
一、aws-sdk-go
1 初始化S3配置
初始化session和连接，同时打印错误日志
var (
sess *session.Session
svc *s3.S3
)
func init() {
access_key := "xxxxxx"
secret_key := "xxxxxxxxxxx"
end_point := "http://xxx.xxx.xxx.xxx:xxx/" //根据自己endpint修改
sess, _ = session.NewSession(&aws.Config{
Credentials: credentials.NewStaticCredentials(access_key, secret_key, ""),
Endpoint: aws.String(end_point),
Region: aws.String("us-east-1"),
DisableSSL: aws.Bool(true),
S3ForcePathStyle: aws.Bool(false), //virtual-host style方式，不要修改
})
svc = s3.New(sess)
}
func main(){
....
}
func exitErrorf(msg string, args ...interface{}) {
fmt.Fprintf(os.Stderr, msg+"\n", args...)
os.Exit(1)
}
2 具体操作
2.1 获取所有桶信息
//获取所有桶
func ListBuckets() {
result, err := svc.ListBuckets(nil)
if err != nil {
exitErrorf("Unable to list buckets, %v", err)
}
fmt.Println("Buckets:")
for _, b := range result.Buckets {
fmt.Printf("* %s created on %s\n",
aws.StringValue(b.Name), aws.TimeValue(b.CreationDate))
}
for _, b := range result.Buckets {
fmt.Printf("%s\n", aws.StringValue(b.Name))
}
}
2.2 创建桶
func createBucket(bucket string) {
params := &s3.CreateBucketInput{
Bucket: aws.String(bucket),
}
_, err := svc.CreateBucket(params)
if err != nil {
exitErrorf("Unable to create bucket %q, %v", bucket, err)
}
// Wait until bucket is created before finishing
fmt.Printf("Waiting for bucket %q to be created...\n", bucket)
err = svc.WaitUntilBucketExists(&s3.HeadBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
exitErrorf("Error occurred while waiting for bucket to be created, %v", bucket)
}
fmt.Printf("Bucket %q successfully created\n", bucket)
}
2.3 删除桶
func deleteBucket(bucket string) {
params := &s3.DeleteBucketInput{
Bucket: aws.String(bucket),
}
_, err := svc.DeleteBucket(params)
if err != nil {
exitErrorf("Unable to delete bucket %q, %v", bucket, err)
}
//wait until bucket is deleted
err = svc.WaitUntilBucketNotExists(&s3.HeadBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
exitErrorf("Error occurred while waiting for bucket to be deleted, %v", bucket)
}
fmt.Printf("Bucket %q successfully delete\n", bucket)
}
2.4 下载桶里的文件
func downloadFile(bucket, item string) {
file, err := os.Create(item)
if err != nil {
exitErrorf("Unable to open file %q, %v", err)
}
defer file.Close()
downloader := s3manager.NewDownloader(sess)
numBytes, err := downloader.Download(file,
&s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(item),
})
if err != nil {
exitErrorf("Unable to download item %q, %v", item, err)
}
fmt.Println("Downloaded", file.Name(), numBytes, "bytes")
}
2.5 删除桶里的文件
func deleteFile(bucket, obj string) {
_, err := svc.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(obj)})
if err != nil {
exitErrorf("Unable to delete object %q from bucket %q, %v", obj, bucket, err)
}
err = svc.WaitUntilObjectNotExists(&s3.HeadObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(obj),
})
fmt.Printf("Object %q successfully deleted\n", obj)
}
2.6 上传文件到指定桶
func uploadFile(bucket, filename string) {
file, err := os.Open(filename)
if err != nil {
exitErrorf("Unable to open file %q, %v", err)
}
defer file.Close()
uploader := s3manager.NewUploader(sess)
_, err = uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(bucket),
Key: aws.String(filename),
Body: file,
})
if err != nil {
// Print the error and exit.
exitErrorf("Unable to upload %q to %q, %v", filename, bucket, err)
}
fmt.Printf("Successfully uploaded %q to %q\n", filename, bucket)
}
3 全部代码
package main
import (
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
_ "github.com/aws/aws-sdk-go/service/s3/s3manager"
"os"
)
var (
sess *session.Session
svc *s3.S3
)
func init() {
access_key := "xxxxxx"
secret_key := "xxxxxxxxxxx"
end_point := "http://xxx.xxx.xxx.xxx:xxx/" //根据自己endpint修改
sess, _ = session.NewSession(&aws.Config{
Credentials: credentials.NewStaticCredentials(access_key, secret_key, ""),
Endpoint: aws.String(end_point),
Region: aws.String("us-east-1"),
DisableSSL: aws.Bool(true),
S3ForcePathStyle: aws.Bool(false), //virtual-host style方式，不要修改
})
svc = s3.New(sess)
}
func main() {
//注意！！！桶名称后面一定要根`/`
//downloadFile("bucket-ziyi/", "banner.png")
//uploadFile("bucket-ziyi/", "test/test1/test2/test.csv")
//createBuckt("bucket-demo1/")
//deleteBucket("bucket-demo1/")
deleteFile("bucket-ziyi/test/", "log.txt")
}
func deleteFile(bucket, obj string) {
_, err := svc.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(obj)})
if err != nil {
exitErrorf("Unable to delete object %q from bucket %q, %v", obj, bucket, err)
}
err = svc.WaitUntilObjectNotExists(&s3.HeadObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(obj),
})
fmt.Printf("Object %q successfully deleted\n", obj)
}
func deleteBucket(bucket string) {
params := &s3.DeleteBucketInput{
Bucket: aws.String(bucket),
}
_, err := svc.DeleteBucket(params)
if err != nil {
exitErrorf("Unable to delete bucket %q, %v", bucket, err)
}
//wait until bucket is deleted
err = svc.WaitUntilBucketNotExists(&s3.HeadBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
exitErrorf("Error occurred while waiting for bucket to be deleted, %v", bucket)
}
fmt.Printf("Bucket %q successfully delete\n", bucket)
}
func createBucket(bucket string) {
params := &s3.CreateBucketInput{
Bucket: aws.String(bucket),
}
_, err := svc.CreateBucket(params)
if err != nil {
exitErrorf("Unable to create bucket %q, %v", bucket, err)
}
// Wait until bucket is created before finishing
fmt.Printf("Waiting for bucket %q to be created...\n", bucket)
err = svc.WaitUntilBucketExists(&s3.HeadBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
exitErrorf("Error occurred while waiting for bucket to be created, %v", bucket)
}
fmt.Printf("Bucket %q successfully created\n", bucket)
}
func uploadFile(bucket, filename string) {
file, err := os.Open(filename)
if err != nil {
exitErrorf("Unable to open file %q, %v", err)
}
defer file.Close()
uploader := s3manager.NewUploader(sess)
_, err = uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(bucket),
Key: aws.String(filename),
Body: file,
})
if err != nil {
// Print the error and exit.
exitErrorf("Unable to upload %q to %q, %v", filename, bucket, err)
}
fmt.Printf("Successfully uploaded %q to %q\n", filename, bucket)
}
func downloadFile(bucket, item string) {
file, err := os.Create(item)
if err != nil {
exitErrorf("Unable to open file %q, %v", err)
}
defer file.Close()
downloader := s3manager.NewDownloader(sess)
numBytes, err := downloader.Download(file,
&s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(item),
})
if err != nil {
exitErrorf("Unable to download item %q, %v", item, err)
}
fmt.Println("Downloaded", file.Name(), numBytes, "bytes")
}
//获取所有桶
func ListBuckets() {
result, err := svc.ListBuckets(nil)
if err != nil {
exitErrorf("Unable to list buckets, %v", err)
}
fmt.Println("Buckets:")
for _, b := range result.Buckets {
fmt.Printf("* %s created on %s\n",
aws.StringValue(b.Name), aws.TimeValue(b.CreationDate))
}
for _, b := range result.Buckets {
fmt.Printf("%s\n", aws.StringValue(b.Name))
}
}
func exitErrorf(msg string, args ...interface{}) {
fmt.Fprintf(os.Stderr, msg+"\n", args...)
os.Exit(1)
}
二、aws-sdk-go-v2
package main
import (
"archive/zip"
"bufio"
"context"
"fmt"
"github.com/aobco/log"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"io"
"os"
"path/filepath"
"strings"
)
var (
ak = "xxxxxxxxxxxxxxx"
sk = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
endpoint = "http://xxx.xxx.xxx:xx/" //endpoint设置，不要动
client *s3.Client
)
func init() {
// 配置AWS SDK
var credentialsProvider aws.CredentialsProvider
credentialsProvider = credentials.StaticCredentialsProvider{Value: aws.Credentials{AccessKeyID: ak, SecretAccessKey: sk, SessionToken: ""}}
cfg := aws.Config{
Credentials: credentialsProvider,
EndpointResolverWithOptions: aws.EndpointResolverWithOptionsFunc(
func(service string, region string, options ...interface{}) (aws.Endpoint, error) {
return aws.Endpoint{
URL: endpoint,
HostnameImmutable: true,
}, nil
}),
}
// 创建S3客户端
client = s3.NewFromConfig(cfg)
}
func main() {
//step1 : download target file
key1 := "log1.txt"
key2 := "log/log2.txt"
bucket1 := "bucket-ziyi"
bucket2 := "bucket-ziyi"
tmpDir := "test/test1/test2/tmp"
DownloadFile(bucket1, key1, fmt.Sprintf("%v/%v", tmpDir, key1))
DownloadFile(bucket2, key2, fmt.Sprintf("%v/%v", tmpDir, key2))
//step2 : compression
zipFileName, err := compressionDir(tmpDir)
if err != nil {
log.Fatal("compression file err=", err)
return
}
//fmt.Println(zipFileName)
//step3 : upload zip
key3 := "demo.zip"
bucket3 := "bucket-ziyi"
UploadFile(bucket3, key3, zipFileName)
}
// UploadFile 上传文件到S3
func UploadFile(bucket string, key string, filePath string) {
// 打开本地文件
file, err := os.Open(filePath)
if err != nil {
log.Info("upload file err=", err)
}
defer file.Close()
// 上传文件到S3 bucket
_, err = client.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: &bucket,
Key: &key,
Body: file,
})
if err != nil {
log.Error("upload file err=", err)
return
}
log.Infof("upload file success, bucket=%v, name=%v", bucket, key)
}
// DownloadFile 下载S3文件到本地
func DownloadFile(bucket string, key string, filePath string) {
// 下载文件内容
resp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
Bucket: &bucket,
Key: &key,
})
if err != nil {
log.Error("download file err=", err)
return
}
defer resp.Body.Close()
index := strings.LastIndex(filePath, "/")
dir := filePath[:index]
err = os.MkdirAll(dir, os.ModePerm)
if err != nil {
log.Error("download file err=", err)
}
// 写入文件
file, err := os.Create(filePath)
if err != nil {
log.Error("download file err=", err)
return
}
defer file.Close()
buffer := make([]byte, 1024*2)
for {
reader := bufio.NewReader(resp.Body)
n, err := reader.Read(buffer)
if err == io.EOF {
break
}
if err != nil {
log.Fatal("download file err=", err)
return
}
file.Write(buffer[:n])
}
log.Info("download file success, filePath=", filePath)
}
func compressionDir(baseDir string) (string, error) {
zipFileName := baseDir + ".zip"
// 创建一个新的 zip 文件
zipFile, err := os.Create(zipFileName)
if err != nil {
return "", err
}
defer zipFile.Close()
// 创建一个 zip.Writer
zipWriter := zip.NewWriter(zipFile)
defer zipWriter.Close()
// 遍历目录下的所有文件和子目录
err = filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
// 创建一个 zip 文件中的文件或目录
relativePath := strings.TrimPrefix(path, baseDir)
zipPath := strings.TrimLeft(filepath.Join("/", relativePath), "/")
// 如果是目录或空目录，则在 zip 文件中创建一个目录
if info.IsDir() || isEmptyDir(path) {
_, err := zipWriter.Create(zipPath + "/")
if err != nil {
return err
}
} else {
// 如果是文件，则创建一个 zip 文件中的文件
zipFile, err := zipWriter.Create(zipPath)
if err != nil {
return err
}
// 打开原始文件
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()
// 将原始文件的内容拷贝到 zip 文件中
_, err = io.Copy(zipFile, file)
if err != nil {
return err
}
}
return nil
})
if err != nil {
return "", err
}
return zipFileName, nil
}
// 判断目录是否为空目录
func isEmptyDir(dirPath string) bool {
dir, err := os.Open(dirPath)
if err != nil {
return false
}
defer dir.Close()
_, err = dir.Readdirnames(1)
return err == io.EOF
}
① 遍历某个桶目录下的所有文件并下载
- 检查ak、sk、endpoint等是否正确
ctx := context.TODO()
// 发送 ListBuckets 请求
_, err := client.ListBuckets(ctx, &s3.ListBucketsInput{})
if err != nil {
fmt.Println("Connection failed:", err)
return
}
fmt.Println("Connection succeeded")
- 可以先检查桶是否存在
// 指定要验证的存储桶名称
bucketName := "bucket-ziyi"
// 创建输入参数
input := &s3.HeadBucketInput{
Bucket: &bucketName,
}
// 发送 HeadBucket 请求
_, err := client.HeadBucket(context.TODO(), input)
if err != nil {
// 如果返回错误，说明存储桶不存在
fmt.Println("Bucket does not exist")
return
}
// 存储桶存在
fmt.Println("Bucket exists")
业务操作：遍历桶并下载：
var(
ak = ""
sk = ""
endpoint = ""
client *s3.Client
)
func init() {
// 配置AWS SDK
var credentialsProvider aws.CredentialsProvider
credentialsProvider = credentials.StaticCredentialsProvider{Value: aws.Credentials{AccessKeyID: ak, SecretAccessKey: sk, SessionToken: ""}}
cfg := aws.Config{
Credentials: credentialsProvider,
EndpointResolverWithOptions: aws.EndpointResolverWithOptionsFunc(
func(service string, region string, options ...interface{}) (aws.Endpoint, error) {
return aws.Endpoint{
URL: endpoint,
HostnameImmutable: true,
}, nil
}),
}
// 创建S3客户端
client = s3.NewFromConfig(cfg)
}
func main() {
// AWS 配置
bucket := "bucket-ziyi"
prefix := "log/"
// 获取对象列表
resp, err := client.ListObjects(context.TODO(), &s3.ListObjectsInput{
Bucket: &bucket,
Prefix: &prefix,
})
if err != nil {
fmt.Println("Failed to list objects:", err)
return
}
// 创建下载根目录
downloadRootDir := "./download"
if err := os.MkdirAll(downloadRootDir, 0700); err != nil {
fmt.Println("Failed to create download directory:", err)
return
}
keys := make([]string, 0)
// 遍历对象列表并下载
for i, obj := range resp.Contents {
if strings.HasSuffix(*obj.Key, "/") {
continue
}
keys = append(keys, *obj.Key)
// 构建下载路径
//downloadPath := filepath.Join(downloadRootDir, *obj.Key)
downloadPath := filepath.Join(downloadRootDir, *obj.Key)
//fmt.Println("downloadPath=", *obj.Key, i)
// 如果对象是目录，则创建对应的目录
if strings.HasSuffix(*obj.Key, "/") {
if err := os.MkdirAll(downloadPath, 0700); err != nil {
fmt.Printf("Failed to create directory '%s': %v\n", downloadPath, err)
continue
}
fmt.Printf("Created directory '%s'\n", downloadPath)
continue
}
// 下载对象
err := downloadObject(context.TODO(), client, bucket, *obj.Key, downloadPath)
if err != nil {
fmt.Printf("Failed to download object '%s': %v\n", *obj.Key, err)
} else {
fmt.Printf("Downloaded object '%s' to '%s'\n", *obj.Key, downloadPath)
}
}
fmt.Println(keys)
}
// 下载对象
func downloadObject(ctx context.Context, client *s3.Client, bucket, key, filePath string) error {
// 创建文件
file, err := os.Create(filePath)
if err != nil {
return err
}
defer file.Close()
// 下载对象
resp, err := client.GetObject(ctx, &s3.GetObjectInput{
Bucket: &bucket,
Key: &key,
})
if err != nil {
return err
}
defer resp.Body.Close()
// 写入文件
_, err = io.Copy(file, resp.Body)
if err != nil {
return err
}
return nil
}
三、aws相关bug
①Versions bucket is not support origin.
对应bucket没有开版本控制
四、业务相关
4.1 将下载下来的文件进行压缩
将文件下载到临时目录，然后通过zip命令或者golang自带的压缩api进行解压处理
- zip -r -compressionLevel xxx.zip *
- 其中compressionLevel范围是0-9，0表示不压缩；-r表示如果有目录，则也对应压缩目录内的内容
//os.TempDir()：获取临时目录
downloadDir := filepath.Join(os.TempDir(), fmt.Sprintf("%d-%d", id1, id2))
注意：golang的exec.Command()无法支持通配符*，因此需要通过filepath.Glob(“*”)的方式来列出所有的文件
zipName := filepath.Base(dest)
command := []string{
"-r",
compressionLevel,
zipName,
}
tmp, err := filepath.Glob("*")
if len(tmp) == 0 {
log.Infof("No matching files found")
}
log.Infof("* tmp=%v", tmp)
command = append(command, tmp...)
if err != nil {
log.Error(err)
}
log.Infof("args=%v", command)
cmd := exec.Command("/usr/bin/zip", command...)
log.Infof("compression cmd=%v", cmd)
err = cmd.Run()
4.2 开启多个协程压缩文件，超出大小时暂停
package main
import (
"context"
"fmt"
"github.com/aobco/log"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
"os"
"strings"
"sync"
"sync/atomic"
)
var (
access_key = "xxxxxx"
secret_key = "xxxx"
end_point = "xxxxx/" //endpoint设置，不要动
client *s3.Client
maxWorkers = 3
//keys = []string{"视频/黑鹰坠落.mp4", "视频/多文件打包接口文档.md", "视频/啊1Aa~!@#$%^&()_+{}.txt"}
keys = []string{"视频/黑鹰坠落.mp4", "视频/多文件打包接口文档.md", "goroutine/docker-wsl-cli.iso"}
fileSizeLimit int64 = 23 * 1024
totalFileSize int64 = 0
)
func init() {
// 配置AWS SDK
var credentialsProvider aws.CredentialsProvider
credentialsProvider = credentials.StaticCredentialsProvider{Value: aws.Credentials{AccessKeyID: access_key, SecretAccessKey: secret_key, SessionToken: ""}}
cfg := aws.Config{
Credentials: credentialsProvider,
EndpointResolverWithOptions: aws.EndpointResolverWithOptionsFunc(
func(service string, region string, options ...interface{}) (aws.Endpoint, error) {
return aws.Endpoint{
URL: end_point,
HostnameImmutable: true,
}, nil
}),
}
// 创建S3客户端
client = s3.NewFromConfig(cfg)
}
func main() {
wg := new(sync.WaitGroup)
bucket := "bucket-ziyi"
path := ""
ctx, cancel := context.WithCancel(context.Background())
for i, key := range keys {
fileName := strings.Split(key, "/")[1]
path = fmt.Sprintf("%s\\%d-%s", "E:\\Go\\GoPro\\src\\go_code\\gouitest\\context\\downloader", i, fileName)
wg.Add(1)
go func(key, path string, cancel context.CancelFunc) {
_, _, err := downLoadFile(ctx, client, bucket, key, path, wg, cancel)
if err != nil {
fmt.Println("err=", err)
}
}(key, path, cancel)
}
//key = "视频/多文件打包接口文档.md"
//timer := time.NewTicker(time.Millisecond * 2000)
//defer timer.Stop()
//go func(timer *time.Ticker, cancel context.CancelFunc) {
// <-timer.C
// cancel()
//}(timer, cancel)
wg.Wait()
fmt.Println("all the task is end...........")
}
func downLoadFile(ctx context.Context, client *s3.Client, bucket, key, path string, wg *sync.WaitGroup, cancel context.CancelFunc) (*os.File, int64, error) {
downloader := manager.NewDownloader(client)
input := &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
}
downloadFile, err := os.Create(path)
defer func() {
wg.Done()
if err := downloadFile.Close(); err != nil {
log.Errorf("error when close file %v %v", downloadFile, err)
return
}
}()
downloadSize, err := downloader.Download(ctx, downloadFile, input, func(downloader *manager.Downloader) {
downloader.PartSize = 10 * 1024 * 1024
downloader.Concurrency = 5
})
atomic.AddInt64(&totalFileSize, downloadSize)
if totalFileSize > fileSizeLimit {
cancel()
}
if err != nil {
fmt.Errorf("download file err %v", err)
}
fmt.Println("downloadSize=", downloadSize)
return downloadFile, downloadSize, err
}
1490
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
