6、AWS SDK for Go-使用s3manager批量上传下载

简介

介绍见：传送门

代码示例

package main

import (
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"os"
)

var (
	accessKey = "xxx"x
	secretKey = "xxx"
	region    = "oss-cn-beijing"
	endpoint  = "oss-cn-beijing.aliyuncs.com"
)

func main() {
	//只要不修改session，session就可以安全的并发使用。
	sess, err := session.NewSession(&aws.Config{
		Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""),
		Endpoint:    aws.String(endpoint),
		Region:      aws.String(region),
		//minio:true,oss:false
		S3ForcePathStyle: aws.Bool(false),
		//SDK 支持使用客户端 TLS 证书配置的环境和会话选项，这些证书作为客户端 TLS 握手的一部分发送以进行客户端身份验证。
		//如果使用，则需要 Cert 和 Key 值。如果缺少一个，或者无法加载文件的内容，则会返回一个错误。
		//ClientTLSCert:              nil,
		//ClientTLSKey:               nil,
	})
	if err != nil {
		panic(err)
	}

	svc := s3.New(sess)
	keys := []string{"3.jpg", "4.jpg"}
	BatchUploadWithClient(svc, "bkt-bj1", keys)
	BatchDownloadWithClient(svc, "bkt-bj1", keys)

}
func BatchUploadWithClient(svc *s3.S3, bucket string, keys []string) {
	uploader := s3manager.NewUploaderWithClient(svc, func(u *s3manager.Uploader) {
		// 定义将在内存中缓冲25个MiB的策略
		u.BufferProvider = s3manager.NewBufferedReadSeekerWriteToPool(25 * 1024 * 1024)
		u.PartSize = 64 * 1024 * 1024 // 每个部分 64MB
	})
	objects := make([]s3manager.BatchUploadObject, 0, len(keys))
	for _, key := range keys {
		fOpen, err := os.Open(key)
		if err != nil {
			panic(err)
		}
		defer fOpen.Close()
		objects = append(objects, s3manager.BatchUploadObject{
			Object: &s3manager.UploadInput{
				Body:   fOpen,
				Bucket: aws.String(bucket),
				Key:    aws.String(key),
			},
		})
	}

	iter := &s3manager.UploadObjectsIterator{Objects: objects}
	if err := uploader.UploadWithIterator(aws.BackgroundContext(), iter); err != nil {
		panic(err)
	}
}
func BatchDownloadWithClient(svc *s3.S3, bucket string, keys []string) {
	download := s3manager.NewDownloaderWithClient(svc, func(d *s3manager.Downloader) {
		d.PartSize = 64 * 1024 * 1024 // 64MB per part
		d.BufferProvider = s3manager.NewPooledBufferedWriterReadFromProvider(25 * 1024 * 1024)
	})
	objects := make([]s3manager.BatchDownloadObject, 0, len(keys))
	for _, key := range keys {
		f, err := os.Create("d_" + key)
		if err != nil {
			panic(err)
		}
		defer f.Close()
		objects = append(objects, s3manager.BatchDownloadObject{
			Object: &s3.GetObjectInput{
				Bucket: aws.String(bucket),
				Key:    aws.String(key),
			},
			Writer: f,
		})
	}

	iter := &s3manager.DownloadObjectsIterator{Objects: objects}
	if err := download.DownloadWithIterator(aws.BackgroundContext(), iter); err != nil {
		panic(err)
	}
}