springboot AWS S3的使用案例
能文华
项目中有将文件存储在S3上，怎么获取，以下是使用案例。
pom.xml
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.12.184</version>
</dependency>
application.yml
aws-s3:
bucketName: hsx-test
region: cn-northwest-1
hosts: s3.cn-northwest-1.amazonaws.com.cn
accessKey: XXXX
secretKey: XXXXXXXXXXX
amazonS3MaxConnections: 200
amazonS3SocketTimeout: 10000
amazonS3MaxErrorRetry: 2
amazonS3Protocol: http
AwsS3Config.java
/**
* @program: aws_s3
* @description: UploadConfig
* @author: sunyuhua
* @create: 2022-03-28 10:07
**/
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import java.util.List;
@ConfigurationProperties(prefix = "aws-s3")
@Data
public class AwsS3Config {
/**
* S3支持集群的链接方式
*/
private List<String> hosts;
private String accessKey;
private String secretKey;
/**
* 区域
*/
private String region;
/**
* 连接协议
*/
private String amazonS3Protocol;
/**
* 失败请求重试次数
*/
private Integer amazonS3MaxErrorRetry;
/**
* 超时时间
*/
private Integer amazonS3SocketTimeout;
/**
* 最大连接数
*/
private Integer amazonS3MaxConnections;
/**
* 桶名称
*/
private String bucketName;
}
AmazonS3Config.java
package com.fastonetech.aws_s3;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.Protocol;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.CreateBucketRequest;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import javax.annotation.Resource;
@Configuration
@EnableConfigurationProperties({AwsS3Config.class})
public class AmazonS3Config {
@Resource
private AwsS3Config awsS3Config;
@Bean(name = "amazonS3")
public AmazonS3 getAmazonS3() {
String accessKey = awsS3Config.getAccessKey();
String secretKey = awsS3Config.getSecretKey();
AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
ClientConfiguration conf = new ClientConfiguration();
// 设置AmazonS3使用的最大连接数
conf.setMaxConnections(awsS3Config.getAmazonS3MaxConnections());
// 设置socket超时时间
conf.setSocketTimeout(awsS3Config.getAmazonS3SocketTimeout());
// 设置失败请求重试次数
conf.setMaxErrorRetry(awsS3Config.getAmazonS3MaxErrorRetry());
// 设置协议
if (!"blank".equals(awsS3Config.getAmazonS3Protocol())) {
switch (awsS3Config.getAmazonS3Protocol()) {
case "https":
conf.setProtocol(Protocol.HTTPS);
break;
case "http":
conf.setProtocol(Protocol.HTTP);
break;
default:
break;
}
}
AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
.withCredentials(new AWSStaticCredentialsProvider(credentials))
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(awsS3Config.getHosts().get(0), awsS3Config.getRegion()))
.withClientConfiguration(conf)
.build();
checkAndCreateBucket(s3Client);
return s3Client;
}
@Bean(name = "transferManager")
public TransferManager getTransferManager() {
return TransferManagerBuilder.standard().withS3Client(getAmazonS3()).build();
}
/**
* 检查桶是否存在，不存在则创建创建
*
* @param s3Client
*/
private void checkAndCreateBucket(AmazonS3 s3Client) {
boolean exists = s3Client.doesBucketExistV2(awsS3Config.getBucketName());
if (!exists) {
CreateBucketRequest request = new CreateBucketRequest(awsS3Config.getBucketName());
s3Client.createBucket(request);
}
}
}
AWSS3Controller.java
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.annotation.Resource;
import java.io.IOException;
/**
* @program: aws_s3
* @description:
* @author: sunyuhua
* @create: 2022-03-28 10:25
**/
@RestController
@RequestMapping("/s3/")
public class AWSS3Controller {
@Resource
private AWSS3Service S3Service;
/**
* 获取S3上文件的内容
* @throws IOException
*/
@GetMapping("/getObject")
public void getObject() throws IOException {
String objectName="dev-nat-log/AWSLogs/085211525492/vpcflowlogs/cn-northwest-1/2022/01/02/085211525492_vpcflowlogs_cn-northwest-1_fl-0a3c82b9af904d152_20220102T0000Z_bf6a5f7a.log.gz";
S3Service.getObject(objectName);
}
/**
* 获取文件的ObjectMetadata
* @throws IOException
*/
@GetMapping("/getObjectMetadata")
public void getObjectMetadata() throws IOException{
S3Service.getObjectMetadata();
}
/**
* 获取文件的ObjectSummaries
*/
@GetMapping("/getObjectSummaries")
public void getObjectSummaries(){
S3Service.getObjectSummaries();
}
/**
* 列出指定文件夹下的所有文件 V1版本
* @throws IOException
*/
@GetMapping("/listObjects")
public void listObjects() throws IOException {
S3Service.listObjects();
}
/**
* 列出指定文件夹下的所有文件 V2版本
* @throws IOException
*/
@GetMapping("/listObjectsV2")
public void listObjectsV2() throws IOException{
S3Service.listObjectsV2();
}
/**
* 复制文件
* @throws IOException
*/
@GetMapping("/copyObject")
public void copyObject() throws IOException{
S3Service.copyObject();
}
/**
* 删除文件
* @throws IOException
*/
@GetMapping("/deleteObject")
public void deleteObject() throws IOException{
S3Service.deleteObject();
}
}
AWSS3Service.java
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.DeleteObjectRequest;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ListObjectsV2Request;
import com.amazonaws.services.s3.model.ListObjectsV2Result;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.zip.GZIPInputStream;
/**
* @program: aws_s3
* @description:
* @author: sunyuhua
* @create: 2022-03-28 10:22
**/
@Service
@Slf4j
public class AWSS3Service {
@Resource
private AmazonS3 amazonS3;
public void getObject(String objectName) throws IOException {
String bucketName="hsx-test";
S3Object fileObject = amazonS3.getObject(bucketName,objectName);
//可以通过getObjectMetadata方法获取对象的ContentType等元数据信息
String contentType = fileObject.getObjectMetadata().getContentType();
GZIPInputStream gZIPInputStream = null;
if(contentType.equals("application/x-gzip")){
gZIPInputStream = new GZIPInputStream(fileObject.getObjectContent());
BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(gZIPInputStream));
String line = null;
//第一行是标题，跳过第一行
bufferedReader.readLine();
while ((line = bufferedReader.readLine()) != null) {
//转化为UTF-8的编码格式
line = new String(line.getBytes(StandardCharsets.UTF_8));
//---此五分钟代表没有流量
if(line.contains("-")) continue;
System.out.println("\n" + line);
System.out.println("******"+(line.split("\\s+")[0].trim())+"******");
System.out.println(line+"是否是内网"+IpUtils.internalIp((line.split("\\s+")[0])));
//System.out.println("\n" + line);
}
bufferedReader.close();
}else{
BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(fileObject.getObjectContent()));
String line = null;
while ((line = bufferedReader.readLine()) != null) {
//转化为UTF-8的编码格式
//IpUtils.internalIp(line.getBytes(StandardCharsets.UTF_8))
System.out.println("\n********" +new String(subBytes(line.getBytes(StandardCharsets.UTF_8),0,15)) );
line = new String(line.getBytes(StandardCharsets.UTF_8));
//System.out.println("\n" + line);
}
bufferedReader.close();
}
}
public static byte[] subBytes(byte[] src, int begin, int count) {
byte[] bs = new byte[count];
for (int i=begin;i<begin+count; i++) bs[i-begin] = src[i];
return bs;
}
public void getObjectMetadata(){
String bucketName="hsx-test";
String objectName="dev-nat-log/AWSLogs/085211525492/vpcflowlogs/cn-northwest-1/2022/01/02/085211525492_vpcflowlogs_cn-northwest-1_fl-0a3c82b9af904d152_20220102T0000Z_bf6a5f7a.log.gz";
ObjectMetadata objectMetadata=amazonS3.getObjectMetadata(bucketName,objectName);
log.info("objectMetadata"+objectMetadata.toString());
}
/**
* ObjectSummaries
*/
public void getObjectSummaries(){
String bucketName="hsx-test";
ObjectListing objectListing = amazonS3.listObjects(bucketName);
List<S3ObjectSummary> sums = objectListing.getObjectSummaries();
for (S3ObjectSummary s : sums) {
System.out.println("\t" + s.getKey());
}
}
public void listObjects() throws IOException {
String bucketName="hsx-test";
ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
listObjectsRequest.setBucketName(bucketName);
listObjectsRequest.setMaxKeys(20);
listObjectsRequest.setDelimiter("/");
listObjectsRequest.setPrefix("dev-nat-log/AWSLogs/085211525492/vpcflowlogs/cn-northwest-1/2022/02/07/");
ObjectListing listObjects = amazonS3.listObjects(listObjectsRequest);
do {
if (listObjects.isTruncated()) {
ListObjectsRequest request = new ListObjectsRequest();
request.setBucketName(listObjectsRequest.getBucketName());
request.setMarker(listObjects.getNextMarker());
listObjects = amazonS3.listObjects(request);
List<S3ObjectSummary> sums = listObjects.getObjectSummaries();
for (S3ObjectSummary s : sums) {
if(s.getKey().contains("dev-nat-log/AWSLogs/085211525492/vpcflowlogs/cn-northwest-1/2022/02/07/")){
//System.out.println("\t" + s.getKey());
getObject(s.getKey());
}
}
} else {
break;
}
} while (listObjects != null);
}
public void listObjectsV2() throws IOException {
String bucketName="hsx-test";
ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucketName).withMaxKeys(300)
.withDelimiter("/").withPrefix("dev-nat-log/AWSLogs/085211525492/vpcflowlogs/cn-northwest-1/2022/02/07/");
ListObjectsV2Result listObjects = amazonS3.listObjectsV2(req);
do {
if (listObjects.isTruncated()) {
ListObjectsV2Request request = new ListObjectsV2Request();
request.withPrefix(listObjects.getPrefix());
request.withBucketName(listObjects.getBucketName());
listObjects = amazonS3.listObjectsV2(request);
List<S3ObjectSummary> sums = listObjects.getObjectSummaries();
for (S3ObjectSummary s : sums) {
if(s.getKey().contains("dev-nat-log/AWSLogs/085211525492/vpcflowlogs/cn-northwest-1/2022/02/07/")
&& !s.getKey().contains("back")
){
getObject(s.getKey());
}
}
} else {
break;
}
} while (listObjects != null);
}
public void copyObject(){
String from_bucket="hsx-test";
String from_object_key="dev-nat-log/AWSLogs/085211525492/vpcflowlogs/cn-northwest-1/2022/02/02/085211525492_vpcflowlogs_cn-northwest-1_fl-0a3c82b9af904d152_20220202T0000Z_36cb77fa.log.gz";
String to_bucket="hsx-test";
String to_object_key="dev-nat-log/AWSLogs/085211525492/vpcflowlogs/cn-northwest-1/2022/02/02/back/085211525492_vpcflowlogs_cn-northwest-1_fl-0a3c82b9af904d152_20220202T0000Z_36cb77fa.log.gz";
amazonS3.copyObject(from_bucket, from_object_key, to_bucket, to_object_key);
}
public void deleteObject(){
String from_bucket="hsx-test";
String from_object_key="dev-nat-log/AWSLogs/085211525492/vpcflowlogs/cn-northwest-1/2022/02/02/085211525492_vpcflowlogs_cn-northwest-1_fl-0a3c82b9af904d152_20220202T0005Z_331483d7.log.gz";
amazonS3.deleteObject(new DeleteObjectRequest(from_bucket, from_object_key));
log.info("删除执行完毕");
}
}
类似资料：