解决AWS S3 SDK V2操作阿里云OSS的Bucket路径问题

背景

• 由于项目有私网部署需求，对象存储选用MinIO，公网使用阿里云OSS，故选用S3 SDK统一进行操作
• MinIO全流程正常
• 阿里云OSS限定必须以Virtual hosted style方式访问，因此在指定bucket的同时，还需要在域名中加入bucket前缀，但经测试任意的对象操作的路径都会被转换为/{bucket}/{key}的方式(Path Style)，导致阿里云多创建了一层bucket目录。
• Aws sdk 版本为software.amazon.awssdk:s3:2.17.214
• 需要使用到新版的签名分片上传链接功能，而v1版本并不支持

原因

• 新版Sdk(v2)针对Object的访问路径实际内部做了两种访问风格的自适应，但目前仅限以s3开头的域名可以仅限Virtual hosted style方式的访问，其他域名均会自动转为Path风格。关于此问题的描述在github上有相关讨论 https://github.com/aws/aws-sdk-java-v2/issues/2317。但需要修改源码，而且并非官方正式版本，不便于后期升级

解决方案

• 在创建S3Client对象时添加拦截器ExecutionInterceptor，根据具体业务场景修改默认的SdkHttpRequest对象
• 在创建S3Presigner 时，由于sdk目前没有提供添加拦截器的接口，但查看源码后发现内部是支持的，因此可以通过反射强制添加拦截器处理，复用上文中的拦截器对象
• 核心代码测试用例如下，可以在使用时根据业务场景进行封装

package s3.aliyunoss;

import lombok.Cleanup;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentials;
import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration;
import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
import software.amazon.awssdk.core.interceptor.Context;
import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.http.SdkHttpRequest;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3Configuration;
import software.amazon.awssdk.services.s3.model.*;
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest;

import java.lang.reflect.Field;
import java.net.URI;
import java.time.Duration;
import java.util.List;

/**
 * S3V2AliyunOssTest 
 *
 * @author ozvale
 */
@Disabled
@Slf4j
public class S3V2AliyunOssTest {
    private final String endpoint = "https://oss-cn-hangzhou.aliyuncs.com";
    private final String region = "oss-cn-hangzhou";
    private final String accessKeyId = "xxx";
    private final String accessKeySecret = "xxx";
    private final String bucket = "xxx";

    private ExecutionInterceptor createProcessRequestKeyExecutionInterceptor() {
        return new ExecutionInterceptor() {
            @Override
            public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context, ExecutionAttributes executionAttributes) {
                return context.httpRequest().copy(builder -> {
                    String key = context.request().getValueForField("Key", String.class).orElse(null);
                    builder.host(bucket + "." + builder.host());
                    if (key == null) {
                        builder.encodedPath("");
                    } else {
                        builder.encodedPath("/" + key);
                    }
                });
            }
        };
    }

    @SneakyThrows
    private S3Client createClient() {
        AwsCredentials credentials = AwsBasicCredentials.create(accessKeyId, accessKeySecret);
        ClientOverrideConfiguration clientOverrideConfiguration = ClientOverrideConfiguration.builder()
                .addExecutionInterceptor(createProcessRequestKeyExecutionInterceptor())
                .build();
        S3Client client = S3Client.builder()
                .region(Region.of(region))
                .credentialsProvider(() -> credentials)
                .endpointOverride(new URI(endpoint))
                .overrideConfiguration(clientOverrideConfiguration)
                .build();
        return client;
    }

    @SneakyThrows
    private S3Presigner createPresigner() {
        AwsCredentials credentials = AwsBasicCredentials.create(accessKeyId, accessKeySecret);
        S3Presigner presigner = S3Presigner.builder()
                .region(Region.of(region))
                .credentialsProvider(() -> credentials)
                .endpointOverride(new URI(endpoint))
                .build();
        //反射注入拦截器
        Field field = presigner.getClass().getDeclaredField("clientInterceptors");
        field.setAccessible(true);
        List<ExecutionInterceptor> clientInterceptors = (List<ExecutionInterceptor>) field.get(presigner);
        clientInterceptors.add(createProcessRequestKeyExecutionInterceptor());
        return presigner;
    }

    @Test
    void createBucket() {
        @Cleanup S3Client client = createClient();
        CreateBucketRequest request = CreateBucketRequest.builder().bucket(bucket).build();
        log.debug("start create bucket,request={}", request);
        CreateBucketResponse response = client.createBucket(request);
        log.debug("end create bucket,response={}", response);
    }

    @Test
    void headBucket() {
        @Cleanup S3Client client = createClient();
        HeadBucketRequest request = HeadBucketRequest.builder().bucket(bucket).build();
        log.debug("start head bucket,request={}", request);
        HeadBucketResponse response = client.headBucket(request);
        log.debug("end head bucket,response={}", response);
    }

    @Test
    void headObject() {
        @Cleanup S3Client client = createClient();
        String key = "123.jpg";
        HeadObjectRequest request = HeadObjectRequest.builder()
                .bucket(bucket).key(key)
                .build();
        log.debug("start head object,request={}", request);
        HeadObjectResponse response = client.headObject(request);
        log.debug("end head object,response={}", response);
    }

    @Test
    void putObject(){
        @Cleanup S3Client client = createClient();
        String key = "123.jpg";
        PutObjectRequest request = PutObjectRequest.builder().bucket(bucket).key(key).build();
        log.debug("start put object,request={}", request);
        PutObjectResponse response = client.putObject(request, RequestBody.fromBytes(new byte[1]));
        log.debug("end put object,response={}", response);
    }

    @Test
    void presignGetUrl() {
        @Cleanup S3Presigner presigner = createPresigner();
        String key = "123.jpg";
        GetObjectRequest getObjectRequest = GetObjectRequest.builder().bucket(bucket).key(key).build();
        GetObjectPresignRequest getObjectPresignRequest = GetObjectPresignRequest.builder()
                .getObjectRequest(getObjectRequest)
                .signatureDuration(Duration.ofSeconds(100))
                .build();
        log.debug("start presign get object,request={}", getObjectPresignRequest);
        PresignedGetObjectRequest presignedGetObjectRequest = presigner.presignGetObject(getObjectPresignRequest);
        log.debug("end presign get object,response={}", presignedGetObjectRequest);
        log.debug("presignedGetObjectRequest.url={}", presignedGetObjectRequest.url());
    }
}

说明

• 以上方法可以解决问题，但个人认为应不是最好的解决方法，如有更好解决方案欢迎提出
• 个人猜测后续版本应该会支持S3Presigner构建时添加拦截器，无需再使用反射的方式实现
• 针对内网对象存储服务如MinIO，可能需要增加配置项控制拦截器添加逻辑
• 经测试，以上方法针对腾讯云同样有效，对于兼容S3标准的其他厂商理论上也支持