From 02fbdc149bad3d00f56a99be080f5e9f95c3f8c2 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Wed, 27 Aug 2025 15:43:41 -0700 Subject: [PATCH] Parse but do not enforce CRC32 This suffices to allow AWS SDK for Java v2 to work without additional configuration. A more complete solution would allow for additional checksum algorithms and actually enforce the checksums. Fixes #830. --- pom.xml | 5 ++ .../java/org/gaul/s3proxy/AwsHttpHeaders.java | 1 + .../java/org/gaul/s3proxy/AwsSignature.java | 2 + .../org/gaul/s3proxy/ChunkedInputStream.java | 10 ++- .../java/org/gaul/s3proxy/S3ProxyHandler.java | 6 ++ .../java/org/gaul/s3proxy/AwsSdk2Test.java | 84 +++++++++++++++++++ .../java/org/gaul/s3proxy/AwsSdkTest.java | 2 +- 7 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 src/test/java/org/gaul/s3proxy/AwsSdk2Test.java diff --git a/pom.xml b/pom.xml index 90d8fa6..4839c77 100644 --- a/pom.xml +++ b/pom.xml @@ -554,6 +554,11 @@ 7.5.1 test + + software.amazon.awssdk + s3 + 2.30.33 + javax.annotation javax.annotation-api diff --git a/src/main/java/org/gaul/s3proxy/AwsHttpHeaders.java b/src/main/java/org/gaul/s3proxy/AwsHttpHeaders.java index 8c2b599..9f29882 100644 --- a/src/main/java/org/gaul/s3proxy/AwsHttpHeaders.java +++ b/src/main/java/org/gaul/s3proxy/AwsHttpHeaders.java @@ -38,6 +38,7 @@ final class AwsHttpHeaders { static final String REQUEST_ID = "x-amz-request-id"; static final String SDK_CHECKSUM_ALGORITHM = "x-amz-sdk-checksum-algorithm"; static final String STORAGE_CLASS = "x-amz-storage-class"; + static final String TRAILER = "x-amz-trailer"; static final String TRANSFER_ENCODING = "x-amz-te"; static final String USER_AGENT = "x-amz-user-agent"; diff --git a/src/main/java/org/gaul/s3proxy/AwsSignature.java b/src/main/java/org/gaul/s3proxy/AwsSignature.java index e3ad048..60d5197 100644 --- a/src/main/java/org/gaul/s3proxy/AwsSignature.java +++ b/src/main/java/org/gaul/s3proxy/AwsSignature.java @@ -281,6 +281,8 @@ final class AwsSignature { } else if ("STREAMING-AWS4-HMAC-SHA256-PAYLOAD".equals( xAmzContentSha256)) { digest = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"; + } else if ("STREAMING-UNSIGNED-PAYLOAD-TRAILER".equals(xAmzContentSha256)) { + digest = "STREAMING-UNSIGNED-PAYLOAD-TRAILER"; } else if ("UNSIGNED-PAYLOAD".equals(xAmzContentSha256)) { digest = "UNSIGNED-PAYLOAD"; } else { diff --git a/src/main/java/org/gaul/s3proxy/ChunkedInputStream.java b/src/main/java/org/gaul/s3proxy/ChunkedInputStream.java index 3e57f69..5eeb693 100644 --- a/src/main/java/org/gaul/s3proxy/ChunkedInputStream.java +++ b/src/main/java/org/gaul/s3proxy/ChunkedInputStream.java @@ -49,8 +49,14 @@ final class ChunkedInputStream extends FilterInputStream { return -1; } String[] parts = line.split(";", 2); - currentLength = Integer.parseInt(parts[0], 16); - currentSignature = parts[1]; + if (parts[0].startsWith("x-amz-checksum-crc32:")) { + currentLength = 0; + } else { + currentLength = Integer.parseInt(parts[0], 16); + } + if (parts.length > 1) { + currentSignature = parts[1]; + } chunk = new byte[currentLength]; currentIndex = 0; ByteStreams.readFully(in, chunk); diff --git a/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java b/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java index a7d8853..59ac1a0 100644 --- a/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java +++ b/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java @@ -177,6 +177,7 @@ public class S3ProxyHandler { AwsHttpHeaders.METADATA_DIRECTIVE, AwsHttpHeaders.SDK_CHECKSUM_ALGORITHM, // TODO: ignoring header AwsHttpHeaders.STORAGE_CLASS, + AwsHttpHeaders.TRAILER, // TODO: ignoring header AwsHttpHeaders.TRANSFER_ENCODING, // TODO: ignoring header AwsHttpHeaders.USER_AGENT ); @@ -514,6 +515,8 @@ public class S3ProxyHandler { AwsHttpHeaders.CONTENT_SHA256); if ("STREAMING-AWS4-HMAC-SHA256-PAYLOAD".equals(contentSha256)) { is = new ChunkedInputStream(is); + } else if ("STREAMING-UNSIGNED-PAYLOAD-TRAILER".equals(contentSha256)) { + is = new ChunkedInputStream(is); } } else if (requestIdentity == null) { throw new S3Exception(S3ErrorCode.ACCESS_DENIED); @@ -603,6 +606,9 @@ public class S3ProxyHandler { contentSha256)) { payload = new byte[0]; is = new ChunkedInputStream(is); + } else if ("STREAMING-UNSIGNED-PAYLOAD-TRAILER".equals(contentSha256)) { + payload = new byte[0]; + is = new ChunkedInputStream(is); } else if ("UNSIGNED-PAYLOAD".equals(contentSha256)) { payload = new byte[0]; } else { diff --git a/src/test/java/org/gaul/s3proxy/AwsSdk2Test.java b/src/test/java/org/gaul/s3proxy/AwsSdk2Test.java new file mode 100644 index 0000000..685208a --- /dev/null +++ b/src/test/java/org/gaul/s3proxy/AwsSdk2Test.java @@ -0,0 +1,84 @@ +/* + * Copyright 2014-2025 Andrew Gaul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gaul.s3proxy; + +import org.jclouds.blobstore.BlobStoreContext; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.http.SdkHttpConfigurationOption; +import software.amazon.awssdk.http.apache.ApacheHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.utils.AttributeMap; + +public final class AwsSdk2Test { + private BlobStoreContext context; + private S3Client s3Client; + private String containerName; + + @Before + public void setUp() throws Exception { + var info = TestUtils.startS3Proxy(System.getProperty("s3proxy.test.conf", "s3proxy.conf")); + context = info.getBlobStore().getContext(); + + var attributeMap = AttributeMap.builder() + .put(SdkHttpConfigurationOption.TRUST_ALL_CERTIFICATES, true) + .build(); + s3Client = S3Client.builder() + .credentialsProvider( + StaticCredentialsProvider.create( + AwsBasicCredentials.create(info.getS3Identity(), info.getS3Credential()))) + .region(Region.US_EAST_1) + .endpointOverride(info.getSecureEndpoint()) + .httpClient(ApacheHttpClient.builder() + .buildWithDefaults(attributeMap)) + .build(); + + containerName = AwsSdkTest.createRandomContainerName(); + info.getBlobStore().createContainerInLocation(null, containerName); + } + + @After + public void tearDown() throws Exception { + if (s3Client != null) { + s3Client.close(); + } + if (context != null) { + context.getBlobStore().deleteContainer(containerName); + context.close(); + } + } + + @Test + public void testPutObject() throws Exception { + var key = "testPutObject"; + var byteSource = TestUtils.randomByteSource().slice(0, 1024); + + var putRequest = PutObjectRequest.builder() + .bucket(containerName) + .key(key) + .build(); + + s3Client.putObject(putRequest, RequestBody.fromBytes(byteSource.read())); + } +} diff --git a/src/test/java/org/gaul/s3proxy/AwsSdkTest.java b/src/test/java/org/gaul/s3proxy/AwsSdkTest.java index 5d5ddcc..f0c55f8 100644 --- a/src/test/java/org/gaul/s3proxy/AwsSdkTest.java +++ b/src/test/java/org/gaul/s3proxy/AwsSdkTest.java @@ -1828,7 +1828,7 @@ public final class AwsSdkTest { } } - private static String createRandomContainerName() { + static String createRandomContainerName() { return "s3proxy-" + new Random().nextInt(Integer.MAX_VALUE); } }