From b5f6ae64182d60e0bb20b60772e5407b88e95a81 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Sat, 6 Nov 2021 04:28:54 +0900 Subject: [PATCH] Add required Content-MD5 middleware This enforces that the Content-MD5 header is present in PutObject and UploadPart requests. --- README.md | 1 + src/main/java/org/gaul/s3proxy/Main.java | 7 + .../gaul/s3proxy/RequiredMD5BlobStore.java | 64 +++++++++ .../org/gaul/s3proxy/S3ProxyConstants.java | 3 + .../java/org/gaul/s3proxy/S3ProxyHandler.java | 8 +- .../s3proxy/RequiredMD5BlobStoreTest.java | 131 ++++++++++++++++++ 6 files changed, 212 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/gaul/s3proxy/RequiredMD5BlobStore.java create mode 100644 src/test/java/org/gaul/s3proxy/RequiredMD5BlobStoreTest.java diff --git a/README.md b/README.md index d06a1f2..6abcea7 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,7 @@ S3Proxy can modify its behavior based on middlewares: * [eventual consistency modeling](https://github.com/gaul/s3proxy/wiki/Middleware---eventual-consistency) * [large object mocking](https://github.com/gaul/s3proxy/wiki/Middleware-large-object-mocking) * [read-only](https://github.com/gaul/s3proxy/wiki/Middleware-read-only) +* [required Content-MD5](https://github.com/gaul/s3proxy/wiki/Middleware-required-Content-MD5) * [sharded backend containers](https://github.com/gaul/s3proxy/wiki/Middleware-sharded-backend) ## Limitations diff --git a/src/main/java/org/gaul/s3proxy/Main.java b/src/main/java/org/gaul/s3proxy/Main.java index e0b6b28..70a59aa 100644 --- a/src/main/java/org/gaul/s3proxy/Main.java +++ b/src/main/java/org/gaul/s3proxy/Main.java @@ -240,6 +240,13 @@ public final class Main { blobStore = ReadOnlyBlobStore.newReadOnlyBlobStore(blobStore); } + String requiredContentMD5BlobStore = properties.getProperty( + S3ProxyConstants.PROPERTY_REQUIRED_CONTENT_MD5_BLOBSTORE); + if ("true".equalsIgnoreCase(requiredContentMD5BlobStore)) { + System.err.println("Using required Content-MD5 storage backend"); + blobStore = RequiredMD5BlobStore.newRequiredMD5BlobStore(blobStore); + } + ImmutableBiMap aliases = AliasBlobStore.parseAliases( properties); if (!aliases.isEmpty()) { diff --git a/src/main/java/org/gaul/s3proxy/RequiredMD5BlobStore.java b/src/main/java/org/gaul/s3proxy/RequiredMD5BlobStore.java new file mode 100644 index 0000000..58b1553 --- /dev/null +++ b/src/main/java/org/gaul/s3proxy/RequiredMD5BlobStore.java @@ -0,0 +1,64 @@ +/* + * Copyright 2014-2021 Andrew Gaul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gaul.s3proxy; + +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.domain.MultipartPart; +import org.jclouds.blobstore.domain.MultipartUpload; +import org.jclouds.blobstore.options.PutOptions; +import org.jclouds.blobstore.util.ForwardingBlobStore; +import org.jclouds.io.Payload; + +/** This class is a BlobStore wrapper which requires the Content-MD5 header. */ +final class RequiredMD5BlobStore extends ForwardingBlobStore { + private RequiredMD5BlobStore(BlobStore blobStore) { + super(blobStore); + } + + static BlobStore newRequiredMD5BlobStore(BlobStore blobStore) { + return new RequiredMD5BlobStore(blobStore); + } + + @Override + public String putBlob(String containerName, Blob blob) { + if (blob.getMetadata().getContentMetadata().getContentMD5AsHashCode() == + null) { + throw new IllegalArgumentException("Content-MD5 header required"); + } + return super.putBlob(containerName, blob); + } + + @Override + public String putBlob(final String containerName, Blob blob, + final PutOptions options) { + if (blob.getMetadata().getContentMetadata().getContentMD5AsHashCode() == + null) { + throw new IllegalArgumentException("Content-MD5 header required"); + } + return super.putBlob(containerName, blob, options); + } + + @Override + public MultipartPart uploadMultipartPart(MultipartUpload mpu, + int partNumber, Payload payload) { + if (payload.getContentMetadata().getContentMD5AsHashCode() == null) { + throw new IllegalArgumentException("Content-MD5 header required"); + } + return super.uploadMultipartPart(mpu, partNumber, payload); + } +} diff --git a/src/main/java/org/gaul/s3proxy/S3ProxyConstants.java b/src/main/java/org/gaul/s3proxy/S3ProxyConstants.java index eec35bd..66669f7 100644 --- a/src/main/java/org/gaul/s3proxy/S3ProxyConstants.java +++ b/src/main/java/org/gaul/s3proxy/S3ProxyConstants.java @@ -99,6 +99,9 @@ public final class S3ProxyConstants { /** Prevent mutations. */ public static final String PROPERTY_READ_ONLY_BLOBSTORE = "s3proxy.read-only-blobstore"; + /** Require Content-MD5 header when creating objects. */ + public static final String PROPERTY_REQUIRED_CONTENT_MD5_BLOBSTORE = + "s3proxy.required-content-md5-blobstore"; /** Shard objects across a specified number of buckets. */ public static final String PROPERTY_SHARDED_BLOBSTORE = "s3proxy.sharded-blobstore"; diff --git a/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java b/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java index ffdf6f1..772b9c5 100644 --- a/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java +++ b/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java @@ -2178,8 +2178,11 @@ public class S3ProxyHandler { if (Quirks.MULTIPART_REQUIRES_STUB.contains(getBlobStoreType( blobStore))) { - blobStore.putBlob(containerName, builder.name(mpu.id()).build(), - options); + byte[] stubPayload = new byte[0]; + blobStore.putBlob(containerName, builder.name(mpu.id()) + .payload(stubPayload) + .contentMD5(Hashing.md5().hashBytes(stubPayload)) + .build(), options); } response.setCharacterEncoding(UTF_8); @@ -2654,6 +2657,7 @@ public class S3ProxyHandler { his.hash().asBytes()); } else { Payload payload = Payloads.newInputStreamPayload(is); + // TODO: Content-MD5 payload.getContentMetadata().setContentLength(contentLength); MultipartPart part = blobStore.uploadMultipartPart(mpu, diff --git a/src/test/java/org/gaul/s3proxy/RequiredMD5BlobStoreTest.java b/src/test/java/org/gaul/s3proxy/RequiredMD5BlobStoreTest.java new file mode 100644 index 0000000..a7a2009 --- /dev/null +++ b/src/test/java/org/gaul/s3proxy/RequiredMD5BlobStoreTest.java @@ -0,0 +1,131 @@ +/* + * Copyright 2014-2021 Andrew Gaul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gaul.s3proxy; + +import java.util.Random; + +import com.google.common.collect.ImmutableList; +import com.google.common.hash.Hashing; +import com.google.inject.Module; + +import org.assertj.core.api.Fail; +import org.jclouds.ContextBuilder; +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.domain.MultipartUpload; +import org.jclouds.blobstore.options.PutOptions; +import org.jclouds.io.Payload; +import org.jclouds.io.Payloads; +import org.jclouds.logging.slf4j.config.SLF4JLoggingModule; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public final class RequiredMD5BlobStoreTest { + private BlobStoreContext context; + private BlobStore blobStore; + private String containerName; + private BlobStore requiredMD5BlobStore; + + @Before + public void setUp() throws Exception { + containerName = createRandomContainerName(); + + context = ContextBuilder + .newBuilder("transient") + .credentials("identity", "credential") + .modules(ImmutableList.of(new SLF4JLoggingModule())) + .build(BlobStoreContext.class); + blobStore = context.getBlobStore(); + blobStore.createContainerInLocation(null, containerName); + requiredMD5BlobStore = RequiredMD5BlobStore.newRequiredMD5BlobStore( + blobStore); + } + + @After + public void tearDown() throws Exception { + if (context != null) { + blobStore.deleteContainer(containerName); + context.close(); + } + } + + @Test + public void testPutBlob() throws Exception { + String blobName = TestUtils.createRandomBlobName(); + byte[] data = new byte[1]; + Blob blob = requiredMD5BlobStore.blobBuilder(blobName).payload(data) + .build(); + try { + requiredMD5BlobStore.putBlob(containerName, blob); + Fail.failBecauseExceptionWasNotThrown( + IllegalArgumentException.class); + } catch (IllegalArgumentException iae) { + // expected + } + + blob.getMetadata().getContentMetadata().setContentMD5( + Hashing.md5().hashBytes(data)); + requiredMD5BlobStore.putBlob(containerName, blob); + } + + @Test + public void testPutBlobOptions() throws Exception { + String blobName = TestUtils.createRandomBlobName(); + byte[] data = new byte[1]; + Blob blob = requiredMD5BlobStore.blobBuilder(blobName).payload(data) + .build(); + try { + requiredMD5BlobStore.putBlob(containerName, blob, new PutOptions()); + Fail.failBecauseExceptionWasNotThrown( + IllegalArgumentException.class); + } catch (IllegalArgumentException iae) { + // expected + } + + blob.getMetadata().getContentMetadata().setContentMD5( + Hashing.md5().hashBytes(data)); + requiredMD5BlobStore.putBlob(containerName, blob, new PutOptions()); + } + + @Test + public void testUploadMultipartPart() throws Exception { + String blobName = TestUtils.createRandomBlobName(); + Blob blob = requiredMD5BlobStore.blobBuilder(blobName).build(); + MultipartUpload mpu = requiredMD5BlobStore.initiateMultipartUpload( + containerName, blob.getMetadata(), PutOptions.NONE); + byte[] data = new byte[1]; + Payload payload = Payloads.newPayload(data); + + try { + requiredMD5BlobStore.uploadMultipartPart(mpu, 1, payload); + Fail.failBecauseExceptionWasNotThrown( + IllegalArgumentException.class); + } catch (IllegalArgumentException iae) { + // expected + } + + payload.getContentMetadata().setContentMD5( + Hashing.md5().hashBytes(data)); + requiredMD5BlobStore.uploadMultipartPart(mpu, 1, payload); + } + + private static String createRandomContainerName() { + return "container-" + new Random().nextInt(Integer.MAX_VALUE); + } +}