kopia lustrzana https://github.com/gaul/s3proxy
Add required Content-MD5 middleware
This enforces that the Content-MD5 header is present in PutObject and UploadPart requests.pull/386/head
Andrew Gaul
rodzic
db2cc2a0ff
commit
b5f6ae6418
|
|
@ -105,6 +105,7 @@ S3Proxy can modify its behavior based on middlewares:
|
|||
* [eventual consistency modeling](https://github.com/gaul/s3proxy/wiki/Middleware---eventual-consistency)
|
||||
* [large object mocking](https://github.com/gaul/s3proxy/wiki/Middleware-large-object-mocking)
|
||||
* [read-only](https://github.com/gaul/s3proxy/wiki/Middleware-read-only)
|
||||
* [required Content-MD5](https://github.com/gaul/s3proxy/wiki/Middleware-required-Content-MD5)
|
||||
* [sharded backend containers](https://github.com/gaul/s3proxy/wiki/Middleware-sharded-backend)
|
||||
|
||||
## Limitations
|
||||
|
|
|
|||
|
|
@ -240,6 +240,13 @@ public final class Main {
|
|||
blobStore = ReadOnlyBlobStore.newReadOnlyBlobStore(blobStore);
|
||||
}
|
||||
|
||||
String requiredContentMD5BlobStore = properties.getProperty(
|
||||
S3ProxyConstants.PROPERTY_REQUIRED_CONTENT_MD5_BLOBSTORE);
|
||||
if ("true".equalsIgnoreCase(requiredContentMD5BlobStore)) {
|
||||
System.err.println("Using required Content-MD5 storage backend");
|
||||
blobStore = RequiredMD5BlobStore.newRequiredMD5BlobStore(blobStore);
|
||||
}
|
||||
|
||||
ImmutableBiMap<String, String> aliases = AliasBlobStore.parseAliases(
|
||||
properties);
|
||||
if (!aliases.isEmpty()) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,64 @@
|
|||
/*
|
||||
* Copyright 2014-2021 Andrew Gaul <andrew@gaul.org>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.gaul.s3proxy;
|
||||
|
||||
import org.jclouds.blobstore.BlobStore;
|
||||
import org.jclouds.blobstore.domain.Blob;
|
||||
import org.jclouds.blobstore.domain.MultipartPart;
|
||||
import org.jclouds.blobstore.domain.MultipartUpload;
|
||||
import org.jclouds.blobstore.options.PutOptions;
|
||||
import org.jclouds.blobstore.util.ForwardingBlobStore;
|
||||
import org.jclouds.io.Payload;
|
||||
|
||||
/** This class is a BlobStore wrapper which requires the Content-MD5 header. */
|
||||
final class RequiredMD5BlobStore extends ForwardingBlobStore {
|
||||
private RequiredMD5BlobStore(BlobStore blobStore) {
|
||||
super(blobStore);
|
||||
}
|
||||
|
||||
static BlobStore newRequiredMD5BlobStore(BlobStore blobStore) {
|
||||
return new RequiredMD5BlobStore(blobStore);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String putBlob(String containerName, Blob blob) {
|
||||
if (blob.getMetadata().getContentMetadata().getContentMD5AsHashCode() ==
|
||||
null) {
|
||||
throw new IllegalArgumentException("Content-MD5 header required");
|
||||
}
|
||||
return super.putBlob(containerName, blob);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String putBlob(final String containerName, Blob blob,
|
||||
final PutOptions options) {
|
||||
if (blob.getMetadata().getContentMetadata().getContentMD5AsHashCode() ==
|
||||
null) {
|
||||
throw new IllegalArgumentException("Content-MD5 header required");
|
||||
}
|
||||
return super.putBlob(containerName, blob, options);
|
||||
}
|
||||
|
||||
@Override
|
||||
public MultipartPart uploadMultipartPart(MultipartUpload mpu,
|
||||
int partNumber, Payload payload) {
|
||||
if (payload.getContentMetadata().getContentMD5AsHashCode() == null) {
|
||||
throw new IllegalArgumentException("Content-MD5 header required");
|
||||
}
|
||||
return super.uploadMultipartPart(mpu, partNumber, payload);
|
||||
}
|
||||
}
|
||||
|
|
@ -99,6 +99,9 @@ public final class S3ProxyConstants {
|
|||
/** Prevent mutations. */
|
||||
public static final String PROPERTY_READ_ONLY_BLOBSTORE =
|
||||
"s3proxy.read-only-blobstore";
|
||||
/** Require Content-MD5 header when creating objects. */
|
||||
public static final String PROPERTY_REQUIRED_CONTENT_MD5_BLOBSTORE =
|
||||
"s3proxy.required-content-md5-blobstore";
|
||||
/** Shard objects across a specified number of buckets. */
|
||||
public static final String PROPERTY_SHARDED_BLOBSTORE =
|
||||
"s3proxy.sharded-blobstore";
|
||||
|
|
|
|||
|
|
@ -2178,8 +2178,11 @@ public class S3ProxyHandler {
|
|||
|
||||
if (Quirks.MULTIPART_REQUIRES_STUB.contains(getBlobStoreType(
|
||||
blobStore))) {
|
||||
blobStore.putBlob(containerName, builder.name(mpu.id()).build(),
|
||||
options);
|
||||
byte[] stubPayload = new byte[0];
|
||||
blobStore.putBlob(containerName, builder.name(mpu.id())
|
||||
.payload(stubPayload)
|
||||
.contentMD5(Hashing.md5().hashBytes(stubPayload))
|
||||
.build(), options);
|
||||
}
|
||||
|
||||
response.setCharacterEncoding(UTF_8);
|
||||
|
|
@ -2654,6 +2657,7 @@ public class S3ProxyHandler {
|
|||
his.hash().asBytes());
|
||||
} else {
|
||||
Payload payload = Payloads.newInputStreamPayload(is);
|
||||
// TODO: Content-MD5
|
||||
payload.getContentMetadata().setContentLength(contentLength);
|
||||
|
||||
MultipartPart part = blobStore.uploadMultipartPart(mpu,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,131 @@
|
|||
/*
|
||||
* Copyright 2014-2021 Andrew Gaul <andrew@gaul.org>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.gaul.s3proxy;
|
||||
|
||||
import java.util.Random;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.hash.Hashing;
|
||||
import com.google.inject.Module;
|
||||
|
||||
import org.assertj.core.api.Fail;
|
||||
import org.jclouds.ContextBuilder;
|
||||
import org.jclouds.blobstore.BlobStore;
|
||||
import org.jclouds.blobstore.BlobStoreContext;
|
||||
import org.jclouds.blobstore.domain.Blob;
|
||||
import org.jclouds.blobstore.domain.MultipartUpload;
|
||||
import org.jclouds.blobstore.options.PutOptions;
|
||||
import org.jclouds.io.Payload;
|
||||
import org.jclouds.io.Payloads;
|
||||
import org.jclouds.logging.slf4j.config.SLF4JLoggingModule;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
public final class RequiredMD5BlobStoreTest {
|
||||
private BlobStoreContext context;
|
||||
private BlobStore blobStore;
|
||||
private String containerName;
|
||||
private BlobStore requiredMD5BlobStore;
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
containerName = createRandomContainerName();
|
||||
|
||||
context = ContextBuilder
|
||||
.newBuilder("transient")
|
||||
.credentials("identity", "credential")
|
||||
.modules(ImmutableList.<Module>of(new SLF4JLoggingModule()))
|
||||
.build(BlobStoreContext.class);
|
||||
blobStore = context.getBlobStore();
|
||||
blobStore.createContainerInLocation(null, containerName);
|
||||
requiredMD5BlobStore = RequiredMD5BlobStore.newRequiredMD5BlobStore(
|
||||
blobStore);
|
||||
}
|
||||
|
||||
@After
|
||||
public void tearDown() throws Exception {
|
||||
if (context != null) {
|
||||
blobStore.deleteContainer(containerName);
|
||||
context.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPutBlob() throws Exception {
|
||||
String blobName = TestUtils.createRandomBlobName();
|
||||
byte[] data = new byte[1];
|
||||
Blob blob = requiredMD5BlobStore.blobBuilder(blobName).payload(data)
|
||||
.build();
|
||||
try {
|
||||
requiredMD5BlobStore.putBlob(containerName, blob);
|
||||
Fail.failBecauseExceptionWasNotThrown(
|
||||
IllegalArgumentException.class);
|
||||
} catch (IllegalArgumentException iae) {
|
||||
// expected
|
||||
}
|
||||
|
||||
blob.getMetadata().getContentMetadata().setContentMD5(
|
||||
Hashing.md5().hashBytes(data));
|
||||
requiredMD5BlobStore.putBlob(containerName, blob);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPutBlobOptions() throws Exception {
|
||||
String blobName = TestUtils.createRandomBlobName();
|
||||
byte[] data = new byte[1];
|
||||
Blob blob = requiredMD5BlobStore.blobBuilder(blobName).payload(data)
|
||||
.build();
|
||||
try {
|
||||
requiredMD5BlobStore.putBlob(containerName, blob, new PutOptions());
|
||||
Fail.failBecauseExceptionWasNotThrown(
|
||||
IllegalArgumentException.class);
|
||||
} catch (IllegalArgumentException iae) {
|
||||
// expected
|
||||
}
|
||||
|
||||
blob.getMetadata().getContentMetadata().setContentMD5(
|
||||
Hashing.md5().hashBytes(data));
|
||||
requiredMD5BlobStore.putBlob(containerName, blob, new PutOptions());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUploadMultipartPart() throws Exception {
|
||||
String blobName = TestUtils.createRandomBlobName();
|
||||
Blob blob = requiredMD5BlobStore.blobBuilder(blobName).build();
|
||||
MultipartUpload mpu = requiredMD5BlobStore.initiateMultipartUpload(
|
||||
containerName, blob.getMetadata(), PutOptions.NONE);
|
||||
byte[] data = new byte[1];
|
||||
Payload payload = Payloads.newPayload(data);
|
||||
|
||||
try {
|
||||
requiredMD5BlobStore.uploadMultipartPart(mpu, 1, payload);
|
||||
Fail.failBecauseExceptionWasNotThrown(
|
||||
IllegalArgumentException.class);
|
||||
} catch (IllegalArgumentException iae) {
|
||||
// expected
|
||||
}
|
||||
|
||||
payload.getContentMetadata().setContentMD5(
|
||||
Hashing.md5().hashBytes(data));
|
||||
requiredMD5BlobStore.uploadMultipartPart(mpu, 1, payload);
|
||||
}
|
||||
|
||||
private static String createRandomContainerName() {
|
||||
return "container-" + new Random().nextInt(Integer.MAX_VALUE);
|
||||
}
|
||||
}
|
||||
Ładowanie…
Reference in New Issue