diff --git a/src/main/java/org/gaul/s3proxy/Main.java b/src/main/java/org/gaul/s3proxy/Main.java index af7604e..304db21 100644 --- a/src/main/java/org/gaul/s3proxy/Main.java +++ b/src/main/java/org/gaul/s3proxy/Main.java @@ -171,6 +171,13 @@ public final class Main { delay, TimeUnit.SECONDS, probability); } + String nullBlobStore = properties.getProperty( + S3ProxyConstants.PROPERTY_NULL_BLOBSTORE); + if ("true".equalsIgnoreCase(nullBlobStore)) { + System.err.println("Using null storage backend"); + blobStore = NullBlobStore.newNullBlobStore(blobStore); + } + S3Proxy s3Proxy; try { S3Proxy.Builder s3ProxyBuilder = S3Proxy.builder() diff --git a/src/main/java/org/gaul/s3proxy/NullBlobStore.java b/src/main/java/org/gaul/s3proxy/NullBlobStore.java new file mode 100644 index 0000000..b841cc4 --- /dev/null +++ b/src/main/java/org/gaul/s3proxy/NullBlobStore.java @@ -0,0 +1,179 @@ +/* + * Copyright 2014-2017 Andrew Gaul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gaul.s3proxy; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +import com.google.common.collect.ImmutableList; +import com.google.common.hash.HashCode; +import com.google.common.io.ByteSource; +import com.google.common.io.ByteStreams; +import com.google.common.primitives.Longs; + +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.domain.BlobMetadata; +import org.jclouds.blobstore.domain.MultipartPart; +import org.jclouds.blobstore.domain.MultipartUpload; +import org.jclouds.blobstore.options.GetOptions; +import org.jclouds.blobstore.options.PutOptions; +import org.jclouds.blobstore.util.ForwardingBlobStore; +import org.jclouds.io.Payload; +import org.jclouds.io.payloads.ByteSourcePayload; + +final class NullBlobStore extends ForwardingBlobStore { + private NullBlobStore(BlobStore blobStore) { + super(blobStore); + } + + static BlobStore newNullBlobStore(BlobStore blobStore) { + return new NullBlobStore(blobStore); + } + + @Override + public BlobMetadata blobMetadata(String container, String name) { + Blob blob = getBlob(container, name); + if (blob == null) { + return null; + } + return blob.getMetadata(); + } + + @Override + public Blob getBlob(String container, String name) { + return getBlob(container, name, GetOptions.NONE); + } + + @Override + public Blob getBlob(String container, String name, GetOptions options) { + Blob blob = super.getBlob(container, name, options); + if (blob == null) { + return null; + } + + byte[] array; + try (InputStream is = blob.getPayload().openStream()) { + array = ByteStreams.toByteArray(is); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + + long length = Longs.fromByteArray(array); + ByteSourcePayload payload = new ByteSourcePayload( + new NullByteSource().slice(0, length)); + payload.setContentMetadata(blob.getPayload().getContentMetadata()); + payload.getContentMetadata().setContentLength(length); + payload.getContentMetadata().setContentMD5((HashCode) null); + blob.setPayload(payload); + return blob; + } + + @Override + public String putBlob(String containerName, Blob blob) { + return putBlob(containerName, blob, PutOptions.NONE); + } + + @Override + public String putBlob(String containerName, Blob blob, + PutOptions options) { + long length; + try (InputStream is = blob.getPayload().openStream()) { + length = ByteStreams.copy(is, ByteStreams.nullOutputStream()); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + + byte[] array = Longs.toByteArray(length); + ByteSourcePayload payload = new ByteSourcePayload( + ByteSource.wrap(array)); + payload.setContentMetadata(blob.getPayload().getContentMetadata()); + payload.getContentMetadata().setContentLength((long) array.length); + payload.getContentMetadata().setContentMD5((HashCode) null); + blob.setPayload(payload); + + return super.putBlob(containerName, blob, options); + } + + @Override + public String completeMultipartUpload(final MultipartUpload mpu, + final List parts) { + long length = 0; + for (MultipartPart part : parts) { + length += part.partSize(); + } + + byte[] array = Longs.toByteArray(length); + ByteSourcePayload payload = new ByteSourcePayload( + ByteSource.wrap(array)); + + super.abortMultipartUpload(mpu); + + MultipartPart part = delegate().uploadMultipartPart(mpu, 1, payload); + + return delegate().completeMultipartUpload(mpu, ImmutableList.of(part)); + } + + @Override + public MultipartPart uploadMultipartPart(MultipartUpload mpu, + int partNumber, Payload payload) { + long length; + try (InputStream is = payload.openStream()) { + length = ByteStreams.copy(is, ByteStreams.nullOutputStream()); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + + byte[] array = Longs.toByteArray(length); + ByteSourcePayload newPayload = new ByteSourcePayload( + ByteSource.wrap(array)); + newPayload.setContentMetadata(payload.getContentMetadata()); + newPayload.getContentMetadata().setContentLength((long) array.length); + newPayload.getContentMetadata().setContentMD5((HashCode) null); + + MultipartPart part = super.uploadMultipartPart(mpu, partNumber, + newPayload); + return MultipartPart.create(part.partNumber(), length, part.partETag(), + part.lastModified()); + } + + // Cannot read parts to get the embedded size so return zero instead. + @Override + public List listMultipartUpload(MultipartUpload mpu) { + ImmutableList.Builder builder = ImmutableList.builder(); + for (MultipartPart part : super.listMultipartUpload(mpu)) { + builder.add(MultipartPart.create(part.partNumber(), 0, + part.partETag(), part.lastModified())); + } + return builder.build(); + } + + private static final class NullByteSource extends ByteSource { + @Override + public InputStream openStream() throws IOException { + return new NullInputStream(); + } + } + + private static final class NullInputStream extends InputStream { + @Override + public int read() throws IOException { + return 0; + } + } +} diff --git a/src/main/java/org/gaul/s3proxy/S3ProxyConstants.java b/src/main/java/org/gaul/s3proxy/S3ProxyConstants.java index 5e081e8..1938f0a 100644 --- a/src/main/java/org/gaul/s3proxy/S3ProxyConstants.java +++ b/src/main/java/org/gaul/s3proxy/S3ProxyConstants.java @@ -64,6 +64,9 @@ public final class S3ProxyConstants { /** Probability of eventual consistency, between 0.0 and 1.0. */ public static final String PROPERTY_EVENTUAL_CONSISTENCY_PROBABILITY = "s3proxy.eventual-consistency.probability"; + /** Discard object data. */ + public static final String PROPERTY_NULL_BLOBSTORE = + "s3proxy.null-blobstore"; static final String PROPERTY_ALT_JCLOUDS_PREFIX = "alt."; diff --git a/src/test/java/org/gaul/s3proxy/NullBlobStoreTest.java b/src/test/java/org/gaul/s3proxy/NullBlobStoreTest.java new file mode 100644 index 0000000..5e12e82 --- /dev/null +++ b/src/test/java/org/gaul/s3proxy/NullBlobStoreTest.java @@ -0,0 +1,195 @@ +/* + * Copyright 2014-2017 Andrew Gaul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gaul.s3proxy; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.Random; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.hash.Hashing; +import com.google.common.io.ByteSource; +import com.google.common.io.ByteStreams; +import com.google.common.net.MediaType; +import com.google.inject.Module; + +import org.jclouds.ContextBuilder; +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.domain.BlobMetadata; +import org.jclouds.blobstore.domain.MultipartPart; +import org.jclouds.blobstore.domain.MultipartUpload; +import org.jclouds.blobstore.options.PutOptions; +import org.jclouds.io.ContentMetadata; +import org.jclouds.io.Payload; +import org.jclouds.io.Payloads; +import org.jclouds.logging.slf4j.config.SLF4JLoggingModule; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public final class NullBlobStoreTest { + private static final ByteSource BYTE_SOURCE = + TestUtils.randomByteSource().slice(0, 1024); + private BlobStoreContext context; + private BlobStore blobStore; + private String containerName; + private BlobStore nullBlobStore; + + @Before + public void setUp() throws Exception { + containerName = createRandomContainerName(); + + context = ContextBuilder + .newBuilder("transient") + .credentials("identity", "credential") + .modules(ImmutableList.of(new SLF4JLoggingModule())) + .build(BlobStoreContext.class); + blobStore = context.getBlobStore(); + blobStore.createContainerInLocation(null, containerName); + + nullBlobStore = NullBlobStore.newNullBlobStore(blobStore); + } + + @After + public void tearDown() throws Exception { + if (context != null) { + blobStore.deleteContainer(containerName); + context.close(); + } + } + + @Test + public void testCreateBlobGetBlob() throws Exception { + String blobName = createRandomBlobName(); + Blob blob = makeBlob(nullBlobStore, blobName); + nullBlobStore.putBlob(containerName, blob); + + blob = nullBlobStore.getBlob(containerName, blobName); + validateBlobMetadata(blob.getMetadata(), BYTE_SOURCE); + + // content differs, only compare length + try (InputStream actual = blob.getPayload().openStream(); + InputStream expected = BYTE_SOURCE.openStream()) { + long actualLength = ByteStreams.copy(actual, + ByteStreams.nullOutputStream()); + long expectedLength = ByteStreams.copy(expected, + ByteStreams.nullOutputStream()); + assertThat(actualLength).isEqualTo(expectedLength); + } + } + + @Test + public void testCreateBlobBlobMetadata() throws Exception { + String blobName = createRandomBlobName(); + Blob blob = makeBlob(nullBlobStore, blobName); + nullBlobStore.putBlob(containerName, blob); + BlobMetadata metadata = nullBlobStore.blobMetadata(containerName, + blobName); + validateBlobMetadata(metadata, BYTE_SOURCE); + } + + @Test + public void testCreateMultipartBlobGetBlob() throws Exception { + String blobName = "multipart-upload"; + BlobMetadata blobMetadata = makeBlob(nullBlobStore, blobName) + .getMetadata(); + MultipartUpload mpu = nullBlobStore.initiateMultipartUpload( + containerName, blobMetadata, new PutOptions()); + + ByteSource byteSource = TestUtils.randomByteSource().slice( + 0, nullBlobStore.getMinimumMultipartPartSize() + 1); + ByteSource byteSource1 = byteSource.slice( + 0, nullBlobStore.getMinimumMultipartPartSize()); + ByteSource byteSource2 = byteSource.slice( + nullBlobStore.getMinimumMultipartPartSize(), 1); + Payload payload1 = Payloads.newByteSourcePayload(byteSource1); + Payload payload2 = Payloads.newByteSourcePayload(byteSource2); + payload1.getContentMetadata().setContentLength(byteSource1.size()); + payload2.getContentMetadata().setContentLength(byteSource2.size()); + MultipartPart part1 = nullBlobStore.uploadMultipartPart(mpu, 1, + payload1); + MultipartPart part2 = nullBlobStore.uploadMultipartPart(mpu, 2, + payload2); + + List parts = nullBlobStore.listMultipartUpload(mpu); + assertThat(parts.get(0).partNumber()).isEqualTo(1); + assertThat(parts.get(0).partSize()).isZero(); + assertThat(parts.get(0).partETag()).isEqualTo(part1.partETag()); + assertThat(parts.get(1).partNumber()).isEqualTo(2); + assertThat(parts.get(1).partSize()).isZero(); + assertThat(parts.get(1).partETag()).isEqualTo(part2.partETag()); + + nullBlobStore.completeMultipartUpload(mpu, ImmutableList.of(part1, + part2)); + + Blob newBlob = nullBlobStore.getBlob(containerName, blobName); + validateBlobMetadata(newBlob.getMetadata(), byteSource); + + // content differs, only compare length + try (InputStream actual = newBlob.getPayload().openStream(); + InputStream expected = byteSource.openStream()) { + long actualLength = ByteStreams.copy(actual, + ByteStreams.nullOutputStream()); + long expectedLength = ByteStreams.copy(expected, + ByteStreams.nullOutputStream()); + assertThat(actualLength).isEqualTo(expectedLength); + } + } + + private static String createRandomContainerName() { + return "container-" + new Random().nextInt(Integer.MAX_VALUE); + } + + private static String createRandomBlobName() { + return "blob-" + new Random().nextInt(Integer.MAX_VALUE); + } + + private static Blob makeBlob(BlobStore blobStore, String blobName) + throws IOException { + return blobStore.blobBuilder(blobName) + .payload(BYTE_SOURCE) + .contentDisposition("attachment; filename=foo.mp4") + .contentEncoding("compress") + .contentLength(BYTE_SOURCE.size()) + .contentType(MediaType.MP4_AUDIO) + .contentMD5(BYTE_SOURCE.hash(Hashing.md5())) + .userMetadata(ImmutableMap.of("key", "value")) + .build(); + } + + private static void validateBlobMetadata(BlobMetadata metadata, + ByteSource byteSource) throws IOException { + assertThat(metadata).isNotNull(); + + ContentMetadata contentMetadata = metadata.getContentMetadata(); + assertThat(contentMetadata.getContentDisposition()) + .isEqualTo("attachment; filename=foo.mp4"); + assertThat(contentMetadata.getContentEncoding()) + .isEqualTo("compress"); + assertThat(contentMetadata.getContentType()) + .isEqualTo(MediaType.MP4_AUDIO.toString()); + + assertThat(metadata.getUserMetadata()) + .isEqualTo(ImmutableMap.of("key", "value")); + } +}