parts, int skipParts,
+ long skipPartBytes)
+ throws IOException {
+ super(is);
+ in = is;
+ this.parts = parts;
+ this.key = key;
+
+ PartPadding partPadding = parts.get(parts.size() - skipParts);
+
+ try {
+ // init the cipher
+ cipher = Cipher.getInstance(Constants.AES_CIPHER);
+ cipher.init(Cipher.DECRYPT_MODE, key, partPadding.getIv());
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ // set the part to begin with
+ part = parts.size() - skipParts;
+
+ // adjust part size due to offset
+ partBytesRemain = parts.get(part).getSize() - skipPartBytes;
+ }
+
+ /**
+ * Ensure obuffer is big enough for the next update or doFinal
+ * operation, given the input length inLen (in bytes)
+ * The ostart and ofinish indices are reset to 0.
+ *
+ * @param inLen the input length (in bytes)
+ */
+ private void ensureCapacity(int inLen) {
+ int minLen = cipher.getOutputSize(inLen);
+ if (obuffer == null || obuffer.length < minLen) {
+ obuffer = new byte[minLen];
+ }
+ ostart = 0;
+ ofinish = 0;
+ }
+
+ /**
+ * Private convenience function, read in data from the underlying
+ * input stream and process them with cipher. This method is called
+ * when the processed bytes inside obuffer has been exhausted.
+ *
+ * Entry condition: ostart = ofinish
+ *
+ * Exit condition: ostart = 0 AND ostart <= ofinish
+ *
+ * return (ofinish-ostart) (we have this many bytes for you)
+ * return 0 (no data now, but could have more later)
+ * return -1 (absolutely no more data)
+ *
+ * Note: Exceptions are only thrown after the stream is completely read.
+ * For AEAD ciphers a read() of any length will internally cause the
+ * whole stream to be read fully and verify the authentication tag before
+ * returning decrypted data or exceptions.
+ */
+ private int getMoreData() throws IOException {
+ if (done) {
+ return -1;
+ }
+
+ int readLimit = ibuffer.length;
+ if (partBytesRemain < ibuffer.length) {
+ readLimit = (int) partBytesRemain;
+ }
+
+ int readin;
+ if (partBytesRemain == 0) {
+ readin = -1;
+ } else {
+ readin = in.read(ibuffer, 0, readLimit);
+ }
+
+ if (readin == -1) {
+ ensureCapacity(0);
+ try {
+ ofinish = cipher.doFinal(obuffer, 0);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ int nextPart = part - 1;
+ if (parts.containsKey(nextPart)) {
+ // reset cipher
+ PartPadding partPadding = parts.get(nextPart);
+ try {
+ cipher.init(Cipher.DECRYPT_MODE, key, partPadding.getIv());
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ // update to the next part
+ part = nextPart;
+
+ // update the remaining bytes of the next part
+ partBytesRemain = parts.get(nextPart).getSize();
+
+ IOUtils.skip(in, Constants.PADDING_BLOCK_SIZE);
+
+ return ofinish;
+ } else {
+ done = true;
+ if (ofinish == 0) {
+ return -1;
+ } else {
+ return ofinish;
+ }
+ }
+ }
+ ensureCapacity(readin);
+ try {
+ ofinish = cipher.update(ibuffer, 0, readin, obuffer, ostart);
+ } catch (ShortBufferException e) {
+ throw new IOException(e);
+ }
+
+ partBytesRemain = partBytesRemain - readin;
+ return ofinish;
+ }
+
+ /**
+ * Reads the next byte of data from this input stream. The value
+ * byte is returned as an int in the range
+ * 0 to 255. If no byte is available
+ * because the end of the stream has been reached, the value
+ * -1 is returned. This method blocks until input data
+ * is available, the end of the stream is detected, or an exception
+ * is thrown.
+ *
+ * @return the next byte of data, or -1 if the end of the
+ * stream is reached.
+ * @throws IOException if an I/O error occurs.
+ */
+ @Override
+ public final int read() throws IOException {
+ if (ostart >= ofinish) {
+ // we loop for new data as the spec says we are blocking
+ int i = 0;
+ while (i == 0) {
+ i = getMoreData();
+ }
+ if (i == -1) {
+ return -1;
+ }
+ }
+ return (int) obuffer[ostart++] & 0xff;
+ }
+
+ /**
+ * Reads up to b.length bytes of data from this input
+ * stream into an array of bytes.
+ *
+ * The read method of InputStream calls
+ * the read method of three arguments with the arguments
+ * b, 0, and b.length.
+ *
+ * @param b the buffer into which the data is read.
+ * @return the total number of bytes read into the buffer, or
+ * -1 is there is no more data because the end of
+ * the stream has been reached.
+ * @throws IOException if an I/O error occurs.
+ * @see java.io.InputStream#read(byte[], int, int)
+ */
+ @Override
+ public final int read(byte[] b) throws IOException {
+ return read(b, 0, b.length);
+ }
+
+ /**
+ * Reads up to len bytes of data from this input stream
+ * into an array of bytes. This method blocks until some input is
+ * available. If the first argument is null, up to
+ * len bytes are read and discarded.
+ *
+ * @param b the buffer into which the data is read.
+ * @param off the start offset in the destination array
+ * buf
+ * @param len the maximum number of bytes read.
+ * @return the total number of bytes read into the buffer, or
+ * -1 if there is no more data because the end of
+ * the stream has been reached.
+ * @throws IOException if an I/O error occurs.
+ * @see java.io.InputStream#read()
+ */
+ @Override
+ public final int read(byte[] b, int off, int len) throws IOException {
+ if (ostart >= ofinish) {
+ // we loop for new data as the spec says we are blocking
+ int i = 0;
+ while (i == 0) {
+ i = getMoreData();
+ }
+ if (i == -1) {
+ return -1;
+ }
+ }
+ if (len <= 0) {
+ return 0;
+ }
+ int available = ofinish - ostart;
+ if (len < available) {
+ available = len;
+ }
+ if (b != null) {
+ System.arraycopy(obuffer, ostart, b, off, available);
+ }
+ ostart = ostart + available;
+ return available;
+ }
+
+ /**
+ * Skips n bytes of input from the bytes that can be read
+ * from this input stream without blocking.
+ *
+ *
Fewer bytes than requested might be skipped.
+ * The actual number of bytes skipped is equal to n or
+ * the result of a call to
+ * {@link #available() available},
+ * whichever is smaller.
+ * If n is less than zero, no bytes are skipped.
+ *
+ *
The actual number of bytes skipped is returned.
+ *
+ * @param n the number of bytes to be skipped.
+ * @return the actual number of bytes skipped.
+ * @throws IOException if an I/O error occurs.
+ */
+ @Override
+ public final long skip(long n) throws IOException {
+ int available = ofinish - ostart;
+ if (n > available) {
+ n = available;
+ }
+ if (n < 0) {
+ return 0;
+ }
+ ostart += n;
+ return n;
+ }
+
+ /**
+ * Returns the number of bytes that can be read from this input
+ * stream without blocking. The available method of
+ * InputStream returns 0. This method
+ * should be overridden by subclasses.
+ *
+ * @return the number of bytes that can be read from this input stream
+ * without blocking.
+ */
+ @Override
+ public final int available() {
+ return ofinish - ostart;
+ }
+
+ /**
+ * Closes this input stream and releases any system resources
+ * associated with the stream.
+ *
+ * The close method of CipherInputStream
+ * calls the close method of its underlying input
+ * stream.
+ *
+ * @throws IOException if an I/O error occurs.
+ */
+ @Override
+ public final void close() throws IOException {
+ if (closed) {
+ return;
+ }
+ closed = true;
+ in.close();
+
+ // Throw away the unprocessed data and throw no crypto exceptions.
+ // AEAD ciphers are fully readed before closing. Any authentication
+ // exceptions would occur while reading.
+ if (!done) {
+ ensureCapacity(0);
+ try {
+ cipher.doFinal(obuffer, 0);
+ } catch (Exception e) {
+ // Catch exceptions as the rest of the stream is unused.
+ }
+ }
+ obuffer = null;
+ }
+
+ /**
+ * Tests if this input stream supports the mark
+ * and reset methods, which it does not.
+ *
+ * @return false, since this class does not support the
+ * mark and reset methods.
+ * @see java.io.InputStream#mark(int)
+ * @see java.io.InputStream#reset()
+ */
+ @Override
+ public final boolean markSupported() {
+ return false;
+ }
+}
diff --git a/src/main/java/org/gaul/s3proxy/crypto/Encryption.java b/src/main/java/org/gaul/s3proxy/crypto/Encryption.java
new file mode 100644
index 0000000..3aafa3e
--- /dev/null
+++ b/src/main/java/org/gaul/s3proxy/crypto/Encryption.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2014-2021 Andrew Gaul
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.gaul.s3proxy.crypto;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.SecureRandom;
+
+import javax.annotation.concurrent.ThreadSafe;
+import javax.crypto.Cipher;
+import javax.crypto.CipherInputStream;
+import javax.crypto.spec.IvParameterSpec;
+import javax.crypto.spec.SecretKeySpec;
+
+@ThreadSafe
+public class Encryption {
+ private final InputStream cis;
+ private final IvParameterSpec iv;
+ private final int part;
+
+ public Encryption(SecretKeySpec key, InputStream isRaw, int partNumber)
+ throws Exception {
+ iv = generateIV();
+
+ Cipher cipher = Cipher.getInstance(Constants.AES_CIPHER);
+ cipher.init(Cipher.ENCRYPT_MODE, key, iv);
+ cis = new CipherInputStream(isRaw, cipher);
+ part = partNumber;
+ }
+
+ public final InputStream openStream() throws IOException {
+ return new EncryptionInputStream(cis, part, iv);
+ }
+
+ private IvParameterSpec generateIV() {
+ byte[] iv = new byte[Constants.AES_BLOCK_SIZE];
+ SecureRandom randomSecureRandom = new SecureRandom();
+ randomSecureRandom.nextBytes(iv);
+
+ return new IvParameterSpec(iv);
+ }
+}
diff --git a/src/main/java/org/gaul/s3proxy/crypto/EncryptionInputStream.java b/src/main/java/org/gaul/s3proxy/crypto/EncryptionInputStream.java
new file mode 100644
index 0000000..7a67eec
--- /dev/null
+++ b/src/main/java/org/gaul/s3proxy/crypto/EncryptionInputStream.java
@@ -0,0 +1,126 @@
+/*
+ * Copyright 2014-2021 Andrew Gaul
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.gaul.s3proxy.crypto;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+
+import javax.crypto.spec.IvParameterSpec;
+
+public class EncryptionInputStream extends InputStream {
+
+ private final int part;
+ private final IvParameterSpec iv;
+ private boolean hasPadding;
+ private long size;
+ private InputStream in;
+
+ public EncryptionInputStream(InputStream in, int part,
+ IvParameterSpec iv) {
+ this.part = part;
+ this.iv = iv;
+ this.in = in;
+ }
+
+ // Padding (64 byte)
+ // Delimiter (8 byte)
+ // IV (16 byte)
+ // Part (4 byte)
+ // Size (8 byte)
+ // Version (2 byte)
+ // Reserved (26 byte)
+ final void padding() throws IOException {
+ if (in != null) {
+ in.close();
+ }
+
+ if (!hasPadding) {
+ ByteBuffer bb = ByteBuffer.allocate(Constants.PADDING_BLOCK_SIZE);
+ bb.put(Constants.DELIMITER);
+ bb.put(iv.getIV());
+ bb.putInt(part);
+ bb.putLong(size);
+ bb.putShort(Constants.VERSION);
+
+ in = new ByteArrayInputStream(bb.array());
+ hasPadding = true;
+ } else {
+ in = null;
+ }
+ }
+
+ public final int available() throws IOException {
+ if (in == null) {
+ return 0; // no way to signal EOF from available()
+ }
+ return in.available();
+ }
+
+ public final int read() throws IOException {
+ while (in != null) {
+ int c = in.read();
+ if (c != -1) {
+ size++;
+ return c;
+ }
+ padding();
+ }
+ return -1;
+ }
+
+ public final int read(byte[] b, int off, int len) throws IOException {
+ if (in == null) {
+ return -1;
+ } else if (b == null) {
+ throw new NullPointerException();
+ } else if (off < 0 || len < 0 || len > b.length - off) {
+ throw new IndexOutOfBoundsException();
+ } else if (len == 0) {
+ return 0;
+ }
+ do {
+ int n = in.read(b, off, len);
+ if (n > 0) {
+ size = size + n;
+ return n;
+ }
+ padding();
+ } while (in != null);
+ return -1;
+ }
+
+ public final void close() throws IOException {
+ IOException ioe = null;
+ while (in != null) {
+ try {
+ in.close();
+ } catch (IOException e) {
+ if (ioe == null) {
+ ioe = e;
+ } else {
+ ioe.addSuppressed(e);
+ }
+ }
+ padding();
+ }
+ if (ioe != null) {
+ throw ioe;
+ }
+ }
+}
diff --git a/src/main/java/org/gaul/s3proxy/crypto/PartPadding.java b/src/main/java/org/gaul/s3proxy/crypto/PartPadding.java
new file mode 100644
index 0000000..983ae27
--- /dev/null
+++ b/src/main/java/org/gaul/s3proxy/crypto/PartPadding.java
@@ -0,0 +1,88 @@
+/*
+ * Copyright 2014-2021 Andrew Gaul
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.gaul.s3proxy.crypto;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+
+import javax.crypto.spec.IvParameterSpec;
+
+import org.apache.commons.io.IOUtils;
+import org.jclouds.blobstore.domain.Blob;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PartPadding {
+ private static final Logger logger =
+ LoggerFactory.getLogger(PartPadding.class);
+
+ private String delimiter;
+ private IvParameterSpec iv;
+ private int part;
+ private long size;
+ private short version;
+
+ public static PartPadding readPartPaddingFromBlob(Blob blob)
+ throws IOException {
+ PartPadding partPadding = new PartPadding();
+
+ InputStream is = blob.getPayload().openStream();
+
+ byte[] paddingBytes = IOUtils.toByteArray(is);
+ ByteBuffer bb = ByteBuffer.wrap(paddingBytes);
+
+ byte[] delimiterBytes = new byte[Constants.PADDING_DELIMITER_LENGTH];
+ bb.get(delimiterBytes);
+ partPadding.delimiter =
+ new String(delimiterBytes, StandardCharsets.UTF_8);
+
+ byte[] ivBytes = new byte[Constants.PADDING_IV_LENGTH];
+ bb.get(ivBytes);
+ partPadding.iv = new IvParameterSpec(ivBytes);
+
+ partPadding.part = bb.getInt();
+ partPadding.size = bb.getLong();
+ partPadding.version = bb.getShort();
+
+ logger.debug("delimiter {}", partPadding.delimiter);
+ logger.debug("iv {}", Arrays.toString(ivBytes));
+ logger.debug("part {}", partPadding.part);
+ logger.debug("size {}", partPadding.size);
+ logger.debug("version {}", partPadding.version);
+
+ return partPadding;
+ }
+
+ public final String getDelimiter() {
+ return delimiter;
+ }
+
+ public final IvParameterSpec getIv() {
+ return iv;
+ }
+
+ public final int getPart() {
+ return part;
+ }
+
+ public final long getSize() {
+ return size;
+ }
+}
diff --git a/src/main/resources/run-docker-container.sh b/src/main/resources/run-docker-container.sh
index a162347..3b3cb71 100755
--- a/src/main/resources/run-docker-container.sh
+++ b/src/main/resources/run-docker-container.sh
@@ -12,6 +12,9 @@ exec java \
-Ds3proxy.cors-allow-methods="${S3PROXY_CORS_ALLOW_METHODS}" \
-Ds3proxy.cors-allow-headers="${S3PROXY_CORS_ALLOW_HEADERS}" \
-Ds3proxy.ignore-unknown-headers="${S3PROXY_IGNORE_UNKNOWN_HEADERS}" \
+ -Ds3proxy.encrypted-blobstore="${S3PROXY_ENCRYPTED_BLOBSTORE}" \
+ -Ds3proxy.encrypted-blobstore-password="${S3PROXY_ENCRYPTED_BLOBSTORE_PASSWORD}" \
+ -Ds3proxy.encrypted-blobstore-salt="${S3PROXY_ENCRYPTED_BLOBSTORE_SALT}" \
-Djclouds.provider="${JCLOUDS_PROVIDER}" \
-Djclouds.identity="${JCLOUDS_IDENTITY}" \
-Djclouds.credential="${JCLOUDS_CREDENTIAL}" \
diff --git a/src/test/java/org/gaul/s3proxy/EncryptedBlobStoreLiveTest.java b/src/test/java/org/gaul/s3proxy/EncryptedBlobStoreLiveTest.java
new file mode 100644
index 0000000..da328a9
--- /dev/null
+++ b/src/test/java/org/gaul/s3proxy/EncryptedBlobStoreLiveTest.java
@@ -0,0 +1,282 @@
+/*
+ * Copyright 2014-2021 Andrew Gaul
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.gaul.s3proxy;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.Properties;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.io.ByteSource;
+import com.google.common.util.concurrent.Uninterruptibles;
+
+import org.assertj.core.api.Fail;
+import org.gaul.s3proxy.crypto.Constants;
+import org.jclouds.aws.AWSResponseException;
+import org.jclouds.blobstore.BlobStoreContext;
+import org.jclouds.blobstore.domain.PageSet;
+import org.jclouds.blobstore.domain.StorageMetadata;
+import org.jclouds.blobstore.options.ListContainerOptions;
+import org.jclouds.http.options.GetOptions;
+import org.jclouds.io.Payload;
+import org.jclouds.io.Payloads;
+import org.jclouds.s3.S3ClientLiveTest;
+import org.jclouds.s3.domain.ListMultipartUploadsResponse;
+import org.jclouds.s3.domain.ObjectMetadataBuilder;
+import org.jclouds.s3.domain.S3Object;
+import org.jclouds.s3.reference.S3Constants;
+import org.testng.SkipException;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.Test;
+
+@SuppressWarnings("UnstableApiUsage")
+@Test(testName = "EncryptedBlobStoreLiveTest")
+public final class EncryptedBlobStoreLiveTest extends S3ClientLiveTest {
+ private static final int AWAIT_CONSISTENCY_TIMEOUT_SECONDS =
+ Integer.parseInt(
+ System.getProperty(
+ "test.blobstore.await-consistency-timeout-seconds",
+ "0"));
+ private static final long MINIMUM_MULTIPART_SIZE = 5 * 1024 * 1024;
+
+ private S3Proxy s3Proxy;
+ private BlobStoreContext context;
+
+ @AfterClass
+ public void tearDown() throws Exception {
+ s3Proxy.stop();
+ context.close();
+ }
+
+ @Override
+ protected void awaitConsistency() {
+ Uninterruptibles.sleepUninterruptibly(
+ AWAIT_CONSISTENCY_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ }
+
+ @Override
+ protected Properties setupProperties() {
+ TestUtils.S3ProxyLaunchInfo info;
+ try {
+ info = TestUtils.startS3Proxy("s3proxy-encryption.conf");
+ s3Proxy = info.getS3Proxy();
+ context = info.getBlobStore().getContext();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ Properties props = super.setupProperties();
+ props.setProperty(org.jclouds.Constants.PROPERTY_IDENTITY,
+ info.getS3Identity());
+ props.setProperty(org.jclouds.Constants.PROPERTY_CREDENTIAL,
+ info.getS3Credential());
+ props.setProperty(org.jclouds.Constants.PROPERTY_ENDPOINT,
+ info.getEndpoint().toString() + info.getServicePath());
+ props.setProperty(org.jclouds.Constants.PROPERTY_STRIP_EXPECT_HEADER,
+ "true");
+ props.setProperty(S3Constants.PROPERTY_S3_SERVICE_PATH,
+ info.getServicePath());
+ endpoint = info.getEndpoint().toString() + info.getServicePath();
+ return props;
+ }
+
+ @Test
+ public void testOneCharAndCopy() throws InterruptedException {
+ String blobName = TestUtils.createRandomBlobName();
+ String containerName = this.getContainerName();
+
+ S3Object object = this.getApi().newS3Object();
+ object.getMetadata().setKey(blobName);
+ object.setPayload("1");
+ this.getApi().putObject(containerName, object);
+
+ object = this.getApi().getObject(containerName, blobName);
+ assertThat(object.getMetadata().getContentMetadata()
+ .getContentLength()).isEqualTo(1L);
+
+ PageSet
+ list = view.getBlobStore().list(containerName);
+ assertThat(list).hasSize(1);
+
+ StorageMetadata md = list.iterator().next();
+ assertThat(md.getName()).isEqualTo(blobName);
+ assertThat(md.getSize()).isEqualTo(1L);
+
+ this.getApi().copyObject(containerName, blobName, containerName,
+ blobName + "-copy");
+ list = view.getBlobStore().list(containerName);
+ assertThat(list).hasSize(2);
+
+ for (StorageMetadata sm : list) {
+ assertThat(sm.getSize()).isEqualTo(1L);
+ assertThat(sm.getName()).doesNotContain(
+ Constants.S3_ENC_SUFFIX);
+ }
+
+ ListContainerOptions lco = new ListContainerOptions();
+ lco.maxResults(1);
+ list = view.getBlobStore().list(containerName, lco);
+ assertThat(list).hasSize(1);
+ assertThat(list.getNextMarker()).doesNotContain(
+ Constants.S3_ENC_SUFFIX);
+ }
+
+ @Test
+ public void testPartialContent() throws InterruptedException, IOException {
+ String blobName = TestUtils.createRandomBlobName();
+ String containerName = this.getContainerName();
+ String content = "123456789A123456789B123456";
+
+ S3Object object = this.getApi().newS3Object();
+ object.getMetadata().setKey(blobName);
+ object.setPayload(content);
+ this.getApi().putObject(containerName, object);
+
+ // get only 20 bytes
+ GetOptions options = new GetOptions();
+ options.range(0, 19);
+ object = this.getApi().getObject(containerName, blobName, options);
+
+ InputStreamReader r =
+ new InputStreamReader(object.getPayload().openStream());
+ BufferedReader reader = new BufferedReader(r);
+ String partialContent = reader.lines().collect(Collectors.joining());
+ assertThat(partialContent).isEqualTo(content.substring(0, 20));
+ }
+
+ @Test
+ public void testMultipart() throws InterruptedException, IOException {
+ String blobName = TestUtils.createRandomBlobName();
+ String containerName = this.getContainerName();
+
+ // 15mb of data
+ ByteSource byteSource = TestUtils.randomByteSource().slice(
+ 0, MINIMUM_MULTIPART_SIZE * 3);
+
+ // first 2 parts with 6mb and last part with 3mb
+ long partSize = 6 * 1024 * 1024;
+ long lastPartSize = 3 * 1024 * 1024;
+ ByteSource byteSource1 = byteSource.slice(0, partSize);
+ ByteSource byteSource2 = byteSource.slice(partSize, partSize);
+ ByteSource byteSource3 = byteSource.slice(partSize * 2,
+ lastPartSize);
+
+ String uploadId = this.getApi().initiateMultipartUpload(containerName,
+ ObjectMetadataBuilder.create().key(blobName).build());
+ assertThat(this.getApi().listMultipartPartsFull(containerName,
+ blobName, uploadId)).isEmpty();
+
+ ListMultipartUploadsResponse
+ response = this.getApi()
+ .listMultipartUploads(containerName, null, null, null, blobName,
+ null);
+ assertThat(response.uploads()).hasSize(1);
+
+ Payload part1 =
+ Payloads.newInputStreamPayload(byteSource1.openStream());
+ part1.getContentMetadata().setContentLength(byteSource1.size());
+ Payload part2 =
+ Payloads.newInputStreamPayload(byteSource2.openStream());
+ part2.getContentMetadata().setContentLength(byteSource2.size());
+ Payload part3 =
+ Payloads.newInputStreamPayload(byteSource3.openStream());
+ part3.getContentMetadata().setContentLength(byteSource3.size());
+
+ String eTagOf1 = this.getApi()
+ .uploadPart(containerName, blobName, 1, uploadId, part1);
+ String eTagOf2 = this.getApi()
+ .uploadPart(containerName, blobName, 2, uploadId, part2);
+ String eTagOf3 = this.getApi()
+ .uploadPart(containerName, blobName, 3, uploadId, part3);
+
+ this.getApi().completeMultipartUpload(containerName, blobName, uploadId,
+ ImmutableMap.of(1, eTagOf1, 2, eTagOf2, 3, eTagOf3));
+ S3Object object = this.getApi().getObject(containerName, blobName);
+
+ try (InputStream actual = object.getPayload().openStream();
+ InputStream expected = byteSource.openStream()) {
+ assertThat(actual).hasContentEqualTo(expected);
+ }
+
+ // get a 5mb slice that overlap parts
+ long partialStart = 5 * 1024 * 1024;
+ ByteSource partialContent =
+ byteSource.slice(partialStart, partialStart);
+
+ GetOptions options = new GetOptions();
+ options.range(partialStart, (partialStart * 2) - 1);
+ object = this.getApi().getObject(containerName, blobName, options);
+
+ try (InputStream actual = object.getPayload().openStream();
+ InputStream expected = partialContent.openStream()) {
+ assertThat(actual).hasContentEqualTo(expected);
+ }
+ }
+
+ @Override
+ public void testMultipartSynchronously() {
+ throw new SkipException("list multipart synchronously not supported");
+ }
+
+ @Override
+ @Test
+ public void testUpdateObjectACL() throws InterruptedException,
+ ExecutionException, TimeoutException, IOException {
+ try {
+ super.testUpdateObjectACL();
+ Fail.failBecauseExceptionWasNotThrown(AWSResponseException.class);
+ } catch (AWSResponseException are) {
+ assertThat(are.getError().getCode()).isEqualTo("NotImplemented");
+ throw new SkipException("XML ACLs not supported", are);
+ }
+ }
+
+ @Override
+ @Test
+ public void testPublicWriteOnObject() throws InterruptedException,
+ ExecutionException, TimeoutException, IOException {
+ try {
+ super.testPublicWriteOnObject();
+ Fail.failBecauseExceptionWasNotThrown(AWSResponseException.class);
+ } catch (AWSResponseException are) {
+ assertThat(are.getError().getCode()).isEqualTo("NotImplemented");
+ throw new SkipException("public-read-write-acl not supported", are);
+ }
+ }
+
+ @Override
+ public void testCopyCannedAccessPolicyPublic() {
+ throw new SkipException("blob access control not supported");
+ }
+
+ @Override
+ public void testPutCannedAccessPolicyPublic() {
+ throw new SkipException("blob access control not supported");
+ }
+
+ @Override
+ public void testUpdateObjectCannedACL() {
+ throw new SkipException("blob access control not supported");
+ }
+}
diff --git a/src/test/java/org/gaul/s3proxy/EncryptedBlobStoreTest.java b/src/test/java/org/gaul/s3proxy/EncryptedBlobStoreTest.java
new file mode 100644
index 0000000..f9fc3a6
--- /dev/null
+++ b/src/test/java/org/gaul/s3proxy/EncryptedBlobStoreTest.java
@@ -0,0 +1,835 @@
+/*
+ * Copyright 2014-2021 Andrew Gaul
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.gaul.s3proxy;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Module;
+
+import org.gaul.s3proxy.crypto.Constants;
+import org.jclouds.ContextBuilder;
+import org.jclouds.blobstore.BlobStore;
+import org.jclouds.blobstore.BlobStoreContext;
+import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.blobstore.domain.BlobAccess;
+import org.jclouds.blobstore.domain.BlobMetadata;
+import org.jclouds.blobstore.domain.MultipartPart;
+import org.jclouds.blobstore.domain.MultipartUpload;
+import org.jclouds.blobstore.domain.PageSet;
+import org.jclouds.blobstore.domain.StorageMetadata;
+import org.jclouds.blobstore.domain.StorageType;
+import org.jclouds.blobstore.options.CopyOptions;
+import org.jclouds.blobstore.options.GetOptions;
+import org.jclouds.blobstore.options.ListContainerOptions;
+import org.jclouds.blobstore.options.PutOptions;
+import org.jclouds.io.Payload;
+import org.jclouds.io.Payloads;
+import org.jclouds.logging.slf4j.config.SLF4JLoggingModule;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@SuppressWarnings("UnstableApiUsage")
+public final class EncryptedBlobStoreTest {
+ private static final Logger logger =
+ LoggerFactory.getLogger(EncryptedBlobStoreTest.class);
+
+ private BlobStoreContext context;
+ private BlobStore blobStore;
+ private String containerName;
+ private BlobStore encryptedBlobStore;
+
+ private static Blob makeBlob(BlobStore blobStore, String blobName,
+ InputStream is, long contentLength) {
+
+ return blobStore.blobBuilder(blobName)
+ .payload(is)
+ .contentLength(contentLength)
+ .build();
+ }
+
+ private static Blob makeBlob(BlobStore blobStore, String blobName,
+ byte[] payload, long contentLength) {
+
+ return blobStore.blobBuilder(blobName)
+ .payload(payload)
+ .contentLength(contentLength)
+ .build();
+ }
+
+ private static Blob makeBlobWithContentType(BlobStore blobStore,
+ String blobName,
+ long contentLength,
+ InputStream is,
+ String contentType) {
+
+ return blobStore.blobBuilder(blobName)
+ .payload(is)
+ .contentLength(contentLength)
+ .contentType(contentType)
+ .build();
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ String password = "Password1234567!";
+ String salt = "12345678";
+
+ containerName = TestUtils.createRandomContainerName();
+
+ //noinspection UnstableApiUsage
+ context = ContextBuilder
+ .newBuilder("transient")
+ .credentials("identity", "credential")
+ .modules(ImmutableList.of(new SLF4JLoggingModule()))
+ .build(BlobStoreContext.class);
+ blobStore = context.getBlobStore();
+ blobStore.createContainerInLocation(null, containerName);
+
+ Properties properties = new Properties();
+ properties.put(S3ProxyConstants.PROPERTY_ENCRYPTED_BLOBSTORE, "true");
+ properties.put(S3ProxyConstants.PROPERTY_ENCRYPTED_BLOBSTORE_PASSWORD,
+ password);
+ properties.put(S3ProxyConstants.PROPERTY_ENCRYPTED_BLOBSTORE_SALT,
+ salt);
+
+ encryptedBlobStore =
+ EncryptedBlobStore.newEncryptedBlobStore(blobStore, properties);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (context != null) {
+ blobStore.deleteContainer(containerName);
+ context.close();
+ }
+ }
+
+ @Test
+ public void testBlobNotExists() {
+
+ String blobName = TestUtils.createRandomBlobName();
+ Blob blob = encryptedBlobStore.getBlob(containerName, blobName);
+ assertThat(blob).isNull();
+
+ blob = encryptedBlobStore.getBlob(containerName, blobName,
+ new GetOptions());
+ assertThat(blob).isNull();
+ }
+
+ @Test
+ public void testBlobNotEncrypted() throws Exception {
+
+ String[] tests = new String[] {
+ "1", // only 1 char
+ "123456789A12345", // lower then the AES block
+ "123456789A1234567", // one byte bigger then the AES block
+ "123456789A123456123456789B123456123456789C" +
+ "1234123456789A123456123456789B123456123456789C1234"
+ };
+
+ Map contentLengths = new HashMap<>();
+ for (String content : tests) {
+ String blobName = TestUtils.createRandomBlobName();
+
+ InputStream is = new ByteArrayInputStream(
+ content.getBytes(StandardCharsets.UTF_8));
+ contentLengths.put(blobName, (long) content.length());
+ Blob blob = makeBlob(blobStore, blobName, is, content.length());
+ blobStore.putBlob(containerName, blob);
+ blob = encryptedBlobStore.getBlob(containerName, blobName);
+
+ InputStream blobIs = blob.getPayload().openStream();
+ InputStreamReader r = new InputStreamReader(blobIs);
+ BufferedReader reader = new BufferedReader(r);
+ String plaintext = reader.lines().collect(Collectors.joining());
+ logger.debug("plaintext {}", plaintext);
+
+ assertThat(content).isEqualTo(plaintext);
+
+ GetOptions options = new GetOptions();
+ blob = encryptedBlobStore.getBlob(containerName, blobName, options);
+
+ blobIs = blob.getPayload().openStream();
+ r = new InputStreamReader(blobIs);
+ reader = new BufferedReader(r);
+ plaintext = reader.lines().collect(Collectors.joining());
+ logger.debug("plaintext {} with empty options ", plaintext);
+
+ assertThat(content).isEqualTo(plaintext);
+ }
+
+ PageSet blobs =
+ encryptedBlobStore.list(containerName, new ListContainerOptions());
+ for (StorageMetadata blob : blobs) {
+ assertThat(blob.getSize()).isEqualTo(
+ contentLengths.get(blob.getName()));
+ }
+
+ blobs = encryptedBlobStore.list();
+ StorageMetadata metadata = blobs.iterator().next();
+ assertThat(StorageType.CONTAINER).isEqualTo(metadata.getType());
+ }
+
+ @Test
+ public void testListEncrypted() {
+ String[] contents = new String[] {
+ "1", // only 1 char
+ "123456789A12345", // lower then the AES block
+ "123456789A1234567", // one byte bigger then the AES block
+ "123456789A123456123456789B123456123456789C1234"
+ };
+
+ Map contentLengths = new HashMap<>();
+ for (String content : contents) {
+ String blobName = TestUtils.createRandomBlobName();
+
+ InputStream is = new ByteArrayInputStream(
+ content.getBytes(StandardCharsets.UTF_8));
+ contentLengths.put(blobName, (long) content.length());
+ Blob blob =
+ makeBlob(encryptedBlobStore, blobName, is, content.length());
+ encryptedBlobStore.putBlob(containerName, blob);
+ }
+
+ PageSet blobs =
+ encryptedBlobStore.list(containerName);
+ for (StorageMetadata blob : blobs) {
+ assertThat(blob.getSize()).isEqualTo(
+ contentLengths.get(blob.getName()));
+ }
+
+ blobs =
+ encryptedBlobStore.list(containerName, new ListContainerOptions());
+ for (StorageMetadata blob : blobs) {
+ assertThat(blob.getSize()).isEqualTo(
+ contentLengths.get(blob.getName()));
+ encryptedBlobStore.removeBlob(containerName, blob.getName());
+ }
+
+ blobs =
+ encryptedBlobStore.list(containerName, new ListContainerOptions());
+ assertThat(blobs.size()).isEqualTo(0);
+ }
+
+ @Test
+ public void testListEncryptedMultipart() {
+
+ String blobName = TestUtils.createRandomBlobName();
+
+ String[] contentParts = new String[] {
+ "123456789A123456123456789B123456123456789C1234",
+ "123456789D123456123456789E123456123456789F123456",
+ "123456789G123456123456789H123456123456789I123"
+ };
+
+ String content = contentParts[0] + contentParts[1] + contentParts[2];
+ BlobMetadata blobMetadata = makeBlob(encryptedBlobStore, blobName,
+ content.getBytes(StandardCharsets.UTF_8),
+ content.length()).getMetadata();
+
+ MultipartUpload mpu =
+ encryptedBlobStore.initiateMultipartUpload(containerName,
+ blobMetadata, new PutOptions());
+
+ Payload payload1 = Payloads.newByteArrayPayload(
+ contentParts[0].getBytes(StandardCharsets.UTF_8));
+ Payload payload2 = Payloads.newByteArrayPayload(
+ contentParts[1].getBytes(StandardCharsets.UTF_8));
+ Payload payload3 = Payloads.newByteArrayPayload(
+ contentParts[2].getBytes(StandardCharsets.UTF_8));
+
+ encryptedBlobStore.uploadMultipartPart(mpu, 1, payload1);
+ encryptedBlobStore.uploadMultipartPart(mpu, 2, payload2);
+ encryptedBlobStore.uploadMultipartPart(mpu, 3, payload3);
+
+ List parts = encryptedBlobStore.listMultipartUpload(mpu);
+
+ int index = 0;
+ for (MultipartPart part : parts) {
+ assertThat((long) contentParts[index].length()).isEqualTo(
+ part.partSize());
+ index++;
+ }
+
+ encryptedBlobStore.completeMultipartUpload(mpu, parts);
+
+ PageSet blobs =
+ encryptedBlobStore.list(containerName);
+ StorageMetadata metadata = blobs.iterator().next();
+ assertThat((long) content.length()).isEqualTo(metadata.getSize());
+
+ ListContainerOptions options = new ListContainerOptions();
+ blobs = encryptedBlobStore.list(containerName, options.withDetails());
+ metadata = blobs.iterator().next();
+ assertThat((long) content.length()).isEqualTo(metadata.getSize());
+
+ blobs = encryptedBlobStore.list();
+ metadata = blobs.iterator().next();
+ assertThat(StorageType.CONTAINER).isEqualTo(metadata.getType());
+
+ List singleList = new ArrayList<>();
+ singleList.add(blobName);
+ encryptedBlobStore.removeBlobs(containerName, singleList);
+ blobs = encryptedBlobStore.list(containerName);
+ assertThat(blobs.size()).isEqualTo(0);
+ }
+
+ @Test
+ public void testBlobNotEncryptedRanges() throws Exception {
+
+ for (int run = 0; run < 100; run++) {
+ String[] tests = new String[] {
+ "123456789A12345", // lower then the AES block
+ "123456789A1234567", // one byte bigger then the AES block
+ "123456789A123456123456789B123456123456789C" +
+ "1234123456789A123456123456789B123456123456789C1234"
+ };
+
+ for (String content : tests) {
+ String blobName = TestUtils.createRandomBlobName();
+ Random rand = new Random();
+
+ InputStream is = new ByteArrayInputStream(
+ content.getBytes(StandardCharsets.UTF_8));
+ Blob blob = makeBlob(blobStore, blobName, is, content.length());
+ blobStore.putBlob(containerName, blob);
+
+ GetOptions options = new GetOptions();
+ int offset = rand.nextInt(content.length() - 1);
+ logger.debug("content {} with offset {}", content, offset);
+
+ options.startAt(offset);
+ blob = encryptedBlobStore.getBlob(containerName, blobName,
+ options);
+
+ InputStream blobIs = blob.getPayload().openStream();
+ InputStreamReader r = new InputStreamReader(blobIs);
+ BufferedReader reader = new BufferedReader(r);
+ String plaintext = reader.lines().collect(Collectors.joining());
+ logger.debug("plaintext {} with offset {}", plaintext, offset);
+
+ assertThat(plaintext).isEqualTo(content.substring(offset));
+
+ options = new GetOptions();
+ int tail = rand.nextInt(content.length());
+ if (tail == 0) {
+ tail++;
+ }
+ logger.debug("content {} with tail {}", content, tail);
+
+ options.tail(tail);
+ blob = encryptedBlobStore.getBlob(containerName, blobName,
+ options);
+
+ blobIs = blob.getPayload().openStream();
+ r = new InputStreamReader(blobIs);
+ reader = new BufferedReader(r);
+ plaintext = reader.lines().collect(Collectors.joining());
+ logger.debug("plaintext {} with tail {}", plaintext, tail);
+
+ assertThat(plaintext).isEqualTo(
+ content.substring(content.length() - tail));
+
+ options = new GetOptions();
+ offset = 1;
+ int end = content.length() - 2;
+ logger.debug("content {} with range {}-{}", content, offset,
+ end);
+
+ options.range(offset, end);
+ blob = encryptedBlobStore.getBlob(containerName, blobName,
+ options);
+
+ blobIs = blob.getPayload().openStream();
+ r = new InputStreamReader(blobIs);
+ reader = new BufferedReader(r);
+ plaintext = reader.lines().collect(Collectors.joining());
+ logger.debug("plaintext {} with range {}-{}", plaintext, offset,
+ end);
+
+ assertThat(plaintext).isEqualTo(
+ content.substring(offset, end + 1));
+ }
+ }
+ }
+
+ @Test
+ public void testEncryptContent() throws Exception {
+ String[] tests = new String[] {
+ "1", // only 1 char
+ "123456789A12345", // lower then the AES block
+ "123456789A1234567", // one byte bigger then the AES block
+ "123456789A123456123456789B123456123456789C1234"
+ };
+
+ for (String content : tests) {
+ String blobName = TestUtils.createRandomBlobName();
+ String contentType = "plain/text";
+
+ InputStream is = new ByteArrayInputStream(
+ content.getBytes(StandardCharsets.UTF_8));
+ Blob blob = makeBlobWithContentType(encryptedBlobStore, blobName,
+ content.length(), is, contentType);
+ encryptedBlobStore.putBlob(containerName, blob);
+
+ blob = encryptedBlobStore.getBlob(containerName, blobName);
+
+ InputStream blobIs = blob.getPayload().openStream();
+ InputStreamReader r = new InputStreamReader(blobIs);
+ BufferedReader reader = new BufferedReader(r);
+ String plaintext = reader.lines().collect(Collectors.joining());
+ logger.debug("plaintext {}", plaintext);
+
+ assertThat(plaintext).isEqualTo(content);
+
+ blob = blobStore.getBlob(containerName,
+ blobName + Constants.S3_ENC_SUFFIX);
+ blobIs = blob.getPayload().openStream();
+ r = new InputStreamReader(blobIs);
+ reader = new BufferedReader(r);
+ String encrypted = reader.lines().collect(Collectors.joining());
+ logger.debug("encrypted {}", encrypted);
+
+ assertThat(content).isNotEqualTo(encrypted);
+
+ assertThat(encryptedBlobStore.blobExists(containerName,
+ blobName)).isTrue();
+
+ BlobAccess access =
+ encryptedBlobStore.getBlobAccess(containerName, blobName);
+ assertThat(access).isEqualTo(BlobAccess.PRIVATE);
+
+ encryptedBlobStore.setBlobAccess(containerName, blobName,
+ BlobAccess.PUBLIC_READ);
+ access = encryptedBlobStore.getBlobAccess(containerName, blobName);
+ assertThat(access).isEqualTo(BlobAccess.PUBLIC_READ);
+ }
+ }
+
+ @Test
+ public void testEncryptContentWithOptions() throws Exception {
+ String[] tests = new String[] {
+ "1", // only 1 char
+ "123456789A12345", // lower then the AES block
+ "123456789A1234567", // one byte bigger then the AES block
+ "123456789A123456123456789B123456123456789C1234"
+ };
+
+ for (String content : tests) {
+ String blobName = TestUtils.createRandomBlobName();
+ String contentType = "plain/text; charset=utf-8";
+
+ InputStream is = new ByteArrayInputStream(
+ content.getBytes(StandardCharsets.UTF_8));
+ Blob blob = makeBlobWithContentType(encryptedBlobStore, blobName,
+ content.length(), is, contentType);
+ PutOptions options = new PutOptions();
+ encryptedBlobStore.putBlob(containerName, blob, options);
+
+ blob = encryptedBlobStore.getBlob(containerName, blobName);
+
+ InputStream blobIs = blob.getPayload().openStream();
+ InputStreamReader r = new InputStreamReader(blobIs);
+ BufferedReader reader = new BufferedReader(r);
+ String plaintext = reader.lines().collect(Collectors.joining());
+ logger.debug("plaintext {}", plaintext);
+
+ assertThat(content).isEqualTo(plaintext);
+
+ blob = blobStore.getBlob(containerName,
+ blobName + Constants.S3_ENC_SUFFIX);
+ blobIs = blob.getPayload().openStream();
+ r = new InputStreamReader(blobIs);
+ reader = new BufferedReader(r);
+ String encrypted = reader.lines().collect(Collectors.joining());
+ logger.debug("encrypted {}", encrypted);
+
+ assertThat(content).isNotEqualTo(encrypted);
+
+ BlobMetadata metadata =
+ encryptedBlobStore.blobMetadata(containerName,
+ blobName + Constants.S3_ENC_SUFFIX);
+ assertThat(contentType).isEqualTo(
+ metadata.getContentMetadata().getContentType());
+
+ encryptedBlobStore.copyBlob(containerName, blobName,
+ containerName, blobName + "-copy", CopyOptions.NONE);
+
+ blob = blobStore.getBlob(containerName,
+ blobName + Constants.S3_ENC_SUFFIX);
+ blobIs = blob.getPayload().openStream();
+ r = new InputStreamReader(blobIs);
+ reader = new BufferedReader(r);
+ encrypted = reader.lines().collect(Collectors.joining());
+ logger.debug("encrypted {}", encrypted);
+
+ assertThat(content).isNotEqualTo(encrypted);
+
+ blob =
+ encryptedBlobStore.getBlob(containerName, blobName + "-copy");
+ blobIs = blob.getPayload().openStream();
+ r = new InputStreamReader(blobIs);
+ reader = new BufferedReader(r);
+ plaintext = reader.lines().collect(Collectors.joining());
+ logger.debug("plaintext {}", plaintext);
+
+ assertThat(content).isEqualTo(plaintext);
+ }
+ }
+
+ @Test
+ public void testEncryptMultipartContent() throws Exception {
+ String blobName = TestUtils.createRandomBlobName();
+
+ String content1 = "123456789A123456123456789B123456123456789C1234";
+ String content2 = "123456789D123456123456789E123456123456789F123456";
+ String content3 = "123456789G123456123456789H123456123456789I123";
+
+ String content = content1 + content2 + content3;
+ BlobMetadata blobMetadata = makeBlob(encryptedBlobStore, blobName,
+ content.getBytes(StandardCharsets.UTF_8),
+ content.length()).getMetadata();
+ MultipartUpload mpu =
+ encryptedBlobStore.initiateMultipartUpload(containerName,
+ blobMetadata, new PutOptions());
+
+ Payload payload1 = Payloads.newByteArrayPayload(
+ content1.getBytes(StandardCharsets.UTF_8));
+ Payload payload2 = Payloads.newByteArrayPayload(
+ content2.getBytes(StandardCharsets.UTF_8));
+ Payload payload3 = Payloads.newByteArrayPayload(
+ content3.getBytes(StandardCharsets.UTF_8));
+
+ encryptedBlobStore.uploadMultipartPart(mpu, 1, payload1);
+ encryptedBlobStore.uploadMultipartPart(mpu, 2, payload2);
+ encryptedBlobStore.uploadMultipartPart(mpu, 3, payload3);
+
+ List mpus =
+ encryptedBlobStore.listMultipartUploads(containerName);
+ assertThat(mpus.size()).isEqualTo(1);
+
+ List parts = encryptedBlobStore.listMultipartUpload(mpu);
+ assertThat(mpus.get(0).id()).isEqualTo(mpu.id());
+
+ encryptedBlobStore.completeMultipartUpload(mpu, parts);
+ Blob blob = encryptedBlobStore.getBlob(containerName, blobName);
+
+ InputStream blobIs = blob.getPayload().openStream();
+ InputStreamReader r = new InputStreamReader(blobIs);
+ BufferedReader reader = new BufferedReader(r);
+ String plaintext = reader.lines().collect(Collectors.joining());
+ logger.debug("plaintext {}", plaintext);
+ assertThat(plaintext).isEqualTo(content);
+
+ blob = blobStore.getBlob(containerName,
+ blobName + Constants.S3_ENC_SUFFIX);
+ blobIs = blob.getPayload().openStream();
+ r = new InputStreamReader(blobIs);
+ reader = new BufferedReader(r);
+ String encrypted = reader.lines().collect(Collectors.joining());
+ logger.debug("encrypted {}", encrypted);
+
+ assertThat(content).isNotEqualTo(encrypted);
+ }
+
+ @Test
+ public void testReadPartial() throws Exception {
+
+ for (int offset = 0; offset < 60; offset++) {
+ logger.debug("Test with offset {}", offset);
+
+ String blobName = TestUtils.createRandomBlobName();
+ String content =
+ "123456789A123456123456789B123456123456789" +
+ "C123456789D123456789E12345";
+ InputStream is = new ByteArrayInputStream(
+ content.getBytes(StandardCharsets.UTF_8));
+
+ Blob blob =
+ makeBlob(encryptedBlobStore, blobName, is, content.length());
+ encryptedBlobStore.putBlob(containerName, blob);
+
+ GetOptions options = new GetOptions();
+ options.startAt(offset);
+ blob = encryptedBlobStore.getBlob(containerName, blobName, options);
+
+ InputStream blobIs = blob.getPayload().openStream();
+ InputStreamReader r = new InputStreamReader(blobIs);
+ BufferedReader reader = new BufferedReader(r);
+ String plaintext = reader.lines().collect(Collectors.joining());
+ logger.debug("plaintext {}", plaintext);
+
+ assertThat(plaintext).isEqualTo(content.substring(offset));
+ }
+ }
+
+ @Test
+ public void testReadTail() throws Exception {
+
+ for (int length = 1; length < 60; length++) {
+ logger.debug("Test with length {}", length);
+
+ String blobName = TestUtils.createRandomBlobName();
+
+ String content =
+ "123456789A123456123456789B123456123456789C" +
+ "123456789D123456789E12345";
+ InputStream is = new ByteArrayInputStream(
+ content.getBytes(StandardCharsets.UTF_8));
+
+ Blob blob =
+ makeBlob(encryptedBlobStore, blobName, is, content.length());
+ encryptedBlobStore.putBlob(containerName, blob);
+
+ GetOptions options = new GetOptions();
+ options.tail(length);
+ blob = encryptedBlobStore.getBlob(containerName, blobName, options);
+
+ InputStream blobIs = blob.getPayload().openStream();
+ InputStreamReader r = new InputStreamReader(blobIs);
+ BufferedReader reader = new BufferedReader(r);
+ String plaintext = reader.lines().collect(Collectors.joining());
+ logger.debug("plaintext {}", plaintext);
+
+ assertThat(plaintext).isEqualTo(
+ content.substring(content.length() - length));
+ }
+ }
+
+ @Test
+ public void testReadPartialWithRandomEnd() throws Exception {
+
+ for (int run = 0; run < 100; run++) {
+ for (int offset = 0; offset < 50; offset++) {
+ Random rand = new Random();
+ int end = offset + rand.nextInt(20) + 2;
+ int size = end - offset + 1;
+
+ logger.debug("Test with offset {} and end {} size {}",
+ offset, end, size);
+
+ String blobName = TestUtils.createRandomBlobName();
+
+ String content =
+ "123456789A123456-123456789B123456-123456789C123456-" +
+ "123456789D123456-123456789E123456";
+ InputStream is = new ByteArrayInputStream(
+ content.getBytes(StandardCharsets.UTF_8));
+
+ Blob blob = makeBlob(encryptedBlobStore, blobName, is,
+ content.length());
+ encryptedBlobStore.putBlob(containerName, blob);
+
+ GetOptions options = new GetOptions();
+ options.range(offset, end);
+ blob = encryptedBlobStore.getBlob(containerName, blobName,
+ options);
+
+ InputStream blobIs = blob.getPayload().openStream();
+ InputStreamReader r = new InputStreamReader(blobIs);
+ BufferedReader reader = new BufferedReader(r);
+ String plaintext = reader.lines().collect(Collectors.joining());
+ logger.debug("plaintext {}", plaintext);
+
+ assertThat(plaintext).hasSize(size);
+ assertThat(plaintext).isEqualTo(
+ content.substring(offset, end + 1));
+ }
+ }
+ }
+
+ @Test
+ public void testMultipartReadPartial() throws Exception {
+
+ for (int offset = 0; offset < 130; offset++) {
+ logger.debug("Test with offset {}", offset);
+
+ String blobName = TestUtils.createRandomBlobName();
+
+ String content1 = "PART1-789A123456123456789B123456123456789C1234";
+ String content2 =
+ "PART2-789D123456123456789E123456123456789F123456";
+ String content3 = "PART3-789G123456123456789H123456123456789I123";
+ String content = content1 + content2 + content3;
+
+ BlobMetadata blobMetadata = makeBlob(encryptedBlobStore, blobName,
+ content.getBytes(StandardCharsets.UTF_8),
+ content.length()).getMetadata();
+ MultipartUpload mpu =
+ encryptedBlobStore.initiateMultipartUpload(containerName,
+ blobMetadata, new PutOptions());
+
+ Payload payload1 = Payloads.newByteArrayPayload(
+ content1.getBytes(StandardCharsets.UTF_8));
+ Payload payload2 = Payloads.newByteArrayPayload(
+ content2.getBytes(StandardCharsets.UTF_8));
+ Payload payload3 = Payloads.newByteArrayPayload(
+ content3.getBytes(StandardCharsets.UTF_8));
+
+ encryptedBlobStore.uploadMultipartPart(mpu, 1, payload1);
+ encryptedBlobStore.uploadMultipartPart(mpu, 2, payload2);
+ encryptedBlobStore.uploadMultipartPart(mpu, 3, payload3);
+
+ List parts =
+ encryptedBlobStore.listMultipartUpload(mpu);
+ encryptedBlobStore.completeMultipartUpload(mpu, parts);
+
+ GetOptions options = new GetOptions();
+ options.startAt(offset);
+ Blob blob =
+ encryptedBlobStore.getBlob(containerName, blobName, options);
+
+ InputStream blobIs = blob.getPayload().openStream();
+ InputStreamReader r = new InputStreamReader(blobIs);
+ BufferedReader reader = new BufferedReader(r);
+ String plaintext = reader.lines().collect(Collectors.joining());
+ logger.debug("plaintext {}", plaintext);
+
+ assertThat(plaintext).isEqualTo(content.substring(offset));
+ }
+ }
+
+ @Test
+ public void testMultipartReadTail() throws Exception {
+
+ for (int length = 1; length < 130; length++) {
+ logger.debug("Test with length {}", length);
+
+ String blobName = TestUtils.createRandomBlobName();
+
+ String content1 = "PART1-789A123456123456789B123456123456789C1234";
+ String content2 =
+ "PART2-789D123456123456789E123456123456789F123456";
+ String content3 = "PART3-789G123456123456789H123456123456789I123";
+ String content = content1 + content2 + content3;
+ BlobMetadata blobMetadata = makeBlob(encryptedBlobStore, blobName,
+ content.getBytes(StandardCharsets.UTF_8),
+ content.length()).getMetadata();
+ MultipartUpload mpu =
+ encryptedBlobStore.initiateMultipartUpload(containerName,
+ blobMetadata, new PutOptions());
+
+ Payload payload1 = Payloads.newByteArrayPayload(
+ content1.getBytes(StandardCharsets.UTF_8));
+ Payload payload2 = Payloads.newByteArrayPayload(
+ content2.getBytes(StandardCharsets.UTF_8));
+ Payload payload3 = Payloads.newByteArrayPayload(
+ content3.getBytes(StandardCharsets.UTF_8));
+
+ encryptedBlobStore.uploadMultipartPart(mpu, 1, payload1);
+ encryptedBlobStore.uploadMultipartPart(mpu, 2, payload2);
+ encryptedBlobStore.uploadMultipartPart(mpu, 3, payload3);
+
+ List parts =
+ encryptedBlobStore.listMultipartUpload(mpu);
+ encryptedBlobStore.completeMultipartUpload(mpu, parts);
+
+ GetOptions options = new GetOptions();
+ options.tail(length);
+ Blob blob =
+ encryptedBlobStore.getBlob(containerName, blobName, options);
+
+ InputStream blobIs = blob.getPayload().openStream();
+ InputStreamReader r = new InputStreamReader(blobIs);
+ BufferedReader reader = new BufferedReader(r);
+ String plaintext = reader.lines().collect(Collectors.joining());
+ logger.debug("plaintext {}", plaintext);
+
+ assertThat(plaintext).isEqualTo(
+ content.substring(content.length() - length));
+ }
+ }
+
+ @Test
+ public void testMultipartReadPartialWithRandomEnd() throws Exception {
+
+ for (int run = 0; run < 100; run++) {
+ // total len = 139
+ for (int offset = 0; offset < 70; offset++) {
+ Random rand = new Random();
+ int end = offset + rand.nextInt(60) + 2;
+ int size = end - offset + 1;
+ logger.debug("Test with offset {} and end {} size {}",
+ offset, end, size);
+
+ String blobName = TestUtils.createRandomBlobName();
+
+ String content1 =
+ "PART1-789A123456123456789B123456123456789C1234";
+ String content2 =
+ "PART2-789D123456123456789E123456123456789F123456";
+ String content3 =
+ "PART3-789G123456123456789H123456123456789I123";
+
+ String content = content1 + content2 + content3;
+ BlobMetadata blobMetadata =
+ makeBlob(encryptedBlobStore, blobName,
+ content.getBytes(StandardCharsets.UTF_8),
+ content.length()).getMetadata();
+ MultipartUpload mpu =
+ encryptedBlobStore.initiateMultipartUpload(containerName,
+ blobMetadata, new PutOptions());
+
+ Payload payload1 = Payloads.newByteArrayPayload(
+ content1.getBytes(StandardCharsets.UTF_8));
+ Payload payload2 = Payloads.newByteArrayPayload(
+ content2.getBytes(StandardCharsets.UTF_8));
+ Payload payload3 = Payloads.newByteArrayPayload(
+ content3.getBytes(StandardCharsets.UTF_8));
+
+ encryptedBlobStore.uploadMultipartPart(mpu, 1, payload1);
+ encryptedBlobStore.uploadMultipartPart(mpu, 2, payload2);
+ encryptedBlobStore.uploadMultipartPart(mpu, 3, payload3);
+
+ List parts =
+ encryptedBlobStore.listMultipartUpload(mpu);
+ encryptedBlobStore.completeMultipartUpload(mpu, parts);
+
+ GetOptions options = new GetOptions();
+ options.range(offset, end);
+ Blob blob = encryptedBlobStore.getBlob(containerName, blobName,
+ options);
+
+ InputStream blobIs = blob.getPayload().openStream();
+ InputStreamReader r = new InputStreamReader(blobIs);
+ BufferedReader reader = new BufferedReader(r);
+ String plaintext = reader.lines().collect(Collectors.joining());
+ logger.debug("plaintext {}", plaintext);
+
+ assertThat(plaintext).isEqualTo(
+ content.substring(offset, end + 1));
+ }
+ }
+ }
+}
diff --git a/src/test/java/org/gaul/s3proxy/TestUtils.java b/src/test/java/org/gaul/s3proxy/TestUtils.java
index e76e9bb..e64d038 100644
--- a/src/test/java/org/gaul/s3proxy/TestUtils.java
+++ b/src/test/java/org/gaul/s3proxy/TestUtils.java
@@ -188,6 +188,14 @@ final class TestUtils {
BlobStoreContext context = builder.build(BlobStoreContext.class);
info.blobStore = context.getBlobStore();
+ String encrypted = info.getProperties().getProperty(
+ S3ProxyConstants.PROPERTY_ENCRYPTED_BLOBSTORE);
+ if (encrypted != null && encrypted.equals("true")) {
+ info.blobStore =
+ EncryptedBlobStore.newEncryptedBlobStore(info.blobStore,
+ info.getProperties());
+ }
+
S3Proxy.Builder s3ProxyBuilder = S3Proxy.Builder.fromProperties(
info.getProperties());
s3ProxyBuilder.blobStore(info.blobStore);
diff --git a/src/test/resources/s3proxy-encryption.conf b/src/test/resources/s3proxy-encryption.conf
new file mode 100644
index 0000000..7d4b83f
--- /dev/null
+++ b/src/test/resources/s3proxy-encryption.conf
@@ -0,0 +1,20 @@
+s3proxy.endpoint=http://127.0.0.1:0
+s3proxy.secure-endpoint=https://127.0.0.1:0
+#s3proxy.service-path=s3proxy
+# authorization must be aws-v2, aws-v4, aws-v2-or-v4, or none
+s3proxy.authorization=aws-v2-or-v4
+s3proxy.identity=local-identity
+s3proxy.credential=local-credential
+s3proxy.keystore-path=keystore.jks
+s3proxy.keystore-password=password
+
+jclouds.provider=transient
+jclouds.identity=remote-identity
+jclouds.credential=remote-credential
+# endpoint is optional for some providers
+#jclouds.endpoint=http://127.0.0.1:8081
+jclouds.filesystem.basedir=/tmp/blobstore
+
+s3proxy.encrypted-blobstore=true
+s3proxy.encrypted-blobstore-password=1234567890123456
+s3proxy.encrypted-blobstore-salt=12345678