From 628fb3bf736077a819fdefc5ee42cb45e8ece901 Mon Sep 17 00:00:00 2001 From: Una Thompson Date: Sat, 7 Sep 2019 19:38:27 -0700 Subject: [PATCH] Increase non-multipart upload limit, disable backups Also try and fail at adding multipart uploads --- .../com/jortage/proxy/JortageBlobStore.java | 123 ++++++++++-------- .../java/com/jortage/proxy/JortageProxy.java | 5 +- src/main/resources/logback.xml | 3 + 3 files changed, 77 insertions(+), 54 deletions(-) diff --git a/src/main/java/com/jortage/proxy/JortageBlobStore.java b/src/main/java/com/jortage/proxy/JortageBlobStore.java index 20d4dc4..539db4e 100644 --- a/src/main/java/com/jortage/proxy/JortageBlobStore.java +++ b/src/main/java/com/jortage/proxy/JortageBlobStore.java @@ -18,8 +18,10 @@ import org.jclouds.blobstore.domain.BlobMetadata; import org.jclouds.blobstore.domain.ContainerAccess; import org.jclouds.blobstore.domain.MultipartPart; import org.jclouds.blobstore.domain.MultipartUpload; +import org.jclouds.blobstore.domain.MutableBlobMetadata; import org.jclouds.blobstore.domain.PageSet; import org.jclouds.blobstore.domain.StorageMetadata; +import org.jclouds.blobstore.domain.internal.MutableBlobMetadataImpl; import org.jclouds.blobstore.options.CopyOptions; import org.jclouds.blobstore.options.CreateContainerOptions; import org.jclouds.blobstore.options.GetOptions; @@ -84,6 +86,26 @@ public class JortageBlobStore extends ForwardingBlobStore { return delegate().getBlob(bucket, map(container, name), getOptions); } + @Override + public void downloadBlob(String container, String name, File destination) { + delegate().downloadBlob(bucket, map(container, name), destination); + } + + @Override + public void downloadBlob(String container, String name, File destination, ExecutorService executor) { + delegate().downloadBlob(bucket, map(container, name), destination, executor); + } + + @Override + public InputStream streamBlob(String container, String name) { + return delegate().streamBlob(bucket, map(container, name)); + } + + @Override + public InputStream streamBlob(String container, String name, ExecutorService executor) { + return delegate().streamBlob(bucket, map(container, name), executor); + } + @Override public BlobAccess getBlobAccess(String container, String name) { return BlobAccess.PUBLIC_READ; @@ -125,6 +147,21 @@ public class JortageBlobStore extends ForwardingBlobStore { throw new UnsupportedOperationException(); } + @Override + public int getMaximumNumberOfParts() { + return delegate().getMaximumNumberOfParts(); + } + + @Override + public long getMinimumMultipartPartSize() { + return delegate().getMinimumMultipartPartSize(); + } + + @Override + public long getMaximumMultipartPartSize() { + return delegate().getMaximumMultipartPartSize(); + } + @Override public String putBlob(String container, Blob blob) { checkContainer(container); @@ -151,7 +188,7 @@ public class JortageBlobStore extends ForwardingBlobStore { .payload(payload) .userMetadata(blob.getMetadata().getUserMetadata()) .build(); - String etag = delegate().putBlob(bucket, blob2, new PutOptions().setBlobAccess(BlobAccess.PUBLIC_READ)); + String etag = delegate().putBlob(bucket, blob2, new PutOptions().setBlobAccess(BlobAccess.PUBLIC_READ).multipart()); paths.put(buildKey(blob.getMetadata().getName()), hash); return etag; } @@ -162,9 +199,41 @@ public class JortageBlobStore extends ForwardingBlobStore { } } + @Override + public MultipartUpload initiateMultipartUpload(String container, BlobMetadata blobMetadata, PutOptions options) { + MutableBlobMetadata mbm = new MutableBlobMetadataImpl(blobMetadata); + mbm.setContainer(bucket); + mbm.setName(map(blobMetadata.getContainer(), blobMetadata.getName())); + return delegate().initiateMultipartUpload(bucket, mbm, options); + } + + @Override + public void abortMultipartUpload(MultipartUpload mpu) { + delegate().abortMultipartUpload(mpu); + } + + @Override + public String completeMultipartUpload(MultipartUpload mpu, List parts) { + return delegate().completeMultipartUpload(mpu, parts); + } + + @Override + public MultipartPart uploadMultipartPart(MultipartUpload mpu, int partNumber, Payload payload) { + return delegate().uploadMultipartPart(mpu, partNumber, payload); + } + + @Override + public List listMultipartUpload(MultipartUpload mpu) { + return delegate().listMultipartUpload(mpu); + } + + @Override + public List listMultipartUploads(String container) { + return delegate().listMultipartUploads(bucket); + } + @Override public String putBlob(String containerName, Blob blob, PutOptions putOptions) { - if (putOptions.isMultipart()) throw new UnsupportedOperationException("multipart blobs not supported"); return putBlob(containerName, blob); } @@ -238,54 +307,4 @@ public class JortageBlobStore extends ForwardingBlobStore { throw new UnsupportedOperationException("Read-only BlobStore"); } - @Override - public MultipartUpload initiateMultipartUpload(String container, BlobMetadata blobMetadata, PutOptions options) { - throw new UnsupportedOperationException("Read-only BlobStore"); - } - - @Override - public void abortMultipartUpload(MultipartUpload mpu) { - throw new UnsupportedOperationException("Read-only BlobStore"); - } - - @Override - public String completeMultipartUpload(MultipartUpload mpu, List parts) { - throw new UnsupportedOperationException("Read-only BlobStore"); - } - - @Override - public MultipartPart uploadMultipartPart(MultipartUpload mpu, int partNumber, Payload payload) { - throw new UnsupportedOperationException("Read-only BlobStore"); - } - - @Override - public List listMultipartUpload(MultipartUpload mpu) { - throw new UnsupportedOperationException("Read-only BlobStore"); - } - - // TODO: should ReadOnlyBlobStore allow listing parts and uploads? - @Override - public List listMultipartUploads(String container) { - throw new UnsupportedOperationException("Read-only BlobStore"); - } - - @Override - public void downloadBlob(String container, String name, File destination) { - throw new UnsupportedOperationException(); - } - - @Override - public void downloadBlob(String container, String name, File destination, ExecutorService executor) { - throw new UnsupportedOperationException(); - } - - @Override - public InputStream streamBlob(String container, String name) { - throw new UnsupportedOperationException(); - } - - @Override - public InputStream streamBlob(String container, String name, ExecutorService executor) { - throw new UnsupportedOperationException(); - } } \ No newline at end of file diff --git a/src/main/java/com/jortage/proxy/JortageProxy.java b/src/main/java/com/jortage/proxy/JortageProxy.java index 2776aa5..0c3b12a 100644 --- a/src/main/java/com/jortage/proxy/JortageProxy.java +++ b/src/main/java/com/jortage/proxy/JortageProxy.java @@ -109,6 +109,7 @@ public class JortageProxy { S3Proxy s3Proxy = S3Proxy.builder() .awsAuthentication(AuthenticationType.AWS_V2_OR_V4, "DUMMY", "DUMMY") .endpoint(URI.create("http://localhost:23278")) + .v4MaxNonChunkedRequestSize(128*1024*1024) .build(); s3Proxy.setBlobStoreLocator(new BlobStoreLocator() { @@ -170,7 +171,8 @@ public class JortageProxy { System.err.println("Commit successful."); i++; // every 10 minutes (roughly) - if (i % 40 == 0) { + // FIXME this is causing OOMEs in production + if (false && i % 40 == 0) { System.err.println("Creating backup..."); File backups = new File("backups"); if (!backups.exists()) { @@ -222,7 +224,6 @@ public class JortageProxy { bucket = ((JsonPrimitive)config.getObject("backend").get("bucket")).asString(); publicHost = ((JsonPrimitive)config.getObject("backend").get("publicHost")).asString(); Properties props = new Properties(); - props.put("jclouds.wire", "debug"); backingBlobStore = ContextBuilder.newBuilder("s3") .credentials(((JsonPrimitive)config.getObject("backend").get("accessKeyId")).asString(), ((JsonPrimitive)config.getObject("backend").get("secretAccessKey")).asString()) .modules(ImmutableList.of(new SLF4JLoggingModule())) diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 27ce7cb..59c5787 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -10,6 +10,9 @@ + + +