From 37e843a6ef51823900f0fd73040a922e3b45e141 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Mon, 21 Sep 2020 21:34:31 +0900 Subject: [PATCH] Allow GCS backend to upload more than 32 parts This recursively combines up to 32 sets of 32 parts, allowing 1024 part multipart uploads. Fixes #330. --- .../java/org/gaul/s3proxy/S3ProxyHandler.java | 51 +++++++++++++++---- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java b/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java index 4c36e85..209006b 100644 --- a/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java +++ b/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java @@ -2197,20 +2197,20 @@ public class S3ProxyHandler { HttpServletResponse response, InputStream is, final BlobStore blobStore, String containerName, String blobName, String uploadId) throws IOException, S3Exception { - final MultipartUpload mpu; + BlobMetadata metadata; + PutOptions options; if (Quirks.MULTIPART_REQUIRES_STUB.contains(getBlobStoreType( blobStore))) { - Blob stubBlob = blobStore.getBlob(containerName, uploadId); + metadata = blobStore.getBlob(containerName, uploadId).getMetadata(); BlobAccess access = blobStore.getBlobAccess(containerName, uploadId); - mpu = MultipartUpload.create(containerName, - blobName, uploadId, stubBlob.getMetadata(), - new PutOptions().setBlobAccess(access)); + options = new PutOptions().setBlobAccess(access); } else { - mpu = MultipartUpload.create(containerName, - blobName, uploadId, new MutableBlobMetadataImpl(), - new PutOptions()); + metadata = new MutableBlobMetadataImpl(); + options = new PutOptions(); } + final MultipartUpload mpu = MultipartUpload.create(containerName, + blobName, uploadId, metadata, options); // List parts to get part sizes and to map multiple Azure parts // into single parts. @@ -2219,7 +2219,6 @@ public class S3ProxyHandler { for (MultipartPart part : blobStore.listMultipartUpload(mpu)) { builder.put(part.partNumber(), part); } - ImmutableMap partsByListing = builder.build(); final List parts = new ArrayList<>(); String blobStoreType = getBlobStoreType(blobStore); @@ -2228,6 +2227,28 @@ public class S3ProxyHandler { for (MultipartPart part : blobStore.listMultipartUpload(mpu)) { parts.add(part); } + } else if (blobStoreType.equals("google-cloud-storage")) { + // GCS only supports 32 parts but we can support up to 1024 by + // recursively combining objects. + for (int partNumber = 1;; ++partNumber) { + MultipartUpload mpu2 = MultipartUpload.create( + containerName, + String.format("%s_%08d", mpu.id(), partNumber), + String.format("%s_%08d", mpu.id(), partNumber), + metadata, options); + List subParts = blobStore.listMultipartUpload( + mpu2); + if (subParts.isEmpty()) { + break; + } + long partSize = 0; + for (MultipartPart part : subParts) { + partSize += part.partSize(); + } + String eTag = blobStore.completeMultipartUpload(mpu2, subParts); + parts.add(MultipartPart.create( + partNumber, partSize, eTag, /*lastModified=*/ null)); + } } else { CompleteMultipartUploadRequest cmu; try { @@ -2244,6 +2265,9 @@ public class S3ProxyHandler { requestParts.put(part.partNumber, part.eTag); } } + + ImmutableMap partsByListing = + builder.build(); for (Iterator> it = requestParts.entrySet().iterator(); it.hasNext();) { Map.Entry entry = it.next(); @@ -2722,6 +2746,15 @@ public class S3ProxyHandler { "ArgumentValue", partNumberString)); } + // GCS only supports 32 parts so partition MPU into 32-part chunks. + String blobStoreType = getBlobStoreType(blobStore); + if (blobStoreType.equals("google-cloud-storage")) { + // fix up 1-based part numbers + uploadId = String.format( + "%s_%08d", uploadId, ((partNumber - 1) / 32) + 1); + partNumber = ((partNumber - 1) % 32) + 1; + } + // TODO: how to reconstruct original mpu? BlobMetadata blobMetadata; if (Quirks.MULTIPART_REQUIRES_STUB.contains(getBlobStoreType(