kopia lustrzana https://github.com/gaul/s3proxy
Allow GCS backend to upload more than 32 parts
This recursively combines up to 32 sets of 32 parts, allowing 1024 part multipart uploads. Fixes #330.pull/361/head
rodzic
29413b12a7
commit
37e843a6ef
|
@ -2197,20 +2197,20 @@ public class S3ProxyHandler {
|
|||
HttpServletResponse response, InputStream is,
|
||||
final BlobStore blobStore, String containerName, String blobName,
|
||||
String uploadId) throws IOException, S3Exception {
|
||||
final MultipartUpload mpu;
|
||||
BlobMetadata metadata;
|
||||
PutOptions options;
|
||||
if (Quirks.MULTIPART_REQUIRES_STUB.contains(getBlobStoreType(
|
||||
blobStore))) {
|
||||
Blob stubBlob = blobStore.getBlob(containerName, uploadId);
|
||||
metadata = blobStore.getBlob(containerName, uploadId).getMetadata();
|
||||
BlobAccess access = blobStore.getBlobAccess(containerName,
|
||||
uploadId);
|
||||
mpu = MultipartUpload.create(containerName,
|
||||
blobName, uploadId, stubBlob.getMetadata(),
|
||||
new PutOptions().setBlobAccess(access));
|
||||
options = new PutOptions().setBlobAccess(access);
|
||||
} else {
|
||||
mpu = MultipartUpload.create(containerName,
|
||||
blobName, uploadId, new MutableBlobMetadataImpl(),
|
||||
new PutOptions());
|
||||
metadata = new MutableBlobMetadataImpl();
|
||||
options = new PutOptions();
|
||||
}
|
||||
final MultipartUpload mpu = MultipartUpload.create(containerName,
|
||||
blobName, uploadId, metadata, options);
|
||||
|
||||
// List parts to get part sizes and to map multiple Azure parts
|
||||
// into single parts.
|
||||
|
@ -2219,7 +2219,6 @@ public class S3ProxyHandler {
|
|||
for (MultipartPart part : blobStore.listMultipartUpload(mpu)) {
|
||||
builder.put(part.partNumber(), part);
|
||||
}
|
||||
ImmutableMap<Integer, MultipartPart> partsByListing = builder.build();
|
||||
|
||||
final List<MultipartPart> parts = new ArrayList<>();
|
||||
String blobStoreType = getBlobStoreType(blobStore);
|
||||
|
@ -2228,6 +2227,28 @@ public class S3ProxyHandler {
|
|||
for (MultipartPart part : blobStore.listMultipartUpload(mpu)) {
|
||||
parts.add(part);
|
||||
}
|
||||
} else if (blobStoreType.equals("google-cloud-storage")) {
|
||||
// GCS only supports 32 parts but we can support up to 1024 by
|
||||
// recursively combining objects.
|
||||
for (int partNumber = 1;; ++partNumber) {
|
||||
MultipartUpload mpu2 = MultipartUpload.create(
|
||||
containerName,
|
||||
String.format("%s_%08d", mpu.id(), partNumber),
|
||||
String.format("%s_%08d", mpu.id(), partNumber),
|
||||
metadata, options);
|
||||
List<MultipartPart> subParts = blobStore.listMultipartUpload(
|
||||
mpu2);
|
||||
if (subParts.isEmpty()) {
|
||||
break;
|
||||
}
|
||||
long partSize = 0;
|
||||
for (MultipartPart part : subParts) {
|
||||
partSize += part.partSize();
|
||||
}
|
||||
String eTag = blobStore.completeMultipartUpload(mpu2, subParts);
|
||||
parts.add(MultipartPart.create(
|
||||
partNumber, partSize, eTag, /*lastModified=*/ null));
|
||||
}
|
||||
} else {
|
||||
CompleteMultipartUploadRequest cmu;
|
||||
try {
|
||||
|
@ -2244,6 +2265,9 @@ public class S3ProxyHandler {
|
|||
requestParts.put(part.partNumber, part.eTag);
|
||||
}
|
||||
}
|
||||
|
||||
ImmutableMap<Integer, MultipartPart> partsByListing =
|
||||
builder.build();
|
||||
for (Iterator<Map.Entry<Integer, String>> it =
|
||||
requestParts.entrySet().iterator(); it.hasNext();) {
|
||||
Map.Entry<Integer, String> entry = it.next();
|
||||
|
@ -2722,6 +2746,15 @@ public class S3ProxyHandler {
|
|||
"ArgumentValue", partNumberString));
|
||||
}
|
||||
|
||||
// GCS only supports 32 parts so partition MPU into 32-part chunks.
|
||||
String blobStoreType = getBlobStoreType(blobStore);
|
||||
if (blobStoreType.equals("google-cloud-storage")) {
|
||||
// fix up 1-based part numbers
|
||||
uploadId = String.format(
|
||||
"%s_%08d", uploadId, ((partNumber - 1) / 32) + 1);
|
||||
partNumber = ((partNumber - 1) % 32) + 1;
|
||||
}
|
||||
|
||||
// TODO: how to reconstruct original mpu?
|
||||
BlobMetadata blobMetadata;
|
||||
if (Quirks.MULTIPART_REQUIRES_STUB.contains(getBlobStoreType(
|
||||
|
|
Ładowanie…
Reference in New Issue