kopia lustrzana https://github.com/gaul/s3proxy
Allow GCS backend to upload more than 32 parts
This recursively combines up to 32 sets of 32 parts, allowing 1024 part multipart uploads. Fixes #330.pull/361/head
rodzic
29413b12a7
commit
37e843a6ef
|
@ -2197,20 +2197,20 @@ public class S3ProxyHandler {
|
||||||
HttpServletResponse response, InputStream is,
|
HttpServletResponse response, InputStream is,
|
||||||
final BlobStore blobStore, String containerName, String blobName,
|
final BlobStore blobStore, String containerName, String blobName,
|
||||||
String uploadId) throws IOException, S3Exception {
|
String uploadId) throws IOException, S3Exception {
|
||||||
final MultipartUpload mpu;
|
BlobMetadata metadata;
|
||||||
|
PutOptions options;
|
||||||
if (Quirks.MULTIPART_REQUIRES_STUB.contains(getBlobStoreType(
|
if (Quirks.MULTIPART_REQUIRES_STUB.contains(getBlobStoreType(
|
||||||
blobStore))) {
|
blobStore))) {
|
||||||
Blob stubBlob = blobStore.getBlob(containerName, uploadId);
|
metadata = blobStore.getBlob(containerName, uploadId).getMetadata();
|
||||||
BlobAccess access = blobStore.getBlobAccess(containerName,
|
BlobAccess access = blobStore.getBlobAccess(containerName,
|
||||||
uploadId);
|
uploadId);
|
||||||
mpu = MultipartUpload.create(containerName,
|
options = new PutOptions().setBlobAccess(access);
|
||||||
blobName, uploadId, stubBlob.getMetadata(),
|
|
||||||
new PutOptions().setBlobAccess(access));
|
|
||||||
} else {
|
} else {
|
||||||
mpu = MultipartUpload.create(containerName,
|
metadata = new MutableBlobMetadataImpl();
|
||||||
blobName, uploadId, new MutableBlobMetadataImpl(),
|
options = new PutOptions();
|
||||||
new PutOptions());
|
|
||||||
}
|
}
|
||||||
|
final MultipartUpload mpu = MultipartUpload.create(containerName,
|
||||||
|
blobName, uploadId, metadata, options);
|
||||||
|
|
||||||
// List parts to get part sizes and to map multiple Azure parts
|
// List parts to get part sizes and to map multiple Azure parts
|
||||||
// into single parts.
|
// into single parts.
|
||||||
|
@ -2219,7 +2219,6 @@ public class S3ProxyHandler {
|
||||||
for (MultipartPart part : blobStore.listMultipartUpload(mpu)) {
|
for (MultipartPart part : blobStore.listMultipartUpload(mpu)) {
|
||||||
builder.put(part.partNumber(), part);
|
builder.put(part.partNumber(), part);
|
||||||
}
|
}
|
||||||
ImmutableMap<Integer, MultipartPart> partsByListing = builder.build();
|
|
||||||
|
|
||||||
final List<MultipartPart> parts = new ArrayList<>();
|
final List<MultipartPart> parts = new ArrayList<>();
|
||||||
String blobStoreType = getBlobStoreType(blobStore);
|
String blobStoreType = getBlobStoreType(blobStore);
|
||||||
|
@ -2228,6 +2227,28 @@ public class S3ProxyHandler {
|
||||||
for (MultipartPart part : blobStore.listMultipartUpload(mpu)) {
|
for (MultipartPart part : blobStore.listMultipartUpload(mpu)) {
|
||||||
parts.add(part);
|
parts.add(part);
|
||||||
}
|
}
|
||||||
|
} else if (blobStoreType.equals("google-cloud-storage")) {
|
||||||
|
// GCS only supports 32 parts but we can support up to 1024 by
|
||||||
|
// recursively combining objects.
|
||||||
|
for (int partNumber = 1;; ++partNumber) {
|
||||||
|
MultipartUpload mpu2 = MultipartUpload.create(
|
||||||
|
containerName,
|
||||||
|
String.format("%s_%08d", mpu.id(), partNumber),
|
||||||
|
String.format("%s_%08d", mpu.id(), partNumber),
|
||||||
|
metadata, options);
|
||||||
|
List<MultipartPart> subParts = blobStore.listMultipartUpload(
|
||||||
|
mpu2);
|
||||||
|
if (subParts.isEmpty()) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
long partSize = 0;
|
||||||
|
for (MultipartPart part : subParts) {
|
||||||
|
partSize += part.partSize();
|
||||||
|
}
|
||||||
|
String eTag = blobStore.completeMultipartUpload(mpu2, subParts);
|
||||||
|
parts.add(MultipartPart.create(
|
||||||
|
partNumber, partSize, eTag, /*lastModified=*/ null));
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
CompleteMultipartUploadRequest cmu;
|
CompleteMultipartUploadRequest cmu;
|
||||||
try {
|
try {
|
||||||
|
@ -2244,6 +2265,9 @@ public class S3ProxyHandler {
|
||||||
requestParts.put(part.partNumber, part.eTag);
|
requestParts.put(part.partNumber, part.eTag);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ImmutableMap<Integer, MultipartPart> partsByListing =
|
||||||
|
builder.build();
|
||||||
for (Iterator<Map.Entry<Integer, String>> it =
|
for (Iterator<Map.Entry<Integer, String>> it =
|
||||||
requestParts.entrySet().iterator(); it.hasNext();) {
|
requestParts.entrySet().iterator(); it.hasNext();) {
|
||||||
Map.Entry<Integer, String> entry = it.next();
|
Map.Entry<Integer, String> entry = it.next();
|
||||||
|
@ -2722,6 +2746,15 @@ public class S3ProxyHandler {
|
||||||
"ArgumentValue", partNumberString));
|
"ArgumentValue", partNumberString));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GCS only supports 32 parts so partition MPU into 32-part chunks.
|
||||||
|
String blobStoreType = getBlobStoreType(blobStore);
|
||||||
|
if (blobStoreType.equals("google-cloud-storage")) {
|
||||||
|
// fix up 1-based part numbers
|
||||||
|
uploadId = String.format(
|
||||||
|
"%s_%08d", uploadId, ((partNumber - 1) / 32) + 1);
|
||||||
|
partNumber = ((partNumber - 1) % 32) + 1;
|
||||||
|
}
|
||||||
|
|
||||||
// TODO: how to reconstruct original mpu?
|
// TODO: how to reconstruct original mpu?
|
||||||
BlobMetadata blobMetadata;
|
BlobMetadata blobMetadata;
|
||||||
if (Quirks.MULTIPART_REQUIRES_STUB.contains(getBlobStoreType(
|
if (Quirks.MULTIPART_REQUIRES_STUB.contains(getBlobStoreType(
|
||||||
|
|
Ładowanie…
Reference in New Issue