kopia lustrzana https://github.com/gaul/s3proxy
Fix NullBlobStore multipart upload
Complete multipart upload lists the parts to get their sizes but the NullBlobStore reported 8 bytes for the physical size instead of the logical size embedded with the part. There is no readable space to store the logical size so we create a companion single-part object for every part.pull/293/head
rodzic
c07adfc1a9
commit
8223f443c7
|
@ -93,6 +93,7 @@ final class NullBlobStore extends ForwardingBlobStore {
|
|||
payload.getContentMetadata().setContentLength(length);
|
||||
payload.getContentMetadata().setContentMD5((HashCode) null);
|
||||
blob.setPayload(payload);
|
||||
blob.getMetadata().setSize(length);
|
||||
return blob;
|
||||
}
|
||||
|
||||
|
@ -140,17 +141,33 @@ final class NullBlobStore extends ForwardingBlobStore {
|
|||
long length = 0;
|
||||
for (MultipartPart part : parts) {
|
||||
length += part.partSize();
|
||||
super.removeBlob(mpu.containerName(), mpu.id() + "-" +
|
||||
part.partNumber());
|
||||
}
|
||||
|
||||
byte[] array = Longs.toByteArray(length);
|
||||
ByteSourcePayload payload = new ByteSourcePayload(
|
||||
ByteSource.wrap(array));
|
||||
payload.getContentMetadata().setContentLength((long) array.length);
|
||||
|
||||
super.abortMultipartUpload(mpu);
|
||||
|
||||
MultipartPart part = delegate().uploadMultipartPart(mpu, 1, payload);
|
||||
MultipartUpload mpu2 = super.initiateMultipartUpload(
|
||||
mpu.containerName(), mpu.blobMetadata(), mpu.putOptions());
|
||||
|
||||
return delegate().completeMultipartUpload(mpu, ImmutableList.of(part));
|
||||
MultipartPart part = super.uploadMultipartPart(mpu2, 1, payload);
|
||||
|
||||
return super.completeMultipartUpload(mpu2, ImmutableList.of(part));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void abortMultipartUpload(MultipartUpload mpu) {
|
||||
for (MultipartPart part : super.listMultipartUpload(mpu)) {
|
||||
super.removeBlob(mpu.containerName(), mpu.id() + "-" +
|
||||
part.partNumber());
|
||||
}
|
||||
|
||||
super.abortMultipartUpload(mpu);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -170,18 +187,29 @@ final class NullBlobStore extends ForwardingBlobStore {
|
|||
newPayload.getContentMetadata().setContentLength((long) array.length);
|
||||
newPayload.getContentMetadata().setContentMD5((HashCode) null);
|
||||
|
||||
// create a single-part object which contains the logical length which
|
||||
// list and complete will read later
|
||||
Blob blob = blobBuilder(mpu.id() + "-" + partNumber)
|
||||
.payload(newPayload)
|
||||
.build();
|
||||
super.putBlob(mpu.containerName(), blob);
|
||||
|
||||
MultipartPart part = super.uploadMultipartPart(mpu, partNumber,
|
||||
newPayload);
|
||||
return MultipartPart.create(part.partNumber(), length, part.partETag(),
|
||||
part.lastModified());
|
||||
}
|
||||
|
||||
// Cannot read parts to get the embedded size so return zero instead.
|
||||
@Override
|
||||
public List<MultipartPart> listMultipartUpload(MultipartUpload mpu) {
|
||||
ImmutableList.Builder<MultipartPart> builder = ImmutableList.builder();
|
||||
for (MultipartPart part : super.listMultipartUpload(mpu)) {
|
||||
builder.add(MultipartPart.create(part.partNumber(), 0,
|
||||
// get real blob size from stub blob
|
||||
Blob blob = getBlob(mpu.containerName(),
|
||||
mpu.id() + "-" + part.partNumber());
|
||||
long length = blob.getPayload().getContentMetadata()
|
||||
.getContentLength();
|
||||
builder.add(MultipartPart.create(part.partNumber(), length,
|
||||
part.partETag(), part.lastModified()));
|
||||
}
|
||||
return builder.build();
|
||||
|
|
|
@ -141,14 +141,15 @@ public final class NullBlobStoreTest {
|
|||
|
||||
List<MultipartPart> parts = nullBlobStore.listMultipartUpload(mpu);
|
||||
assertThat(parts.get(0).partNumber()).isEqualTo(1);
|
||||
assertThat(parts.get(0).partSize()).isZero();
|
||||
assertThat(parts.get(0).partSize()).isEqualTo(byteSource1.size());
|
||||
assertThat(parts.get(0).partETag()).isEqualTo(part1.partETag());
|
||||
assertThat(parts.get(1).partNumber()).isEqualTo(2);
|
||||
assertThat(parts.get(1).partSize()).isZero();
|
||||
assertThat(parts.get(1).partSize()).isEqualTo(byteSource2.size());
|
||||
assertThat(parts.get(1).partETag()).isEqualTo(part2.partETag());
|
||||
|
||||
nullBlobStore.completeMultipartUpload(mpu, ImmutableList.of(part1,
|
||||
part2));
|
||||
assertThat(nullBlobStore.listMultipartUpload(mpu)).hasSize(2);
|
||||
|
||||
nullBlobStore.completeMultipartUpload(mpu, parts);
|
||||
|
||||
Blob newBlob = nullBlobStore.getBlob(containerName, blobName);
|
||||
validateBlobMetadata(newBlob.getMetadata());
|
||||
|
@ -162,6 +163,9 @@ public final class NullBlobStoreTest {
|
|||
ByteStreams.nullOutputStream());
|
||||
assertThat(actualLength).isEqualTo(expectedLength);
|
||||
}
|
||||
|
||||
nullBlobStore.removeBlob(containerName, blobName);
|
||||
assertThat(nullBlobStore.list(containerName)).isEmpty();
|
||||
}
|
||||
|
||||
private static String createRandomContainerName() {
|
||||
|
|
Ładowanie…
Reference in New Issue