diff --git a/README.md b/README.md index ac982be..b63bcbd 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,7 @@ Maven Central hosts S3Proxy artifacts and the wiki has * atmos * aws-s3 (Amazon-only) * azureblob +* azureblob-sdk (newer but incomplete) * b2 * filesystem (on-disk storage) * google-cloud-storage diff --git a/pom.xml b/pom.xml index 944f395..51e2cdc 100644 --- a/pom.xml +++ b/pom.xml @@ -404,6 +404,26 @@ logback-classic 1.5.8 + + com.azure + azure-storage-blob + 12.28.0 + + + com.azure + azure-identity + 1.13.3 + + + com.google.auto.service + auto-service + 1.0-rc3 + + + com.google.guava + guava + 32.0.0-jre + javax.xml.bind jaxb-api diff --git a/src/main/java/org/gaul/s3proxy/S3ErrorCode.java b/src/main/java/org/gaul/s3proxy/S3ErrorCode.java index e139960..8e0a248 100644 --- a/src/main/java/org/gaul/s3proxy/S3ErrorCode.java +++ b/src/main/java/org/gaul/s3proxy/S3ErrorCode.java @@ -26,7 +26,7 @@ import jakarta.servlet.http.HttpServletResponse; * List of S3 error codes. Reference: * http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html */ -enum S3ErrorCode { +public enum S3ErrorCode { ACCESS_DENIED(HttpServletResponse.SC_FORBIDDEN, "Forbidden"), BAD_DIGEST(HttpServletResponse.SC_BAD_REQUEST, "Bad Request"), BUCKET_ALREADY_EXISTS(HttpServletResponse.SC_FORBIDDEN, @@ -44,6 +44,8 @@ enum S3ErrorCode { "Your proposed upload is smaller than the minimum allowed object" + " size. Each part must be at least 5 MB in size, except the last" + " part."), + INTERNAL_ERROR(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, + "An internal error occurred. Try again."), INVALID_ACCESS_KEY_ID(HttpServletResponse.SC_FORBIDDEN, "Forbidden"), INVALID_ARGUMENT(HttpServletResponse.SC_BAD_REQUEST, "Bad Request"), INVALID_BUCKET_NAME(HttpServletResponse.SC_BAD_REQUEST, diff --git a/src/main/java/org/gaul/s3proxy/S3ProxyHandlerJetty.java b/src/main/java/org/gaul/s3proxy/S3ProxyHandlerJetty.java index 9fea5da..f0c1f7d 100644 --- a/src/main/java/org/gaul/s3proxy/S3ProxyHandlerJetty.java +++ b/src/main/java/org/gaul/s3proxy/S3ProxyHandlerJetty.java @@ -22,6 +22,7 @@ import java.util.concurrent.TimeoutException; import javax.annotation.Nullable; +import com.azure.storage.blob.models.BlobStorageException; import com.google.common.collect.ImmutableMap; import com.google.common.net.HttpHeaders; @@ -30,6 +31,7 @@ import jakarta.servlet.http.HttpServletResponse; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.handler.AbstractHandler; +import org.gaul.s3proxy.azureblob.AzureBlobStore; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.ContainerNotFoundException; import org.jclouds.blobstore.KeyNotFoundException; @@ -79,6 +81,12 @@ final class S3ProxyHandlerJetty extends AbstractHandler { handler.doHandle(baseRequest, request, response, is); baseRequest.setHandled(true); + } catch (BlobStorageException bse) { + S3ErrorCode code = AzureBlobStore.toS3ErrorCode(bse.getErrorCode()); + handler.sendSimpleErrorResponse(request, response, code, + code.getMessage(), ImmutableMap.of()); + baseRequest.setHandled(true); + return; } catch (ContainerNotFoundException cnfe) { S3ErrorCode code = S3ErrorCode.NO_SUCH_BUCKET; handler.sendSimpleErrorResponse(request, response, code, diff --git a/src/main/java/org/gaul/s3proxy/azureblob/AzureBlobApiMetadata.java b/src/main/java/org/gaul/s3proxy/azureblob/AzureBlobApiMetadata.java new file mode 100644 index 0000000..bb46ee4 --- /dev/null +++ b/src/main/java/org/gaul/s3proxy/azureblob/AzureBlobApiMetadata.java @@ -0,0 +1,97 @@ +/* + * Copyright 2014-2021 Andrew Gaul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gaul.s3proxy.azureblob; + +import java.net.URI; +import java.util.Properties; + +import com.google.common.collect.ImmutableSet; +import com.google.inject.Module; + +import org.jclouds.azure.storage.config.AuthType; +import org.jclouds.azure.storage.config.AzureStorageProperties; +import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.blobstore.reference.BlobStoreConstants; +import org.jclouds.reflect.Reflection2; +import org.jclouds.rest.internal.BaseHttpApiMetadata; + + +public final class AzureBlobApiMetadata extends BaseHttpApiMetadata { + public AzureBlobApiMetadata() { + this(builder()); + } + + protected AzureBlobApiMetadata(Builder builder) { + super(builder); + } + + private static Builder builder() { + return new Builder(); + } + + @Override + public Builder toBuilder() { + return builder().fromApiMetadata(this); + } + + public static Properties defaultProperties() { + Properties properties = BaseHttpApiMetadata.defaultProperties(); + properties.setProperty(BlobStoreConstants.PROPERTY_USER_METADATA_PREFIX, + "x-ms-meta-"); + properties.setProperty(AzureStorageProperties.AUTH_TYPE, + AuthType.AZURE_KEY.toString()); + properties.setProperty(AzureStorageProperties.ACCOUNT, ""); + properties.setProperty(AzureStorageProperties.TENANT_ID, ""); + return properties; + } + + // Fake API client + private interface AzureBlobClient { + } + + public static final class Builder + extends BaseHttpApiMetadata.Builder { + protected Builder() { + super(AzureBlobClient.class); + id("azureblob-sdk") + .name("Microsoft Azure Blob Service API") + .identityName("Account Name") + .credentialName("Access Key") + // TODO: update + .version("2017-11-09") + .defaultEndpoint( + "https://${jclouds.identity}.blob.core.windows.net") + .documentation(URI.create( + "https://learn.microsoft.com/en-us/rest/api/" + + "storageservices/Blob-Service-REST-API")) + .defaultProperties(AzureBlobApiMetadata.defaultProperties()) + .view(Reflection2.typeToken(BlobStoreContext.class)) + .defaultModules(ImmutableSet.>of( + AzureBlobStoreContextModule.class)); + } + + @Override + public AzureBlobApiMetadata build() { + return new AzureBlobApiMetadata(this); + } + + @Override + protected Builder self() { + return this; + } + } +} diff --git a/src/main/java/org/gaul/s3proxy/azureblob/AzureBlobProviderMetadata.java b/src/main/java/org/gaul/s3proxy/azureblob/AzureBlobProviderMetadata.java new file mode 100644 index 0000000..0c3a7f8 --- /dev/null +++ b/src/main/java/org/gaul/s3proxy/azureblob/AzureBlobProviderMetadata.java @@ -0,0 +1,91 @@ +/* + * Copyright 2014-2021 Andrew Gaul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gaul.s3proxy.azureblob; + +import java.net.URI; +import java.util.Properties; + +import com.google.auto.service.AutoService; + +import org.jclouds.azure.storage.config.AzureStorageProperties; +import org.jclouds.oauth.v2.config.CredentialType; +import org.jclouds.oauth.v2.config.OAuthProperties; +import org.jclouds.providers.ProviderMetadata; +import org.jclouds.providers.internal.BaseProviderMetadata; + +/** + * Implementation of org.jclouds.types.ProviderMetadata for Microsoft Azure + * Blob Service. + */ +@AutoService(ProviderMetadata.class) +public final class AzureBlobProviderMetadata extends BaseProviderMetadata { + public AzureBlobProviderMetadata() { + super(builder()); + } + + public AzureBlobProviderMetadata(Builder builder) { + super(builder); + } + + public static Builder builder() { + return new Builder(); + } + + @Override + public Builder toBuilder() { + return builder().fromProviderMetadata(this); + } + + public static Properties defaultProperties() { + Properties properties = new Properties(); + properties.put("oauth.endpoint", "https://login.microsoft.com/${" + + AzureStorageProperties.TENANT_ID + "}/oauth2/token"); + properties.put(OAuthProperties.RESOURCE, "https://storage.azure.com"); + properties.put(OAuthProperties.CREDENTIAL_TYPE, + CredentialType.CLIENT_CREDENTIALS_SECRET.toString()); + properties.put(AzureStorageProperties.ACCOUNT, "${jclouds.identity}"); + return properties; + } + public static final class Builder extends BaseProviderMetadata.Builder { + protected Builder() { + id("azureblob-sdk") + .name("Microsoft Azure Blob Service") + .apiMetadata(new AzureBlobApiMetadata()) + .endpoint("https://${" + AzureStorageProperties.ACCOUNT + + "}.blob.core.windows.net") + .homepage(URI.create( + "http://www.microsoft.com/windowsazure/storage/")) + .console(URI.create("https://windows.azure.com/default.aspx")) + .linkedServices("azureblob", "azurequeue", "azuretable") + .iso3166Codes("US-TX", "US-IL", "IE-D", "SG", "NL-NH", "HK") + .defaultProperties( + AzureBlobProviderMetadata.defaultProperties()); + } + + @Override + public AzureBlobProviderMetadata build() { + return new AzureBlobProviderMetadata(this); + } + + @Override + public Builder fromProviderMetadata( + ProviderMetadata in) { + super.fromProviderMetadata(in); + return this; + } + } +} diff --git a/src/main/java/org/gaul/s3proxy/azureblob/AzureBlobStore.java b/src/main/java/org/gaul/s3proxy/azureblob/AzureBlobStore.java new file mode 100644 index 0000000..bb0e953 --- /dev/null +++ b/src/main/java/org/gaul/s3proxy/azureblob/AzureBlobStore.java @@ -0,0 +1,499 @@ +/* + * Copyright 2014-2021 Andrew Gaul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gaul.s3proxy.azureblob; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; +import java.util.Base64; +import java.util.Date; +import java.util.List; +import java.util.Set; +import java.util.UUID; + +import com.azure.core.credential.AzureNamedKeyCredential; +import com.azure.storage.blob.BlobServiceClient; +import com.azure.storage.blob.BlobServiceClientBuilder; +import com.azure.storage.blob.models.AccessTier; +import com.azure.storage.blob.models.BlobErrorCode; +import com.azure.storage.blob.models.BlobHttpHeaders; +import com.azure.storage.blob.models.BlobListDetails; +import com.azure.storage.blob.models.BlobProperties; +import com.azure.storage.blob.models.BlobSignedIdentifier; +import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.models.BlockListType; +import com.azure.storage.blob.models.ListBlobsOptions; +import com.azure.storage.blob.models.PublicAccessType; +import com.azure.storage.blob.options.BlobContainerCreateOptions; +import com.azure.storage.blob.options.BlockBlobSimpleUploadOptions; +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.primitives.Ints; + +import jakarta.inject.Inject; +import jakarta.inject.Singleton; + +import org.gaul.s3proxy.S3ErrorCode; +import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.domain.BlobAccess; +import org.jclouds.blobstore.domain.BlobMetadata; +import org.jclouds.blobstore.domain.ContainerAccess; +import org.jclouds.blobstore.domain.MultipartPart; +import org.jclouds.blobstore.domain.MultipartUpload; +import org.jclouds.blobstore.domain.PageSet; +import org.jclouds.blobstore.domain.StorageMetadata; +import org.jclouds.blobstore.domain.StorageType; +import org.jclouds.blobstore.domain.Tier; +import org.jclouds.blobstore.domain.internal.BlobBuilderImpl; +import org.jclouds.blobstore.domain.internal.BlobMetadataImpl; +import org.jclouds.blobstore.domain.internal.PageSetImpl; +import org.jclouds.blobstore.domain.internal.StorageMetadataImpl; +import org.jclouds.blobstore.internal.BaseBlobStore; +import org.jclouds.blobstore.options.CopyOptions; +import org.jclouds.blobstore.options.CreateContainerOptions; +import org.jclouds.blobstore.options.GetOptions; +import org.jclouds.blobstore.options.ListContainerOptions; +import org.jclouds.blobstore.options.PutOptions; +import org.jclouds.blobstore.util.BlobUtils; +import org.jclouds.collect.Memoized; +import org.jclouds.domain.Credentials; +import org.jclouds.domain.Location; +import org.jclouds.io.ContentMetadata; +import org.jclouds.io.ContentMetadataBuilder; +import org.jclouds.io.Payload; +import org.jclouds.io.PayloadSlicer; + +@Singleton +public final class AzureBlobStore extends BaseBlobStore { + private final BlobServiceClient blobServiceClient; + + @Inject + AzureBlobStore(BlobStoreContext context, BlobUtils blobUtils, + Supplier defaultLocation, + @Memoized Supplier> locations, + PayloadSlicer slicer, + @org.jclouds.location.Provider Supplier creds) { + super(context, blobUtils, defaultLocation, locations, slicer); + var cred = creds.get(); + blobServiceClient = new BlobServiceClientBuilder() + .credential(new AzureNamedKeyCredential( + cred.identity, cred.credential)) + .endpoint("https://" + cred.identity + ".blob.core.windows.net") + .buildClient(); + } + + @Override + public PageSet list() { + var set = ImmutableSet.builder(); + for (var container : blobServiceClient.listBlobContainers()) { + set.add(new StorageMetadataImpl(StorageType.CONTAINER, /*id=*/ null, + container.getName(), /*location=*/ null, /*uri=*/ null, + /*eTag=*/ null, /*creationDate=*/ null, + toDate(container.getProperties().getLastModified()), + ImmutableMap.of(), /*size=*/ null, + Tier.STANDARD)); + } + return new PageSetImpl(set.build(), null); + } + + @Override + public PageSet list(String container, + ListContainerOptions options) { + var client = blobServiceClient.getBlobContainerClient(container); + var azureOptions = new ListBlobsOptions(); + azureOptions.setPrefix(options.getPrefix()); + azureOptions.setMaxResultsPerPage(options.getMaxResults()); + var marker = options.getMarker() != null ? + URLDecoder.decode(options.getMarker(), StandardCharsets.UTF_8) : + null; + + var set = ImmutableSet.builder(); + var page = client.listBlobsByHierarchy( + options.getDelimiter(), azureOptions, /*timeout=*/ null) + .iterableByPage().iterator().next(); + for (var blob : page.getValue()) { + var properties = blob.getProperties(); + if (blob.isPrefix()) { + set.add(new StorageMetadataImpl(StorageType.RELATIVE_PATH, + /*id=*/ null, blob.getName(), /*location=*/ null, + /*uri=*/ null, /*eTag=*/ null, + /*creationDate=*/ null, + /*lastModified=*/ null, + ImmutableMap.of(), + /*size=*/ null, + toTier(properties.getAccessTier()))); + } else { + set.add(new StorageMetadataImpl(StorageType.BLOB, + /*id=*/ null, blob.getName(), /*location=*/ null, + /*uri=*/ null, properties.getETag(), + toDate(properties.getCreationTime()), + toDate(properties.getLastModified()), + ImmutableMap.of(), + properties.getContentLength(), + toTier(properties.getAccessTier()))); + } + } + + return new PageSetImpl(set.build(), + page.getContinuationToken()); + } + + @Override + public boolean containerExists(String container) { + var client = blobServiceClient.getBlobContainerClient(container); + return client.exists(); + } + + @Override + public boolean createContainerInLocation(Location location, + String container) { + return createContainerInLocation(location, container, + new CreateContainerOptions()); + } + + @Override + public boolean createContainerInLocation(Location location, + String container, CreateContainerOptions options) { + try { + var azureOptions = new BlobContainerCreateOptions(); + if (options.isPublicRead()) { + azureOptions.setPublicAccessType(PublicAccessType.CONTAINER); + } + blobServiceClient.createBlobContainerIfNotExistsWithResponse( + container, azureOptions, /*context=*/ null); + } catch (BlobStorageException bse) { + if (bse.getErrorCode() == BlobErrorCode.CONTAINER_ALREADY_EXISTS) { + return false; + } + throw bse; + } + return true; + } + + @Override + public void deleteContainer(String container) { + blobServiceClient.deleteBlobContainer(container); + } + + @Override + public boolean blobExists(String container, String key) { + var client = blobServiceClient.getBlobContainerClient(container) + .getBlobClient(key); + return client.exists(); + } + + @Override + public Blob getBlob(String container, String key, GetOptions options) { + var client = blobServiceClient.getBlobContainerClient(container) + .getBlobClient(key); + var blobStream = client.openInputStream(); + var properties = blobStream.getProperties(); + var expires = properties.getExpiresOn(); + return new BlobBuilderImpl() + .name(key) + .userMetadata(properties.getMetadata()) + .payload(blobStream) + .cacheControl(properties.getCacheControl()) + .contentDisposition(properties.getContentDisposition()) + .contentEncoding(properties.getContentEncoding()) + .contentLanguage(properties.getContentLanguage()) + .contentLength(properties.getBlobSize()) + .contentType(properties.getContentType()) + .expires(expires != null ? toDate(expires) : null) + .build(); + } + + @Override + public String putBlob(String container, Blob blob) { + return putBlob(container, blob, new PutOptions()); + } + + @Override + public String putBlob(String container, Blob blob, PutOptions options) { + var client = blobServiceClient.getBlobContainerClient(container) + .getBlobClient(blob.getMetadata().getName()) + .getBlockBlobClient(); + try (var is = blob.getPayload().openStream()) { + var azureOptions = new BlockBlobSimpleUploadOptions(is, + blob.getMetadata().getContentMetadata().getContentLength()); + azureOptions.setMetadata(blob.getMetadata().getUserMetadata()); + + // TODO: Expires? + var blobHttpHeaders = new BlobHttpHeaders(); + var contentMetadata = blob.getMetadata().getContentMetadata(); + blobHttpHeaders.setCacheControl(contentMetadata.getCacheControl()); + blobHttpHeaders.setContentDisposition( + contentMetadata.getContentDisposition()); + blobHttpHeaders.setContentEncoding( + contentMetadata.getContentEncoding()); + blobHttpHeaders.setContentLanguage( + contentMetadata.getContentLanguage()); + blobHttpHeaders.setContentType(contentMetadata.getContentType()); + azureOptions.setHeaders(blobHttpHeaders); + if (blob.getMetadata().getTier() != Tier.STANDARD) { + azureOptions.setTier(toAccessTier( + blob.getMetadata().getTier())); + } + + var blockBlobItem = client.uploadWithResponse( + azureOptions, /*timeout=*/ null, /*context=*/ null); + return blockBlobItem.getValue().getETag(); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + } + + @Override + public String copyBlob(String fromContainer, String fromName, + String toContainer, String toName, CopyOptions options) { + throw new UnsupportedOperationException("not yet implemented"); + } + + @Override + public void removeBlob(String container, String key) { + var client = blobServiceClient.getBlobContainerClient(container) + .getBlobClient(key); + try { + client.delete(); + } catch (BlobStorageException bse) { + if (bse.getErrorCode() != BlobErrorCode.BLOB_NOT_FOUND) { + throw bse; + } + } + } + + @Override + public BlobMetadata blobMetadata(String container, String key) { + var client = blobServiceClient.getBlobContainerClient(container) + .getBlobClient(key); + var properties = client.getProperties(); + return new BlobMetadataImpl(/*id=*/ null, key, /*location=*/ null, + /*uri=*/ null, properties.getETag(), + toDate(properties.getCreationTime()), + toDate(properties.getLastModified()), + properties.getMetadata(), /*publicUri=*/ null, container, + toContentMetadata(properties), + properties.getBlobSize(), toTier(properties.getAccessTier())); + } + + @Override + protected boolean deleteAndVerifyContainerGone(String container) { + blobServiceClient.deleteBlobContainer(container); + return true; + } + + @Override + public ContainerAccess getContainerAccess(String container) { + var client = blobServiceClient.getBlobContainerClient(container); + return client.getAccessPolicy().getBlobAccessType() == + PublicAccessType.CONTAINER ? + ContainerAccess.PUBLIC_READ : + ContainerAccess.PRIVATE; + } + + @Override + public void setContainerAccess(String container, ContainerAccess access) { + var client = blobServiceClient.getBlobContainerClient(container); + var publicAccess = access == ContainerAccess.PUBLIC_READ ? + PublicAccessType.CONTAINER : PublicAccessType.BLOB; + client.setAccessPolicy(publicAccess, + ImmutableList.of()); + } + + @Override + public BlobAccess getBlobAccess(String container, String key) { + return BlobAccess.PRIVATE; + } + + @Override + public void setBlobAccess(String container, String key, BlobAccess access) { + throw new UnsupportedOperationException("unsupported in Azure"); + } + + @Override + public MultipartUpload initiateMultipartUpload(String container, + BlobMetadata blobMetadata, PutOptions options) { + String uploadId = UUID.randomUUID().toString(); + return MultipartUpload.create(container, blobMetadata.getName(), + uploadId, blobMetadata, options); + } + + @Override + public void abortMultipartUpload(MultipartUpload mpu) { + // Azure automatically removes uncommitted blocks after 7 days. + } + + @Override + public String completeMultipartUpload(MultipartUpload mpu, + List parts) { + var client = blobServiceClient + .getBlobContainerClient(mpu.containerName()) + .getBlobClient(mpu.blobName()) + .getBlockBlobClient(); + var blocks = ImmutableList.builder(); + for (var part : parts) { + blocks.add(makeBlockId(part.partNumber())); + } + var blockBlobItem = client.commitBlockList(blocks.build(), + /*overwrite=*/ true); + return blockBlobItem.getETag(); + } + + @Override + public MultipartPart uploadMultipartPart(MultipartUpload mpu, + int partNumber, Payload payload) { + var client = blobServiceClient + .getBlobContainerClient(mpu.containerName()) + .getBlobClient(mpu.blobName()) + .getBlockBlobClient(); + var blockId = makeBlockId(partNumber); + var length = payload.getContentMetadata().getContentLength(); + try (var is = payload.openStream()) { + client.stageBlock(blockId, is, length); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + String eTag = ""; // putBlock does not return ETag + Date lastModified = null; // putBlob does not return Last-Modified + return MultipartPart.create(partNumber, length, eTag, lastModified); + } + + @Override + public List listMultipartUpload(MultipartUpload mpu) { + var client = blobServiceClient + .getBlobContainerClient(mpu.containerName()) + .getBlobClient(mpu.blobName()) + .getBlockBlobClient(); + var blockList = client.listBlocks(BlockListType.ALL); + var parts = ImmutableList.builder(); + for (var properties : blockList.getUncommittedBlocks()) { + int partNumber = Ints.fromByteArray(Base64.getDecoder().decode( + properties.getName())); + String eTag = ""; // listBlocks does not return ETag + Date lastModified = null; // listBlocks does not return LastModified + parts.add(MultipartPart.create(partNumber, properties.getSizeLong(), + eTag, lastModified)); + } + return parts.build(); + } + + @Override + public List listMultipartUploads(String container) { + var client = blobServiceClient.getBlobContainerClient(container); + var azureOptions = new ListBlobsOptions(); + var details = new BlobListDetails(); + details.setRetrieveUncommittedBlobs(true); + azureOptions.setDetails(details); + + var builder = ImmutableList.builder(); + for (var blob : client.listBlobs(azureOptions, + /*continuationToken=*/ null, /*timeout=*/ null)) { + var properties = blob.getProperties(); + // only uncommitted blobs lack ETags + if (properties.getETag() != null) { + continue; + } + // TODO: bogus uploadId + String uploadId = UUID.randomUUID().toString(); + builder.add(MultipartUpload.create(container, blob.getName(), + uploadId, null, null)); + } + + return builder.build(); + } + + @Override + public long getMinimumMultipartPartSize() { + return 1; + } + + @Override + public long getMaximumMultipartPartSize() { + return 100 * 1024 * 1024; + } + + @Override + public int getMaximumNumberOfParts() { + return 50 * 1000; + } + + @Override + public InputStream streamBlob(String container, String name) { + throw new UnsupportedOperationException("not yet implemented"); + } + + // TODO: handle more error codes + public static S3ErrorCode toS3ErrorCode(BlobErrorCode code) { + if (code.equals(BlobErrorCode.CONTAINER_NOT_FOUND)) { + return S3ErrorCode.NO_SUCH_BUCKET; + } else { + return S3ErrorCode.INTERNAL_ERROR; + } + } + + private static Date toDate(OffsetDateTime time) { + return new Date(time.toInstant().toEpochMilli()); + } + + private static AccessTier toAccessTier(Tier tier) { + switch (tier) { + case ARCHIVE: + return AccessTier.ARCHIVE; + case INFREQUENT: + return AccessTier.COOL; + case STANDARD: + default: + return AccessTier.HOT; + } + } + + private static Tier toTier(AccessTier tier) { + if (tier == null) { + return Tier.STANDARD; + } else if (tier.equals(AccessTier.ARCHIVE)) { + return Tier.ARCHIVE; + } else if (tier.equals(AccessTier.COLD) || + tier.equals(AccessTier.COOL)) { + return Tier.INFREQUENT; + } else { + return Tier.STANDARD; + } + } + + private static ContentMetadata toContentMetadata( + BlobProperties properties) { + var expires = properties.getExpiresOn(); + return ContentMetadataBuilder.create() + .cacheControl(properties.getCacheControl()) + .contentDisposition(properties.getContentDisposition()) + .contentEncoding(properties.getContentEncoding()) + .contentLanguage(properties.getContentLanguage()) + .contentLength(properties.getBlobSize()) + .contentType(properties.getContentType()) + .expires(expires != null ? toDate(expires) : null) + .build(); + } + + private static String makeBlockId(int partNumber) { + return Base64.getEncoder().encodeToString(Ints.toByteArray(partNumber)); + } +} diff --git a/src/main/java/org/gaul/s3proxy/azureblob/AzureBlobStoreContextModule.java b/src/main/java/org/gaul/s3proxy/azureblob/AzureBlobStoreContextModule.java new file mode 100644 index 0000000..75bc38d --- /dev/null +++ b/src/main/java/org/gaul/s3proxy/azureblob/AzureBlobStoreContextModule.java @@ -0,0 +1,31 @@ +/* + * Copyright 2014-2021 Andrew Gaul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gaul.s3proxy.azureblob; + +import com.google.inject.AbstractModule; +import com.google.inject.Scopes; + +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.attr.ConsistencyModel; + +public final class AzureBlobStoreContextModule extends AbstractModule { + @Override + protected void configure() { + bind(ConsistencyModel.class).toInstance(ConsistencyModel.STRICT); + bind(BlobStore.class).to(AzureBlobStore.class).in(Scopes.SINGLETON); + } +}