Add azureblob-sdk provider backed by Azure SDK

This may replace the jclouds azureblob provider.  The implementation
lacks some error codes and options.  Fixes #606.
pull/699/head
Andrew Gaul 2024-10-01 21:35:09 -07:00
rodzic c3c9037d19
commit b33f3e2826
8 zmienionych plików z 750 dodań i 1 usunięć

Wyświetl plik

@ -74,6 +74,7 @@ Maven Central hosts S3Proxy artifacts and the wiki has
* atmos
* aws-s3 (Amazon-only)
* azureblob
* azureblob-sdk (newer but incomplete)
* b2
* filesystem (on-disk storage)
* google-cloud-storage

20
pom.xml
Wyświetl plik

@ -404,6 +404,26 @@
<artifactId>logback-classic</artifactId>
<version>1.5.8</version>
</dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-storage-blob</artifactId>
<version>12.28.0</version>
</dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-identity</artifactId>
<version>1.13.3</version>
</dependency>
<dependency>
<groupId>com.google.auto.service</groupId>
<artifactId>auto-service</artifactId>
<version>1.0-rc3</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>32.0.0-jre</version>
</dependency>
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>

Wyświetl plik

@ -26,7 +26,7 @@ import jakarta.servlet.http.HttpServletResponse;
* List of S3 error codes. Reference:
* http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
*/
enum S3ErrorCode {
public enum S3ErrorCode {
ACCESS_DENIED(HttpServletResponse.SC_FORBIDDEN, "Forbidden"),
BAD_DIGEST(HttpServletResponse.SC_BAD_REQUEST, "Bad Request"),
BUCKET_ALREADY_EXISTS(HttpServletResponse.SC_FORBIDDEN,
@ -44,6 +44,8 @@ enum S3ErrorCode {
"Your proposed upload is smaller than the minimum allowed object" +
" size. Each part must be at least 5 MB in size, except the last" +
" part."),
INTERNAL_ERROR(HttpServletResponse.SC_INTERNAL_SERVER_ERROR,
"An internal error occurred. Try again."),
INVALID_ACCESS_KEY_ID(HttpServletResponse.SC_FORBIDDEN, "Forbidden"),
INVALID_ARGUMENT(HttpServletResponse.SC_BAD_REQUEST, "Bad Request"),
INVALID_BUCKET_NAME(HttpServletResponse.SC_BAD_REQUEST,

Wyświetl plik

@ -22,6 +22,7 @@ import java.util.concurrent.TimeoutException;
import javax.annotation.Nullable;
import com.azure.storage.blob.models.BlobStorageException;
import com.google.common.collect.ImmutableMap;
import com.google.common.net.HttpHeaders;
@ -30,6 +31,7 @@ import jakarta.servlet.http.HttpServletResponse;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.handler.AbstractHandler;
import org.gaul.s3proxy.azureblob.AzureBlobStore;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.ContainerNotFoundException;
import org.jclouds.blobstore.KeyNotFoundException;
@ -79,6 +81,12 @@ final class S3ProxyHandlerJetty extends AbstractHandler {
handler.doHandle(baseRequest, request, response, is);
baseRequest.setHandled(true);
} catch (BlobStorageException bse) {
S3ErrorCode code = AzureBlobStore.toS3ErrorCode(bse.getErrorCode());
handler.sendSimpleErrorResponse(request, response, code,
code.getMessage(), ImmutableMap.<String, String>of());
baseRequest.setHandled(true);
return;
} catch (ContainerNotFoundException cnfe) {
S3ErrorCode code = S3ErrorCode.NO_SUCH_BUCKET;
handler.sendSimpleErrorResponse(request, response, code,

Wyświetl plik

@ -0,0 +1,97 @@
/*
* Copyright 2014-2021 Andrew Gaul <andrew@gaul.org>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.gaul.s3proxy.azureblob;
import java.net.URI;
import java.util.Properties;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Module;
import org.jclouds.azure.storage.config.AuthType;
import org.jclouds.azure.storage.config.AzureStorageProperties;
import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.reference.BlobStoreConstants;
import org.jclouds.reflect.Reflection2;
import org.jclouds.rest.internal.BaseHttpApiMetadata;
public final class AzureBlobApiMetadata extends BaseHttpApiMetadata {
public AzureBlobApiMetadata() {
this(builder());
}
protected AzureBlobApiMetadata(Builder builder) {
super(builder);
}
private static Builder builder() {
return new Builder();
}
@Override
public Builder toBuilder() {
return builder().fromApiMetadata(this);
}
public static Properties defaultProperties() {
Properties properties = BaseHttpApiMetadata.defaultProperties();
properties.setProperty(BlobStoreConstants.PROPERTY_USER_METADATA_PREFIX,
"x-ms-meta-");
properties.setProperty(AzureStorageProperties.AUTH_TYPE,
AuthType.AZURE_KEY.toString());
properties.setProperty(AzureStorageProperties.ACCOUNT, "");
properties.setProperty(AzureStorageProperties.TENANT_ID, "");
return properties;
}
// Fake API client
private interface AzureBlobClient {
}
public static final class Builder
extends BaseHttpApiMetadata.Builder<AzureBlobClient, Builder> {
protected Builder() {
super(AzureBlobClient.class);
id("azureblob-sdk")
.name("Microsoft Azure Blob Service API")
.identityName("Account Name")
.credentialName("Access Key")
// TODO: update
.version("2017-11-09")
.defaultEndpoint(
"https://${jclouds.identity}.blob.core.windows.net")
.documentation(URI.create(
"https://learn.microsoft.com/en-us/rest/api/" +
"storageservices/Blob-Service-REST-API"))
.defaultProperties(AzureBlobApiMetadata.defaultProperties())
.view(Reflection2.typeToken(BlobStoreContext.class))
.defaultModules(ImmutableSet.<Class<? extends Module>>of(
AzureBlobStoreContextModule.class));
}
@Override
public AzureBlobApiMetadata build() {
return new AzureBlobApiMetadata(this);
}
@Override
protected Builder self() {
return this;
}
}
}

Wyświetl plik

@ -0,0 +1,91 @@
/*
* Copyright 2014-2021 Andrew Gaul <andrew@gaul.org>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.gaul.s3proxy.azureblob;
import java.net.URI;
import java.util.Properties;
import com.google.auto.service.AutoService;
import org.jclouds.azure.storage.config.AzureStorageProperties;
import org.jclouds.oauth.v2.config.CredentialType;
import org.jclouds.oauth.v2.config.OAuthProperties;
import org.jclouds.providers.ProviderMetadata;
import org.jclouds.providers.internal.BaseProviderMetadata;
/**
* Implementation of org.jclouds.types.ProviderMetadata for Microsoft Azure
* Blob Service.
*/
@AutoService(ProviderMetadata.class)
public final class AzureBlobProviderMetadata extends BaseProviderMetadata {
public AzureBlobProviderMetadata() {
super(builder());
}
public AzureBlobProviderMetadata(Builder builder) {
super(builder);
}
public static Builder builder() {
return new Builder();
}
@Override
public Builder toBuilder() {
return builder().fromProviderMetadata(this);
}
public static Properties defaultProperties() {
Properties properties = new Properties();
properties.put("oauth.endpoint", "https://login.microsoft.com/${" +
AzureStorageProperties.TENANT_ID + "}/oauth2/token");
properties.put(OAuthProperties.RESOURCE, "https://storage.azure.com");
properties.put(OAuthProperties.CREDENTIAL_TYPE,
CredentialType.CLIENT_CREDENTIALS_SECRET.toString());
properties.put(AzureStorageProperties.ACCOUNT, "${jclouds.identity}");
return properties;
}
public static final class Builder extends BaseProviderMetadata.Builder {
protected Builder() {
id("azureblob-sdk")
.name("Microsoft Azure Blob Service")
.apiMetadata(new AzureBlobApiMetadata())
.endpoint("https://${" + AzureStorageProperties.ACCOUNT +
"}.blob.core.windows.net")
.homepage(URI.create(
"http://www.microsoft.com/windowsazure/storage/"))
.console(URI.create("https://windows.azure.com/default.aspx"))
.linkedServices("azureblob", "azurequeue", "azuretable")
.iso3166Codes("US-TX", "US-IL", "IE-D", "SG", "NL-NH", "HK")
.defaultProperties(
AzureBlobProviderMetadata.defaultProperties());
}
@Override
public AzureBlobProviderMetadata build() {
return new AzureBlobProviderMetadata(this);
}
@Override
public Builder fromProviderMetadata(
ProviderMetadata in) {
super.fromProviderMetadata(in);
return this;
}
}
}

Wyświetl plik

@ -0,0 +1,499 @@
/*
* Copyright 2014-2021 Andrew Gaul <andrew@gaul.org>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.gaul.s3proxy.azureblob;
import java.io.IOException;
import java.io.InputStream;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.util.Base64;
import java.util.Date;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import com.azure.core.credential.AzureNamedKeyCredential;
import com.azure.storage.blob.BlobServiceClient;
import com.azure.storage.blob.BlobServiceClientBuilder;
import com.azure.storage.blob.models.AccessTier;
import com.azure.storage.blob.models.BlobErrorCode;
import com.azure.storage.blob.models.BlobHttpHeaders;
import com.azure.storage.blob.models.BlobListDetails;
import com.azure.storage.blob.models.BlobProperties;
import com.azure.storage.blob.models.BlobSignedIdentifier;
import com.azure.storage.blob.models.BlobStorageException;
import com.azure.storage.blob.models.BlockListType;
import com.azure.storage.blob.models.ListBlobsOptions;
import com.azure.storage.blob.models.PublicAccessType;
import com.azure.storage.blob.options.BlobContainerCreateOptions;
import com.azure.storage.blob.options.BlockBlobSimpleUploadOptions;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.primitives.Ints;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import org.gaul.s3proxy.S3ErrorCode;
import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.BlobAccess;
import org.jclouds.blobstore.domain.BlobMetadata;
import org.jclouds.blobstore.domain.ContainerAccess;
import org.jclouds.blobstore.domain.MultipartPart;
import org.jclouds.blobstore.domain.MultipartUpload;
import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.domain.StorageType;
import org.jclouds.blobstore.domain.Tier;
import org.jclouds.blobstore.domain.internal.BlobBuilderImpl;
import org.jclouds.blobstore.domain.internal.BlobMetadataImpl;
import org.jclouds.blobstore.domain.internal.PageSetImpl;
import org.jclouds.blobstore.domain.internal.StorageMetadataImpl;
import org.jclouds.blobstore.internal.BaseBlobStore;
import org.jclouds.blobstore.options.CopyOptions;
import org.jclouds.blobstore.options.CreateContainerOptions;
import org.jclouds.blobstore.options.GetOptions;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.options.PutOptions;
import org.jclouds.blobstore.util.BlobUtils;
import org.jclouds.collect.Memoized;
import org.jclouds.domain.Credentials;
import org.jclouds.domain.Location;
import org.jclouds.io.ContentMetadata;
import org.jclouds.io.ContentMetadataBuilder;
import org.jclouds.io.Payload;
import org.jclouds.io.PayloadSlicer;
@Singleton
public final class AzureBlobStore extends BaseBlobStore {
private final BlobServiceClient blobServiceClient;
@Inject
AzureBlobStore(BlobStoreContext context, BlobUtils blobUtils,
Supplier<Location> defaultLocation,
@Memoized Supplier<Set<? extends Location>> locations,
PayloadSlicer slicer,
@org.jclouds.location.Provider Supplier<Credentials> creds) {
super(context, blobUtils, defaultLocation, locations, slicer);
var cred = creds.get();
blobServiceClient = new BlobServiceClientBuilder()
.credential(new AzureNamedKeyCredential(
cred.identity, cred.credential))
.endpoint("https://" + cred.identity + ".blob.core.windows.net")
.buildClient();
}
@Override
public PageSet<? extends StorageMetadata> list() {
var set = ImmutableSet.<StorageMetadata>builder();
for (var container : blobServiceClient.listBlobContainers()) {
set.add(new StorageMetadataImpl(StorageType.CONTAINER, /*id=*/ null,
container.getName(), /*location=*/ null, /*uri=*/ null,
/*eTag=*/ null, /*creationDate=*/ null,
toDate(container.getProperties().getLastModified()),
ImmutableMap.<String, String>of(), /*size=*/ null,
Tier.STANDARD));
}
return new PageSetImpl<StorageMetadata>(set.build(), null);
}
@Override
public PageSet<? extends StorageMetadata> list(String container,
ListContainerOptions options) {
var client = blobServiceClient.getBlobContainerClient(container);
var azureOptions = new ListBlobsOptions();
azureOptions.setPrefix(options.getPrefix());
azureOptions.setMaxResultsPerPage(options.getMaxResults());
var marker = options.getMarker() != null ?
URLDecoder.decode(options.getMarker(), StandardCharsets.UTF_8) :
null;
var set = ImmutableSet.<StorageMetadata>builder();
var page = client.listBlobsByHierarchy(
options.getDelimiter(), azureOptions, /*timeout=*/ null)
.iterableByPage().iterator().next();
for (var blob : page.getValue()) {
var properties = blob.getProperties();
if (blob.isPrefix()) {
set.add(new StorageMetadataImpl(StorageType.RELATIVE_PATH,
/*id=*/ null, blob.getName(), /*location=*/ null,
/*uri=*/ null, /*eTag=*/ null,
/*creationDate=*/ null,
/*lastModified=*/ null,
ImmutableMap.<String, String>of(),
/*size=*/ null,
toTier(properties.getAccessTier())));
} else {
set.add(new StorageMetadataImpl(StorageType.BLOB,
/*id=*/ null, blob.getName(), /*location=*/ null,
/*uri=*/ null, properties.getETag(),
toDate(properties.getCreationTime()),
toDate(properties.getLastModified()),
ImmutableMap.<String, String>of(),
properties.getContentLength(),
toTier(properties.getAccessTier())));
}
}
return new PageSetImpl<StorageMetadata>(set.build(),
page.getContinuationToken());
}
@Override
public boolean containerExists(String container) {
var client = blobServiceClient.getBlobContainerClient(container);
return client.exists();
}
@Override
public boolean createContainerInLocation(Location location,
String container) {
return createContainerInLocation(location, container,
new CreateContainerOptions());
}
@Override
public boolean createContainerInLocation(Location location,
String container, CreateContainerOptions options) {
try {
var azureOptions = new BlobContainerCreateOptions();
if (options.isPublicRead()) {
azureOptions.setPublicAccessType(PublicAccessType.CONTAINER);
}
blobServiceClient.createBlobContainerIfNotExistsWithResponse(
container, azureOptions, /*context=*/ null);
} catch (BlobStorageException bse) {
if (bse.getErrorCode() == BlobErrorCode.CONTAINER_ALREADY_EXISTS) {
return false;
}
throw bse;
}
return true;
}
@Override
public void deleteContainer(String container) {
blobServiceClient.deleteBlobContainer(container);
}
@Override
public boolean blobExists(String container, String key) {
var client = blobServiceClient.getBlobContainerClient(container)
.getBlobClient(key);
return client.exists();
}
@Override
public Blob getBlob(String container, String key, GetOptions options) {
var client = blobServiceClient.getBlobContainerClient(container)
.getBlobClient(key);
var blobStream = client.openInputStream();
var properties = blobStream.getProperties();
var expires = properties.getExpiresOn();
return new BlobBuilderImpl()
.name(key)
.userMetadata(properties.getMetadata())
.payload(blobStream)
.cacheControl(properties.getCacheControl())
.contentDisposition(properties.getContentDisposition())
.contentEncoding(properties.getContentEncoding())
.contentLanguage(properties.getContentLanguage())
.contentLength(properties.getBlobSize())
.contentType(properties.getContentType())
.expires(expires != null ? toDate(expires) : null)
.build();
}
@Override
public String putBlob(String container, Blob blob) {
return putBlob(container, blob, new PutOptions());
}
@Override
public String putBlob(String container, Blob blob, PutOptions options) {
var client = blobServiceClient.getBlobContainerClient(container)
.getBlobClient(blob.getMetadata().getName())
.getBlockBlobClient();
try (var is = blob.getPayload().openStream()) {
var azureOptions = new BlockBlobSimpleUploadOptions(is,
blob.getMetadata().getContentMetadata().getContentLength());
azureOptions.setMetadata(blob.getMetadata().getUserMetadata());
// TODO: Expires?
var blobHttpHeaders = new BlobHttpHeaders();
var contentMetadata = blob.getMetadata().getContentMetadata();
blobHttpHeaders.setCacheControl(contentMetadata.getCacheControl());
blobHttpHeaders.setContentDisposition(
contentMetadata.getContentDisposition());
blobHttpHeaders.setContentEncoding(
contentMetadata.getContentEncoding());
blobHttpHeaders.setContentLanguage(
contentMetadata.getContentLanguage());
blobHttpHeaders.setContentType(contentMetadata.getContentType());
azureOptions.setHeaders(blobHttpHeaders);
if (blob.getMetadata().getTier() != Tier.STANDARD) {
azureOptions.setTier(toAccessTier(
blob.getMetadata().getTier()));
}
var blockBlobItem = client.uploadWithResponse(
azureOptions, /*timeout=*/ null, /*context=*/ null);
return blockBlobItem.getValue().getETag();
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
}
@Override
public String copyBlob(String fromContainer, String fromName,
String toContainer, String toName, CopyOptions options) {
throw new UnsupportedOperationException("not yet implemented");
}
@Override
public void removeBlob(String container, String key) {
var client = blobServiceClient.getBlobContainerClient(container)
.getBlobClient(key);
try {
client.delete();
} catch (BlobStorageException bse) {
if (bse.getErrorCode() != BlobErrorCode.BLOB_NOT_FOUND) {
throw bse;
}
}
}
@Override
public BlobMetadata blobMetadata(String container, String key) {
var client = blobServiceClient.getBlobContainerClient(container)
.getBlobClient(key);
var properties = client.getProperties();
return new BlobMetadataImpl(/*id=*/ null, key, /*location=*/ null,
/*uri=*/ null, properties.getETag(),
toDate(properties.getCreationTime()),
toDate(properties.getLastModified()),
properties.getMetadata(), /*publicUri=*/ null, container,
toContentMetadata(properties),
properties.getBlobSize(), toTier(properties.getAccessTier()));
}
@Override
protected boolean deleteAndVerifyContainerGone(String container) {
blobServiceClient.deleteBlobContainer(container);
return true;
}
@Override
public ContainerAccess getContainerAccess(String container) {
var client = blobServiceClient.getBlobContainerClient(container);
return client.getAccessPolicy().getBlobAccessType() ==
PublicAccessType.CONTAINER ?
ContainerAccess.PUBLIC_READ :
ContainerAccess.PRIVATE;
}
@Override
public void setContainerAccess(String container, ContainerAccess access) {
var client = blobServiceClient.getBlobContainerClient(container);
var publicAccess = access == ContainerAccess.PUBLIC_READ ?
PublicAccessType.CONTAINER : PublicAccessType.BLOB;
client.setAccessPolicy(publicAccess,
ImmutableList.<BlobSignedIdentifier>of());
}
@Override
public BlobAccess getBlobAccess(String container, String key) {
return BlobAccess.PRIVATE;
}
@Override
public void setBlobAccess(String container, String key, BlobAccess access) {
throw new UnsupportedOperationException("unsupported in Azure");
}
@Override
public MultipartUpload initiateMultipartUpload(String container,
BlobMetadata blobMetadata, PutOptions options) {
String uploadId = UUID.randomUUID().toString();
return MultipartUpload.create(container, blobMetadata.getName(),
uploadId, blobMetadata, options);
}
@Override
public void abortMultipartUpload(MultipartUpload mpu) {
// Azure automatically removes uncommitted blocks after 7 days.
}
@Override
public String completeMultipartUpload(MultipartUpload mpu,
List<MultipartPart> parts) {
var client = blobServiceClient
.getBlobContainerClient(mpu.containerName())
.getBlobClient(mpu.blobName())
.getBlockBlobClient();
var blocks = ImmutableList.<String>builder();
for (var part : parts) {
blocks.add(makeBlockId(part.partNumber()));
}
var blockBlobItem = client.commitBlockList(blocks.build(),
/*overwrite=*/ true);
return blockBlobItem.getETag();
}
@Override
public MultipartPart uploadMultipartPart(MultipartUpload mpu,
int partNumber, Payload payload) {
var client = blobServiceClient
.getBlobContainerClient(mpu.containerName())
.getBlobClient(mpu.blobName())
.getBlockBlobClient();
var blockId = makeBlockId(partNumber);
var length = payload.getContentMetadata().getContentLength();
try (var is = payload.openStream()) {
client.stageBlock(blockId, is, length);
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
String eTag = ""; // putBlock does not return ETag
Date lastModified = null; // putBlob does not return Last-Modified
return MultipartPart.create(partNumber, length, eTag, lastModified);
}
@Override
public List<MultipartPart> listMultipartUpload(MultipartUpload mpu) {
var client = blobServiceClient
.getBlobContainerClient(mpu.containerName())
.getBlobClient(mpu.blobName())
.getBlockBlobClient();
var blockList = client.listBlocks(BlockListType.ALL);
var parts = ImmutableList.<MultipartPart>builder();
for (var properties : blockList.getUncommittedBlocks()) {
int partNumber = Ints.fromByteArray(Base64.getDecoder().decode(
properties.getName()));
String eTag = ""; // listBlocks does not return ETag
Date lastModified = null; // listBlocks does not return LastModified
parts.add(MultipartPart.create(partNumber, properties.getSizeLong(),
eTag, lastModified));
}
return parts.build();
}
@Override
public List<MultipartUpload> listMultipartUploads(String container) {
var client = blobServiceClient.getBlobContainerClient(container);
var azureOptions = new ListBlobsOptions();
var details = new BlobListDetails();
details.setRetrieveUncommittedBlobs(true);
azureOptions.setDetails(details);
var builder = ImmutableList.<MultipartUpload>builder();
for (var blob : client.listBlobs(azureOptions,
/*continuationToken=*/ null, /*timeout=*/ null)) {
var properties = blob.getProperties();
// only uncommitted blobs lack ETags
if (properties.getETag() != null) {
continue;
}
// TODO: bogus uploadId
String uploadId = UUID.randomUUID().toString();
builder.add(MultipartUpload.create(container, blob.getName(),
uploadId, null, null));
}
return builder.build();
}
@Override
public long getMinimumMultipartPartSize() {
return 1;
}
@Override
public long getMaximumMultipartPartSize() {
return 100 * 1024 * 1024;
}
@Override
public int getMaximumNumberOfParts() {
return 50 * 1000;
}
@Override
public InputStream streamBlob(String container, String name) {
throw new UnsupportedOperationException("not yet implemented");
}
// TODO: handle more error codes
public static S3ErrorCode toS3ErrorCode(BlobErrorCode code) {
if (code.equals(BlobErrorCode.CONTAINER_NOT_FOUND)) {
return S3ErrorCode.NO_SUCH_BUCKET;
} else {
return S3ErrorCode.INTERNAL_ERROR;
}
}
private static Date toDate(OffsetDateTime time) {
return new Date(time.toInstant().toEpochMilli());
}
private static AccessTier toAccessTier(Tier tier) {
switch (tier) {
case ARCHIVE:
return AccessTier.ARCHIVE;
case INFREQUENT:
return AccessTier.COOL;
case STANDARD:
default:
return AccessTier.HOT;
}
}
private static Tier toTier(AccessTier tier) {
if (tier == null) {
return Tier.STANDARD;
} else if (tier.equals(AccessTier.ARCHIVE)) {
return Tier.ARCHIVE;
} else if (tier.equals(AccessTier.COLD) ||
tier.equals(AccessTier.COOL)) {
return Tier.INFREQUENT;
} else {
return Tier.STANDARD;
}
}
private static ContentMetadata toContentMetadata(
BlobProperties properties) {
var expires = properties.getExpiresOn();
return ContentMetadataBuilder.create()
.cacheControl(properties.getCacheControl())
.contentDisposition(properties.getContentDisposition())
.contentEncoding(properties.getContentEncoding())
.contentLanguage(properties.getContentLanguage())
.contentLength(properties.getBlobSize())
.contentType(properties.getContentType())
.expires(expires != null ? toDate(expires) : null)
.build();
}
private static String makeBlockId(int partNumber) {
return Base64.getEncoder().encodeToString(Ints.toByteArray(partNumber));
}
}

Wyświetl plik

@ -0,0 +1,31 @@
/*
* Copyright 2014-2021 Andrew Gaul <andrew@gaul.org>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.gaul.s3proxy.azureblob;
import com.google.inject.AbstractModule;
import com.google.inject.Scopes;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.attr.ConsistencyModel;
public final class AzureBlobStoreContextModule extends AbstractModule {
@Override
protected void configure() {
bind(ConsistencyModel.class).toInstance(ConsistencyModel.STRICT);
bind(BlobStore.class).to(AzureBlobStore.class).in(Scopes.SINGLETON);
}
}