Add filesystem-nio2 blobstore

This uses the same code paths as transient-nio2 and will replace the
jclouds filesystem blobstore.
pull/719/head
Andrew Gaul 2024-12-01 18:05:06 -08:00
rodzic 8f5a812c69
commit 7632e9cc3e
11 zmienionych plików z 397 dodań i 77 usunięć

Wyświetl plik

@ -77,6 +77,7 @@ Maven Central hosts S3Proxy artifacts and the wiki has
* azureblob-sdk (newer but lacks multi-part upload, see [Azure/azure-sdk-for-java#42603](https://github.com/Azure/azure-sdk-for-java/issues/42603)) * azureblob-sdk (newer but lacks multi-part upload, see [Azure/azure-sdk-for-java#42603](https://github.com/Azure/azure-sdk-for-java/issues/42603))
* b2 * b2
* filesystem (on-disk storage) * filesystem (on-disk storage)
* filesystem-nio2 (on-disk storage, preview)
* google-cloud-storage * google-cloud-storage
* openstack-swift * openstack-swift
* rackspace-cloudfiles-uk and rackspace-cloudfiles-us * rackspace-cloudfiles-uk and rackspace-cloudfiles-us

Wyświetl plik

@ -51,12 +51,9 @@ import com.google.common.hash.HashingInputStream;
import com.google.common.io.BaseEncoding; import com.google.common.io.BaseEncoding;
import com.google.common.io.ByteSource; import com.google.common.io.ByteSource;
import com.google.common.io.ByteStreams; import com.google.common.io.ByteStreams;
import com.google.common.jimfs.Configuration;
import com.google.common.jimfs.Jimfs;
import com.google.common.net.HttpHeaders; import com.google.common.net.HttpHeaders;
import com.google.common.primitives.Longs; import com.google.common.primitives.Longs;
import jakarta.inject.Inject;
import jakarta.inject.Singleton; import jakarta.inject.Singleton;
import jakarta.ws.rs.core.Response.Status; import jakarta.ws.rs.core.Response.Status;
@ -98,9 +95,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@Singleton @Singleton
public final class Nio2BlobStore extends BaseBlobStore { public abstract class AbstractNio2BlobStore extends BaseBlobStore {
private static final Logger logger = LoggerFactory.getLogger( private static final Logger logger = LoggerFactory.getLogger(
Nio2BlobStore.class); AbstractNio2BlobStore.class);
private static final String XATTR_CACHE_CONTROL = "user.cache-control"; private static final String XATTR_CACHE_CONTROL = "user.cache-control";
private static final String XATTR_CONTENT_DISPOSITION = private static final String XATTR_CONTENT_DISPOSITION =
"user.content-disposition"; "user.content-disposition";
@ -120,31 +117,29 @@ public final class Nio2BlobStore extends BaseBlobStore {
private final Supplier<Set<? extends Location>> locations; private final Supplier<Set<? extends Location>> locations;
private final FileSystem fs; private final FileSystem fs;
private final Path root;
@Inject protected AbstractNio2BlobStore(BlobStoreContext context, BlobUtils blobUtils,
Nio2BlobStore(BlobStoreContext context, BlobUtils blobUtils,
Supplier<Location> defaultLocation, Supplier<Location> defaultLocation,
@Memoized Supplier<Set<? extends Location>> locations, @Memoized Supplier<Set<? extends Location>> locations,
PayloadSlicer slicer, PayloadSlicer slicer,
@org.jclouds.location.Provider Supplier<Credentials> creds) { @org.jclouds.location.Provider Supplier<Credentials> creds,
FileSystem fs, Path root) {
super(context, blobUtils, defaultLocation, locations, slicer); super(context, blobUtils, defaultLocation, locations, slicer);
this.locations = requireNonNull(locations, "locations"); this.locations = requireNonNull(locations, "locations");
// TODO: close this this.fs = fs;
this.fs = Jimfs.newFileSystem(Configuration.unix().toBuilder() this.root = root;
.setAttributeViews("posix", "user")
.setWorkingDirectory("/")
.build());
} }
@Override @Override
public Set<? extends Location> listAssignableLocations() { public final Set<? extends Location> listAssignableLocations() {
return locations.get(); return locations.get();
} }
@Override @Override
public PageSet<? extends StorageMetadata> list() { public final PageSet<? extends StorageMetadata> list() {
var set = ImmutableSortedSet.<StorageMetadata>naturalOrder(); var set = ImmutableSortedSet.<StorageMetadata>naturalOrder();
try (var stream = Files.newDirectoryStream(fs.getPath(""))) { try (var stream = Files.newDirectoryStream(root)) {
for (var path : stream) { for (var path : stream) {
var attr = Files.readAttributes(path, var attr = Files.readAttributes(path,
BasicFileAttributes.class); BasicFileAttributes.class);
@ -164,7 +159,7 @@ public final class Nio2BlobStore extends BaseBlobStore {
} }
@Override @Override
public PageSet<? extends StorageMetadata> list(String container, public final PageSet<? extends StorageMetadata> list(String container,
ListContainerOptions options) { ListContainerOptions options) {
if (!containerExists(container)) { if (!containerExists(container)) {
throw new ContainerNotFoundException(container, ""); throw new ContainerNotFoundException(container, "");
@ -178,7 +173,7 @@ public final class Nio2BlobStore extends BaseBlobStore {
} }
var prefix = options.getPrefix(); var prefix = options.getPrefix();
var dirPrefix = fs.getPath(container); var dirPrefix = root.resolve(container);
if (prefix != null) { if (prefix != null) {
int idx = prefix.lastIndexOf('/'); int idx = prefix.lastIndexOf('/');
if (idx != -1) { if (idx != -1) {
@ -292,22 +287,22 @@ public final class Nio2BlobStore extends BaseBlobStore {
} }
@Override @Override
public boolean containerExists(String container) { public final boolean containerExists(String container) {
return Files.exists(fs.getPath(container)); return Files.exists(root.resolve(container));
} }
@Override @Override
public boolean createContainerInLocation(Location location, public final boolean createContainerInLocation(Location location,
String container) { String container) {
return createContainerInLocation(location, container, return createContainerInLocation(location, container,
new CreateContainerOptions()); new CreateContainerOptions());
} }
@Override @Override
public boolean createContainerInLocation(Location location, public final boolean createContainerInLocation(Location location,
String container, CreateContainerOptions options) { String container, CreateContainerOptions options) {
try { try {
Files.createDirectory(fs.getPath(container)); Files.createDirectory(root.resolve(container));
} catch (FileAlreadyExistsException faee) { } catch (FileAlreadyExistsException faee) {
return false; return false;
} catch (IOException ioe) { } catch (IOException ioe) {
@ -320,9 +315,9 @@ public final class Nio2BlobStore extends BaseBlobStore {
} }
@Override @Override
public void deleteContainer(String container) { public final void deleteContainer(String container) {
try { try {
Files.deleteIfExists(fs.getPath(container)); Files.deleteIfExists(root.resolve(container));
} catch (DirectoryNotEmptyException dnee) { } catch (DirectoryNotEmptyException dnee) {
// TODO: what to do? // TODO: what to do?
} catch (IOException ioe) { } catch (IOException ioe) {
@ -331,17 +326,17 @@ public final class Nio2BlobStore extends BaseBlobStore {
} }
@Override @Override
public boolean blobExists(String container, String key) { public final boolean blobExists(String container, String key) {
return blobMetadata(container, key) != null; return blobMetadata(container, key) != null;
} }
@Override @Override
public Blob getBlob(String container, String key, GetOptions options) { public final Blob getBlob(String container, String key, GetOptions options) {
if (!containerExists(container)) { if (!containerExists(container)) {
throw new ContainerNotFoundException(container, ""); throw new ContainerNotFoundException(container, "");
} }
var path = fs.getPath(container, key); var path = root.resolve(container).resolve(key);
logger.debug("Getting blob at: " + path); logger.debug("Getting blob at: " + path);
try { try {
@ -515,19 +510,19 @@ public final class Nio2BlobStore extends BaseBlobStore {
} }
@Override @Override
public String putBlob(String container, Blob blob) { public final String putBlob(String container, Blob blob) {
return putBlob(container, blob, new PutOptions()); return putBlob(container, blob, new PutOptions());
} }
@Override @Override
public String putBlob(String container, Blob blob, PutOptions options) { public final String putBlob(String container, Blob blob, PutOptions options) {
if (!containerExists(container)) { if (!containerExists(container)) {
throw new ContainerNotFoundException(container, ""); throw new ContainerNotFoundException(container, "");
} }
var path = fs.getPath(container, blob.getMetadata().getName()); var path = root.resolve(container).resolve(blob.getMetadata().getName());
// TODO: should we use a known suffix to filter these out during list? // TODO: should we use a known suffix to filter these out during list?
var tmpPath = fs.getPath(container, blob.getMetadata().getName() + "-" + UUID.randomUUID()); var tmpPath = root.resolve(container).resolve(blob.getMetadata().getName() + "-" + UUID.randomUUID());
logger.debug("Creating blob at: " + path); logger.debug("Creating blob at: " + path);
if (blob.getMetadata().getName().endsWith("/")) { if (blob.getMetadata().getName().endsWith("/")) {
@ -606,7 +601,7 @@ public final class Nio2BlobStore extends BaseBlobStore {
} }
@Override @Override
public String copyBlob(String fromContainer, String fromName, public final String copyBlob(String fromContainer, String fromName,
String toContainer, String toName, CopyOptions options) { String toContainer, String toName, CopyOptions options) {
var blob = getBlob(fromContainer, fromName); var blob = getBlob(fromContainer, fromName);
if (blob == null) { if (blob == null) {
@ -685,9 +680,9 @@ public final class Nio2BlobStore extends BaseBlobStore {
} }
@Override @Override
public void removeBlob(String container, String key) { public final void removeBlob(String container, String key) {
try { try {
var path = fs.getPath(container, key); var path = root.resolve(container).resolve(key);
Files.delete(path); Files.delete(path);
removeEmptyParentDirectories(path.getParent()); removeEmptyParentDirectories(path.getParent());
} catch (NoSuchFileException nsfe) { } catch (NoSuchFileException nsfe) {
@ -698,7 +693,7 @@ public final class Nio2BlobStore extends BaseBlobStore {
} }
@Override @Override
public BlobMetadata blobMetadata(String container, String key) { public final BlobMetadata blobMetadata(String container, String key) {
Blob blob = getBlob(container, key); Blob blob = getBlob(container, key);
if (blob == null) { if (blob == null) {
return null; return null;
@ -713,18 +708,18 @@ public final class Nio2BlobStore extends BaseBlobStore {
} }
@Override @Override
protected boolean deleteAndVerifyContainerGone(String container) { protected final boolean deleteAndVerifyContainerGone(String container) {
deleteContainer(container); deleteContainer(container);
return !containerExists(container); return !containerExists(container);
} }
@Override @Override
public ContainerAccess getContainerAccess(String container) { public final ContainerAccess getContainerAccess(String container) {
if (!containerExists(container)) { if (!containerExists(container)) {
throw new ContainerNotFoundException(container, ""); throw new ContainerNotFoundException(container, "");
} }
var path = fs.getPath(container); var path = root.resolve(container);
Set<PosixFilePermission> permissions; Set<PosixFilePermission> permissions;
try { try {
permissions = Files.getPosixFilePermissions(path); permissions = Files.getPosixFilePermissions(path);
@ -736,12 +731,12 @@ public final class Nio2BlobStore extends BaseBlobStore {
} }
@Override @Override
public void setContainerAccess(String container, ContainerAccess access) { public final void setContainerAccess(String container, ContainerAccess access) {
if (!containerExists(container)) { if (!containerExists(container)) {
throw new ContainerNotFoundException(container, ""); throw new ContainerNotFoundException(container, "");
} }
var path = fs.getPath(container); var path = root.resolve(container);
Set<PosixFilePermission> permissions; Set<PosixFilePermission> permissions;
try { try {
permissions = new HashSet<>(Files.getPosixFilePermissions(path)); permissions = new HashSet<>(Files.getPosixFilePermissions(path));
@ -757,7 +752,7 @@ public final class Nio2BlobStore extends BaseBlobStore {
} }
@Override @Override
public BlobAccess getBlobAccess(String container, String key) { public final BlobAccess getBlobAccess(String container, String key) {
if (!containerExists(container)) { if (!containerExists(container)) {
throw new ContainerNotFoundException(container, ""); throw new ContainerNotFoundException(container, "");
} }
@ -765,7 +760,7 @@ public final class Nio2BlobStore extends BaseBlobStore {
throw new KeyNotFoundException(container, key, ""); throw new KeyNotFoundException(container, key, "");
} }
var path = fs.getPath(container, key); var path = root.resolve(container).resolve(key);
Set<PosixFilePermission> permissions; Set<PosixFilePermission> permissions;
try { try {
permissions = Files.getPosixFilePermissions(path); permissions = Files.getPosixFilePermissions(path);
@ -777,7 +772,7 @@ public final class Nio2BlobStore extends BaseBlobStore {
} }
@Override @Override
public void setBlobAccess(String container, String key, BlobAccess access) { public final void setBlobAccess(String container, String key, BlobAccess access) {
if (!containerExists(container)) { if (!containerExists(container)) {
throw new ContainerNotFoundException(container, ""); throw new ContainerNotFoundException(container, "");
} }
@ -785,7 +780,7 @@ public final class Nio2BlobStore extends BaseBlobStore {
throw new KeyNotFoundException(container, key, ""); throw new KeyNotFoundException(container, key, "");
} }
var path = fs.getPath(container, key); var path = root.resolve(container).resolve(key);
Set<PosixFilePermission> permissions; Set<PosixFilePermission> permissions;
try { try {
permissions = new HashSet<>(Files.getPosixFilePermissions(path)); permissions = new HashSet<>(Files.getPosixFilePermissions(path));
@ -801,7 +796,7 @@ public final class Nio2BlobStore extends BaseBlobStore {
} }
@Override @Override
public MultipartUpload initiateMultipartUpload(String container, public final MultipartUpload initiateMultipartUpload(String container,
BlobMetadata blobMetadata, PutOptions options) { BlobMetadata blobMetadata, PutOptions options) {
var uploadId = UUID.randomUUID().toString(); var uploadId = UUID.randomUUID().toString();
// create a stub blob // create a stub blob
@ -812,7 +807,7 @@ public final class Nio2BlobStore extends BaseBlobStore {
} }
@Override @Override
public void abortMultipartUpload(MultipartUpload mpu) { public final void abortMultipartUpload(MultipartUpload mpu) {
var parts = listMultipartUpload(mpu); var parts = listMultipartUpload(mpu);
for (var part : parts) { for (var part : parts) {
removeBlob(mpu.containerName(), MULTIPART_PREFIX + mpu.id() + "-" + mpu.blobName() + "-" + part.partNumber()); removeBlob(mpu.containerName(), MULTIPART_PREFIX + mpu.id() + "-" + mpu.blobName() + "-" + part.partNumber());
@ -821,7 +816,7 @@ public final class Nio2BlobStore extends BaseBlobStore {
} }
@Override @Override
public String completeMultipartUpload(MultipartUpload mpu, List<MultipartPart> parts) { public final String completeMultipartUpload(MultipartUpload mpu, List<MultipartPart> parts) {
var metas = ImmutableList.<BlobMetadata>builder(); var metas = ImmutableList.<BlobMetadata>builder();
long contentLength = 0; long contentLength = 0;
var md5Hasher = Hashing.md5().newHasher(); var md5Hasher = Hashing.md5().newHasher();
@ -893,7 +888,7 @@ public final class Nio2BlobStore extends BaseBlobStore {
} }
@Override @Override
public MultipartPart uploadMultipartPart(MultipartUpload mpu, int partNumber, Payload payload) { public final MultipartPart uploadMultipartPart(MultipartUpload mpu, int partNumber, Payload payload) {
var partName = MULTIPART_PREFIX + mpu.id() + "-" + mpu.blobName() + "-" + partNumber; var partName = MULTIPART_PREFIX + mpu.id() + "-" + mpu.blobName() + "-" + partNumber;
var blob = blobBuilder(partName) var blob = blobBuilder(partName)
.payload(payload) .payload(payload)
@ -905,7 +900,7 @@ public final class Nio2BlobStore extends BaseBlobStore {
} }
@Override @Override
public List<MultipartPart> listMultipartUpload(MultipartUpload mpu) { public final List<MultipartPart> listMultipartUpload(MultipartUpload mpu) {
var parts = ImmutableList.<MultipartPart>builder(); var parts = ImmutableList.<MultipartPart>builder();
var options = var options =
new ListContainerOptions().prefix(MULTIPART_PREFIX + mpu.id() + "-" + mpu.blobName() + "-").recursive(); new ListContainerOptions().prefix(MULTIPART_PREFIX + mpu.id() + "-" + mpu.blobName() + "-").recursive();
@ -928,7 +923,7 @@ public final class Nio2BlobStore extends BaseBlobStore {
} }
@Override @Override
public List<MultipartUpload> listMultipartUploads(String container) { public final List<MultipartUpload> listMultipartUploads(String container) {
var mpus = ImmutableList.<MultipartUpload>builder(); var mpus = ImmutableList.<MultipartUpload>builder();
var options = new ListContainerOptions().prefix(MULTIPART_PREFIX).recursive(); var options = new ListContainerOptions().prefix(MULTIPART_PREFIX).recursive();
int uuidLength = UUID.randomUUID().toString().length(); int uuidLength = UUID.randomUUID().toString().length();
@ -955,22 +950,22 @@ public final class Nio2BlobStore extends BaseBlobStore {
} }
@Override @Override
public long getMinimumMultipartPartSize() { public final long getMinimumMultipartPartSize() {
return 1; return 1;
} }
@Override @Override
public long getMaximumMultipartPartSize() { public final long getMaximumMultipartPartSize() {
return 100 * 1024 * 1024; return 100 * 1024 * 1024;
} }
@Override @Override
public int getMaximumNumberOfParts() { public final int getMaximumNumberOfParts() {
return 50 * 1000; return 50 * 1000;
} }
@Override @Override
public InputStream streamBlob(String container, String name) { public final InputStream streamBlob(String container, String name) {
throw new UnsupportedOperationException("not yet implemented"); throw new UnsupportedOperationException("not yet implemented");
} }
@ -1073,7 +1068,7 @@ public final class Nio2BlobStore extends BaseBlobStore {
} }
/** /**
* Nio2BlobStore implicitly creates directories when creating a key /a/b/c. * AbstractNio2BlobStore implicitly creates directories when creating a key /a/b/c.
* When removing /a/b/c, it must clean up /a and /a/b, unless a client explicitly created a subdirectory which has file attributes. * When removing /a/b/c, it must clean up /a and /a/b, unless a client explicitly created a subdirectory which has file attributes.
*/ */
private static void removeEmptyParentDirectories(Path path) throws IOException { private static void removeEmptyParentDirectories(Path path) throws IOException {

Wyświetl plik

@ -0,0 +1,80 @@
/*
* Copyright 2014-2024 Andrew Gaul <andrew@gaul.org>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.gaul.s3proxy.nio2blob;
import java.net.URI;
import java.util.Properties;
import java.util.Set;
import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.reflect.Reflection2;
import org.jclouds.rest.internal.BaseHttpApiMetadata;
public final class FilesystemNio2BlobApiMetadata extends BaseHttpApiMetadata {
public FilesystemNio2BlobApiMetadata() {
this(builder());
}
protected FilesystemNio2BlobApiMetadata(Builder builder) {
super(builder);
}
private static Builder builder() {
return new Builder();
}
@Override
public Builder toBuilder() {
return builder().fromApiMetadata(this);
}
public static Properties defaultProperties() {
return BaseHttpApiMetadata.defaultProperties();
}
// Fake API client
private interface FilesystemNio2BlobClient {
}
public static final class Builder
extends BaseHttpApiMetadata.Builder<FilesystemNio2BlobClient, Builder> {
protected Builder() {
super(FilesystemNio2BlobClient.class);
id("filesystem-nio2")
.name("Filesystem NIO.2 Blobstore")
.identityName("Account Name")
.credentialName("Access Key")
.defaultEndpoint("http://localhost/")
.documentation(URI.create(
"http://www.jclouds.org/documentation/userguide" +
"/blobstore-guide"))
.defaultProperties(FilesystemNio2BlobApiMetadata.defaultProperties())
.view(Reflection2.typeToken(BlobStoreContext.class))
.defaultModules(Set.of(FilesystemNio2BlobStoreContextModule.class));
}
@Override
public FilesystemNio2BlobApiMetadata build() {
return new FilesystemNio2BlobApiMetadata(this);
}
@Override
protected Builder self() {
return this;
}
}
}

Wyświetl plik

@ -27,12 +27,12 @@ import org.jclouds.providers.internal.BaseProviderMetadata;
* Implementation of org.jclouds.types.ProviderMetadata for NIO.2 filesystems. * Implementation of org.jclouds.types.ProviderMetadata for NIO.2 filesystems.
*/ */
@AutoService(ProviderMetadata.class) @AutoService(ProviderMetadata.class)
public final class Nio2BlobProviderMetadata extends BaseProviderMetadata { public final class FilesystemNio2BlobProviderMetadata extends BaseProviderMetadata {
public Nio2BlobProviderMetadata() { public FilesystemNio2BlobProviderMetadata() {
super(builder()); super(builder());
} }
public Nio2BlobProviderMetadata(Builder builder) { public FilesystemNio2BlobProviderMetadata(Builder builder) {
super(builder); super(builder);
} }
@ -52,17 +52,17 @@ public final class Nio2BlobProviderMetadata extends BaseProviderMetadata {
} }
public static final class Builder extends BaseProviderMetadata.Builder { public static final class Builder extends BaseProviderMetadata.Builder {
protected Builder() { protected Builder() {
id("transient-nio2") id("filesystem-nio2")
.name("NIO.2 filesystem blobstore") .name("NIO.2 filesystem blobstore")
.apiMetadata(new Nio2BlobApiMetadata()) .apiMetadata(new FilesystemNio2BlobApiMetadata())
.endpoint("https://127.0.0.1") // TODO: .endpoint("https://127.0.0.1") // TODO:
.defaultProperties( .defaultProperties(
Nio2BlobProviderMetadata.defaultProperties()); FilesystemNio2BlobProviderMetadata.defaultProperties());
} }
@Override @Override
public Nio2BlobProviderMetadata build() { public FilesystemNio2BlobProviderMetadata build() {
return new Nio2BlobProviderMetadata(this); return new FilesystemNio2BlobProviderMetadata(this);
} }
@Override @Override

Wyświetl plik

@ -0,0 +1,61 @@
/*
* Copyright 2014-2024 Andrew Gaul <andrew@gaul.org>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.gaul.s3proxy.nio2blob;
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
import java.util.Set;
import com.google.common.base.Supplier;
import jakarta.inject.Inject;
import jakarta.inject.Named;
import jakarta.inject.Singleton;
import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.util.BlobUtils;
import org.jclouds.collect.Memoized;
import org.jclouds.domain.Credentials;
import org.jclouds.domain.Location;
import org.jclouds.filesystem.reference.FilesystemConstants;
import org.jclouds.io.PayloadSlicer;
@Singleton
public final class FilesystemNio2BlobStore extends AbstractNio2BlobStore {
@Inject
FilesystemNio2BlobStore(BlobStoreContext context, BlobUtils blobUtils,
Supplier<Location> defaultLocation,
@Memoized Supplier<Set<? extends Location>> locations,
PayloadSlicer slicer,
@org.jclouds.location.Provider Supplier<Credentials> creds,
@Named(FilesystemConstants.PROPERTY_BASEDIR) String baseDir) {
this(context, blobUtils, defaultLocation, locations, slicer, creds,
baseDir, FileSystems.getDefault());
}
// Helper to create Path
private FilesystemNio2BlobStore(BlobStoreContext context, BlobUtils blobUtils,
Supplier<Location> defaultLocation,
@Memoized Supplier<Set<? extends Location>> locations,
PayloadSlicer slicer,
@org.jclouds.location.Provider Supplier<Credentials> creds,
@Named(FilesystemConstants.PROPERTY_BASEDIR) String baseDir,
FileSystem fs) {
super(context, blobUtils, defaultLocation, locations, slicer, creds,
fs, fs.getPath(baseDir));
}
}

Wyświetl plik

@ -0,0 +1,31 @@
/*
* Copyright 2014-2024 Andrew Gaul <andrew@gaul.org>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.gaul.s3proxy.nio2blob;
import com.google.inject.AbstractModule;
import com.google.inject.Scopes;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.attr.ConsistencyModel;
public final class FilesystemNio2BlobStoreContextModule extends AbstractModule {
@Override
protected void configure() {
bind(ConsistencyModel.class).toInstance(ConsistencyModel.STRICT);
bind(BlobStore.class).to(FilesystemNio2BlobStore.class).in(Scopes.SINGLETON);
}
}

Wyświetl plik

@ -24,12 +24,12 @@ import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.reflect.Reflection2; import org.jclouds.reflect.Reflection2;
import org.jclouds.rest.internal.BaseHttpApiMetadata; import org.jclouds.rest.internal.BaseHttpApiMetadata;
public final class Nio2BlobApiMetadata extends BaseHttpApiMetadata { public final class TransientNio2BlobApiMetadata extends BaseHttpApiMetadata {
public Nio2BlobApiMetadata() { public TransientNio2BlobApiMetadata() {
this(builder()); this(builder());
} }
protected Nio2BlobApiMetadata(Builder builder) { protected TransientNio2BlobApiMetadata(Builder builder) {
super(builder); super(builder);
} }
@ -47,29 +47,29 @@ public final class Nio2BlobApiMetadata extends BaseHttpApiMetadata {
} }
// Fake API client // Fake API client
private interface Nio2BlobClient { private interface TransientNio2BlobClient {
} }
public static final class Builder public static final class Builder
extends BaseHttpApiMetadata.Builder<Nio2BlobClient, Builder> { extends BaseHttpApiMetadata.Builder<TransientNio2BlobClient, Builder> {
protected Builder() { protected Builder() {
super(Nio2BlobClient.class); super(TransientNio2BlobClient.class);
id("transient-nio2") id("transient-nio2")
.name("NIO.2 Blobstore") .name("Transient NIO.2 Blobstore")
.identityName("Account Name") .identityName("Account Name")
.credentialName("Access Key") .credentialName("Access Key")
.defaultEndpoint("http://localhost/") .defaultEndpoint("http://localhost/")
.documentation(URI.create( .documentation(URI.create(
"http://www.jclouds.org/documentation/userguide" + "http://www.jclouds.org/documentation/userguide" +
"/blobstore-guide")) "/blobstore-guide"))
.defaultProperties(Nio2BlobApiMetadata.defaultProperties()) .defaultProperties(TransientNio2BlobApiMetadata.defaultProperties())
.view(Reflection2.typeToken(BlobStoreContext.class)) .view(Reflection2.typeToken(BlobStoreContext.class))
.defaultModules(Set.of(Nio2BlobStoreContextModule.class)); .defaultModules(Set.of(TransientNio2BlobStoreContextModule.class));
} }
@Override @Override
public Nio2BlobApiMetadata build() { public TransientNio2BlobApiMetadata build() {
return new Nio2BlobApiMetadata(this); return new TransientNio2BlobApiMetadata(this);
} }
@Override @Override

Wyświetl plik

@ -0,0 +1,75 @@
/*
* Copyright 2014-2024 Andrew Gaul <andrew@gaul.org>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.gaul.s3proxy.nio2blob;
import java.util.Properties;
import com.google.auto.service.AutoService;
import org.jclouds.providers.ProviderMetadata;
import org.jclouds.providers.internal.BaseProviderMetadata;
/**
* Implementation of org.jclouds.types.ProviderMetadata for NIO.2 filesystems.
*/
@AutoService(ProviderMetadata.class)
public final class TransientNio2BlobProviderMetadata extends BaseProviderMetadata {
public TransientNio2BlobProviderMetadata() {
super(builder());
}
public TransientNio2BlobProviderMetadata(Builder builder) {
super(builder);
}
public static Builder builder() {
return new Builder();
}
@Override
public Builder toBuilder() {
return builder().fromProviderMetadata(this);
}
public static Properties defaultProperties() {
Properties properties = new Properties();
// TODO: filesystem basedir
return properties;
}
public static final class Builder extends BaseProviderMetadata.Builder {
protected Builder() {
id("transient-nio2")
.name("Filesystem NIO.2 blobstore")
.apiMetadata(new TransientNio2BlobApiMetadata())
.endpoint("https://127.0.0.1") // TODO:
.defaultProperties(
TransientNio2BlobProviderMetadata.defaultProperties());
}
@Override
public TransientNio2BlobProviderMetadata build() {
return new TransientNio2BlobProviderMetadata(this);
}
@Override
public Builder fromProviderMetadata(
ProviderMetadata in) {
super.fromProviderMetadata(in);
return this;
}
}
}

Wyświetl plik

@ -0,0 +1,61 @@
/*
* Copyright 2014-2024 Andrew Gaul <andrew@gaul.org>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.gaul.s3proxy.nio2blob;
import java.nio.file.FileSystem;
import java.util.Set;
import com.google.common.base.Supplier;
import com.google.common.jimfs.Configuration;
import com.google.common.jimfs.Jimfs;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.util.BlobUtils;
import org.jclouds.collect.Memoized;
import org.jclouds.domain.Credentials;
import org.jclouds.domain.Location;
import org.jclouds.io.PayloadSlicer;
@Singleton
public final class TransientNio2BlobStore extends AbstractNio2BlobStore {
@Inject
TransientNio2BlobStore(BlobStoreContext context, BlobUtils blobUtils,
Supplier<Location> defaultLocation,
@Memoized Supplier<Set<? extends Location>> locations,
PayloadSlicer slicer,
@org.jclouds.location.Provider Supplier<Credentials> creds) {
this(context, blobUtils, defaultLocation, locations, slicer, creds,
Jimfs.newFileSystem(Configuration.unix().toBuilder()
.setAttributeViews("posix", "user")
.setWorkingDirectory("/")
.build()));
}
// Helper to create Path
private TransientNio2BlobStore(BlobStoreContext context, BlobUtils blobUtils,
Supplier<Location> defaultLocation,
@Memoized Supplier<Set<? extends Location>> locations,
PayloadSlicer slicer,
@org.jclouds.location.Provider Supplier<Credentials> creds,
FileSystem fs) {
super(context, blobUtils, defaultLocation, locations, slicer, creds,
fs, fs.getPath(""));
}
}

Wyświetl plik

@ -22,10 +22,10 @@ import com.google.inject.Scopes;
import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.attr.ConsistencyModel; import org.jclouds.blobstore.attr.ConsistencyModel;
public final class Nio2BlobStoreContextModule extends AbstractModule { public final class TransientNio2BlobStoreContextModule extends AbstractModule {
@Override @Override
protected void configure() { protected void configure() {
bind(ConsistencyModel.class).toInstance(ConsistencyModel.STRICT); bind(ConsistencyModel.class).toInstance(ConsistencyModel.STRICT);
bind(BlobStore.class).to(Nio2BlobStore.class).in(Scopes.SINGLETON); bind(BlobStore.class).to(TransientNio2BlobStore.class).in(Scopes.SINGLETON);
} }
} }

Wyświetl plik

@ -0,0 +1,16 @@
s3proxy.endpoint=http://127.0.0.1:0
s3proxy.secure-endpoint=https://127.0.0.1:0
#s3proxy.service-path=s3proxy
# authorization must be aws-v2, aws-v4, aws-v2-or-v4, or none
s3proxy.authorization=aws-v2-or-v4
s3proxy.identity=local-identity
s3proxy.credential=local-credential
s3proxy.keystore-path=keystore.jks
s3proxy.keystore-password=password
jclouds.provider=filesystem-nio2
jclouds.identity=remote-identity
jclouds.credential=remote-credential
# endpoint is optional for some providers
#jclouds.endpoint=http://127.0.0.1:8081
jclouds.filesystem.basedir=/tmp/blobstore