diff --git a/README.md b/README.md index dca1c8c..c978c5a 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Jortage Storage Pool Proxy +# Jortage Storage Pool Manager A miniature S3-to-S3 proxy that accepts incoming requests, hashes the uploaded file, stores their desired path in a MariaDB database, along with the hash, and then uploads the original file to a backing S3 server named after the diff --git a/build.gradle b/build.gradle index d46bd09..6c5aaa4 100644 --- a/build.gradle +++ b/build.gradle @@ -48,5 +48,5 @@ project.configurations.implementation.setCanBeResolved(true) task capsule(type: FatCapsule) { embedConfiguration configurations.implementation - applicationClass 'com.jortage.proxy.JortageProxy' + applicationClass 'com.jortage.poolmgr.Poolmgr' } \ No newline at end of file diff --git a/src/main/java/com/jortage/proxy/ByteSinkSource.java b/src/main/java/com/jortage/poolmgr/ByteSinkSource.java similarity index 88% rename from src/main/java/com/jortage/proxy/ByteSinkSource.java rename to src/main/java/com/jortage/poolmgr/ByteSinkSource.java index 7796c31..fb150dd 100644 --- a/src/main/java/com/jortage/proxy/ByteSinkSource.java +++ b/src/main/java/com/jortage/poolmgr/ByteSinkSource.java @@ -1,4 +1,4 @@ -package com.jortage.proxy; +package com.jortage.poolmgr; import java.io.Closeable; import com.google.common.io.ByteSink; diff --git a/src/main/java/com/jortage/proxy/FileByteSinkSource.java b/src/main/java/com/jortage/poolmgr/FileByteSinkSource.java similarity index 95% rename from src/main/java/com/jortage/proxy/FileByteSinkSource.java rename to src/main/java/com/jortage/poolmgr/FileByteSinkSource.java index f7178f8..327348a 100644 --- a/src/main/java/com/jortage/proxy/FileByteSinkSource.java +++ b/src/main/java/com/jortage/poolmgr/FileByteSinkSource.java @@ -1,4 +1,4 @@ -package com.jortage.proxy; +package com.jortage.poolmgr; import java.io.File; diff --git a/src/main/java/com/jortage/proxy/JortageBlobStore.java b/src/main/java/com/jortage/poolmgr/JortageBlobStore.java similarity index 93% rename from src/main/java/com/jortage/proxy/JortageBlobStore.java rename to src/main/java/com/jortage/poolmgr/JortageBlobStore.java index 8ca2de7..8cfe77c 100644 --- a/src/main/java/com/jortage/proxy/JortageBlobStore.java +++ b/src/main/java/com/jortage/poolmgr/JortageBlobStore.java @@ -1,4 +1,4 @@ -package com.jortage.proxy; +package com.jortage.poolmgr; import java.io.File; import java.io.FileOutputStream; @@ -69,7 +69,7 @@ public class JortageBlobStore extends ForwardingBlobStore { private String getMapPath(String container, String name) { checkContainer(container); - return JortageProxy.hashToPath(Queries.getMap(dataSource, container, name).toString()); + return Poolmgr.hashToPath(Queries.getMap(dataSource, container, name).toString()); } private boolean isDump(String name) { @@ -195,7 +195,7 @@ public class JortageBlobStore extends ForwardingBlobStore { @Override public String putBlob(String container, Blob blob) { - JortageProxy.checkReadOnly(); + Poolmgr.checkReadOnly(); checkContainer(container); String blobName = blob.getMetadata().getName(); if (isDump(blobName)) { @@ -203,8 +203,8 @@ public class JortageBlobStore extends ForwardingBlobStore { } File tempFile = null; Object mutex = new Object(); - synchronized (JortageProxy.provisionalMaps) { - JortageProxy.provisionalMaps.put(identity, blobName, mutex); + synchronized (Poolmgr.provisionalMaps) { + Poolmgr.provisionalMaps.put(identity, blobName, mutex); } try { File f = File.createTempFile("jortage-proxy-", ".dat"); @@ -220,12 +220,12 @@ public class JortageBlobStore extends ForwardingBlobStore { String hashString = hash.toString(); try (Payload payload = new FilePayload(f)) { payload.getContentMetadata().setContentType(contentType); - if (delegate().blobExists(bucket, JortageProxy.hashToPath(hashString))) { - String etag = delegate().blobMetadata(bucket, JortageProxy.hashToPath(hashString)).getETag(); + if (delegate().blobExists(bucket, Poolmgr.hashToPath(hashString))) { + String etag = delegate().blobMetadata(bucket, Poolmgr.hashToPath(hashString)).getETag(); Queries.putMap(dataSource, identity, blobName, hash); return etag; } - Blob blob2 = blobBuilder(JortageProxy.hashToPath(hashString)) + Blob blob2 = blobBuilder(Poolmgr.hashToPath(hashString)) .payload(payload) .userMetadata(blob.getMetadata().getUserMetadata()) .build(); @@ -239,8 +239,8 @@ public class JortageBlobStore extends ForwardingBlobStore { throw new UncheckedIOException(e); } finally { if (tempFile != null) tempFile.delete(); - synchronized (JortageProxy.provisionalMaps) { - JortageProxy.provisionalMaps.remove(identity, blobName); + synchronized (Poolmgr.provisionalMaps) { + Poolmgr.provisionalMaps.remove(identity, blobName); } synchronized (mutex) { mutex.notifyAll(); @@ -250,7 +250,7 @@ public class JortageBlobStore extends ForwardingBlobStore { @Override public String copyBlob(String fromContainer, String fromName, String toContainer, String toName, CopyOptions options) { - JortageProxy.checkReadOnly(); + Poolmgr.checkReadOnly(); checkContainer(fromContainer); checkContainer(toContainer); if (isDump(fromName)) { @@ -260,12 +260,12 @@ public class JortageBlobStore extends ForwardingBlobStore { // javadoc says options are ignored, so we ignore them too HashCode hash = Queries.getMap(dataSource, identity, fromName); Queries.putMap(dataSource, identity, toName, hash); - return blobMetadata(bucket, JortageProxy.hashToPath(hash.toString())).getETag(); + return blobMetadata(bucket, Poolmgr.hashToPath(hash.toString())).getETag(); } @Override public MultipartUpload initiateMultipartUpload(String container, BlobMetadata blobMetadata, PutOptions options) { - JortageProxy.checkReadOnly(); + Poolmgr.checkReadOnly(); checkContainer(container); if (isDump(blobMetadata.getName())) { return dumpsStore.initiateMultipartUpload(container, blobMetadata, new PutOptions().setBlobAccess(BlobAccess.PUBLIC_READ)); @@ -291,7 +291,7 @@ public class JortageBlobStore extends ForwardingBlobStore { @Override public void abortMultipartUpload(MultipartUpload mpu) { - JortageProxy.checkReadOnly(); + Poolmgr.checkReadOnly(); if (isDump(mpu.blobName())) { checkContainer(mpu.containerName()); dumpsStore.abortMultipartUpload(mpu); @@ -302,7 +302,7 @@ public class JortageBlobStore extends ForwardingBlobStore { @Override public String completeMultipartUpload(MultipartUpload mpu, List parts) { - JortageProxy.checkReadOnly(); + Poolmgr.checkReadOnly(); if (isDump(mpu.blobName())) { checkContainer(mpu.containerName()); return dumpsStore.completeMultipartUpload(mpu, parts); @@ -316,7 +316,7 @@ public class JortageBlobStore extends ForwardingBlobStore { ByteStreams.copy(stream, hos); HashCode hash = hos.hash(); String hashStr = hash.toString(); - String path = JortageProxy.hashToPath(hashStr); + String path = Poolmgr.hashToPath(hashStr); // we're about to do a bunch of stuff at once // sleep so we don't fall afoul of request rate limits // (causes intermittent 429s on at least DigitalOcean) @@ -347,7 +347,7 @@ public class JortageBlobStore extends ForwardingBlobStore { @Override public MultipartPart uploadMultipartPart(MultipartUpload mpu, int partNumber, Payload payload) { - JortageProxy.checkReadOnly(); + Poolmgr.checkReadOnly(); if (isDump(mpu.blobName())) { checkContainer(mpu.containerName()); return dumpsStore.uploadMultipartPart(mpu, partNumber, payload); @@ -384,7 +384,7 @@ public class JortageBlobStore extends ForwardingBlobStore { @Override public void removeBlob(String container, String name) { - JortageProxy.checkReadOnly(); + Poolmgr.checkReadOnly(); checkContainer(container); if (isDump(name)) { dumpsStore.removeBlob(container, name); @@ -395,7 +395,7 @@ public class JortageBlobStore extends ForwardingBlobStore { int rc = Queries.getMapCount(dataSource, hc); if (rc == 0) { String hashString = hc.toString(); - String path = JortageProxy.hashToPath(hashString); + String path = Poolmgr.hashToPath(hashString); delegate().removeBlob(bucket, path); Queries.removeFilesize(dataSource, hc); Queries.removePendingBackup(dataSource, hc); @@ -417,7 +417,7 @@ public class JortageBlobStore extends ForwardingBlobStore { @Override public boolean createContainerInLocation(Location location, String container) { - JortageProxy.checkReadOnly(); + Poolmgr.checkReadOnly(); checkContainer(container); return true; } @@ -425,7 +425,7 @@ public class JortageBlobStore extends ForwardingBlobStore { @Override public boolean createContainerInLocation(Location location, String container, CreateContainerOptions createContainerOptions) { - JortageProxy.checkReadOnly(); + Poolmgr.checkReadOnly(); checkContainer(container); return true; } diff --git a/src/main/java/com/jortage/proxy/MastodonHackHandler.java b/src/main/java/com/jortage/poolmgr/MastodonHackHandler.java similarity index 92% rename from src/main/java/com/jortage/proxy/MastodonHackHandler.java rename to src/main/java/com/jortage/poolmgr/MastodonHackHandler.java index 9fda288..e0a65bb 100644 --- a/src/main/java/com/jortage/proxy/MastodonHackHandler.java +++ b/src/main/java/com/jortage/poolmgr/MastodonHackHandler.java @@ -1,4 +1,4 @@ -package com.jortage.proxy; +package com.jortage.poolmgr; import java.io.IOException; import java.util.concurrent.Executors; @@ -26,7 +26,7 @@ public class MastodonHackHandler extends HandlerWrapper { public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { String ua = request.getHeader("User-Agent"); ScheduledFuture shortCircuit = null; - if (!JortageProxy.readOnly && ua != null && ua.contains("aws-sdk-ruby") && request.getHeader("Jortage-Dont202") == null + if (!Poolmgr.readOnly && ua != null && ua.contains("aws-sdk-ruby") && request.getHeader("Jortage-Dont202") == null && request.getQueryString() == null && request.getHeader("x-amz-copy-source") == null && (request.getMethod().equals("POST") || request.getMethod().equals("PUT"))) { // Mastodon's uploader has a very short timeout. diff --git a/src/main/java/com/jortage/proxy/MemoryByteSinkSource.java b/src/main/java/com/jortage/poolmgr/MemoryByteSinkSource.java similarity index 97% rename from src/main/java/com/jortage/proxy/MemoryByteSinkSource.java rename to src/main/java/com/jortage/poolmgr/MemoryByteSinkSource.java index 9d978d9..338df42 100644 --- a/src/main/java/com/jortage/proxy/MemoryByteSinkSource.java +++ b/src/main/java/com/jortage/poolmgr/MemoryByteSinkSource.java @@ -1,4 +1,4 @@ -package com.jortage.proxy; +package com.jortage.poolmgr; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; diff --git a/src/main/java/com/jortage/proxy/OuterHandler.java b/src/main/java/com/jortage/poolmgr/OuterHandler.java similarity index 97% rename from src/main/java/com/jortage/proxy/OuterHandler.java rename to src/main/java/com/jortage/poolmgr/OuterHandler.java index aaadc4c..c3c2e83 100644 --- a/src/main/java/com/jortage/proxy/OuterHandler.java +++ b/src/main/java/com/jortage/poolmgr/OuterHandler.java @@ -1,4 +1,4 @@ -package com.jortage.proxy; +package com.jortage.poolmgr; import java.io.IOException; diff --git a/src/main/java/com/jortage/proxy/JortageProxy.java b/src/main/java/com/jortage/poolmgr/Poolmgr.java similarity index 98% rename from src/main/java/com/jortage/proxy/JortageProxy.java rename to src/main/java/com/jortage/poolmgr/Poolmgr.java index 1749913..9a45192 100644 --- a/src/main/java/com/jortage/proxy/JortageProxy.java +++ b/src/main/java/com/jortage/poolmgr/Poolmgr.java @@ -1,4 +1,4 @@ -package com.jortage.proxy; +package com.jortage.poolmgr; import java.io.File; import java.lang.reflect.Field; @@ -40,7 +40,7 @@ import blue.endless.jankson.Jankson; import blue.endless.jankson.JsonObject; import blue.endless.jankson.JsonPrimitive; -public class JortageProxy { +public class Poolmgr { private static final File configFile = new File("config.jkson"); public static JsonObject config; @@ -185,7 +185,7 @@ public class JortageProxy { } catch (Exception e) { System.err.println("failed"); } - System.err.println("This proxy has Super Denim Powers. (Done in "+initSw+")"); + System.err.println("This Poolmgr has Super Denim Powers. (Done in "+initSw+")"); } catch (Throwable t) { System.err.println(" failed"); t.printStackTrace(); diff --git a/src/main/java/com/jortage/proxy/Queries.java b/src/main/java/com/jortage/poolmgr/Queries.java similarity index 99% rename from src/main/java/com/jortage/proxy/Queries.java rename to src/main/java/com/jortage/poolmgr/Queries.java index 3c305c3..6f8f431 100644 --- a/src/main/java/com/jortage/proxy/Queries.java +++ b/src/main/java/com/jortage/poolmgr/Queries.java @@ -1,4 +1,4 @@ -package com.jortage.proxy; +package com.jortage.poolmgr; import java.sql.Connection; import java.sql.PreparedStatement; diff --git a/src/main/java/com/jortage/proxy/RedirHandler.java b/src/main/java/com/jortage/poolmgr/RedirHandler.java similarity index 79% rename from src/main/java/com/jortage/proxy/RedirHandler.java rename to src/main/java/com/jortage/poolmgr/RedirHandler.java index 777e74c..d49e444 100644 --- a/src/main/java/com/jortage/proxy/RedirHandler.java +++ b/src/main/java/com/jortage/poolmgr/RedirHandler.java @@ -1,4 +1,4 @@ -package com.jortage.proxy; +package com.jortage.poolmgr; import java.io.IOException; import java.util.List; @@ -49,13 +49,13 @@ public final class RedirHandler extends AbstractHandler { } return; } - JortageProxy.reloadConfigIfChanged(); + Poolmgr.reloadConfigIfChanged(); try { boolean waited = false; while (true) { Object mutex = null; - synchronized (JortageProxy.provisionalMaps) { - mutex = JortageProxy.provisionalMaps.get(identity, name); + synchronized (Poolmgr.provisionalMaps) { + mutex = Poolmgr.provisionalMaps.get(identity, name); } if (mutex == null) break; waited = true; @@ -68,13 +68,13 @@ public final class RedirHandler extends AbstractHandler { if (waited) { response.setHeader("Jortage-Waited", "true"); } - String hash = Queries.getMap(JortageProxy.dataSource, identity, name).toString(); - BlobAccess ba = JortageProxy.backingBlobStore.getBlobAccess(JortageProxy.bucket, JortageProxy.hashToPath(hash)); + String hash = Queries.getMap(Poolmgr.dataSource, identity, name).toString(); + BlobAccess ba = Poolmgr.backingBlobStore.getBlobAccess(Poolmgr.bucket, Poolmgr.hashToPath(hash)); if (ba != BlobAccess.PUBLIC_READ) { - JortageProxy.backingBlobStore.setBlobAccess(JortageProxy.bucket, JortageProxy.hashToPath(hash), BlobAccess.PUBLIC_READ); + Poolmgr.backingBlobStore.setBlobAccess(Poolmgr.bucket, Poolmgr.hashToPath(hash), BlobAccess.PUBLIC_READ); } response.setHeader("Cache-Control", "public"); - response.setHeader("Location", JortageProxy.publicHost+"/"+JortageProxy.hashToPath(hash)); + response.setHeader("Location", Poolmgr.publicHost+"/"+Poolmgr.hashToPath(hash)); response.setStatus(301); } catch (IllegalArgumentException e) { response.sendError(404); diff --git a/src/main/java/com/jortage/proxy/RivetHandler.java b/src/main/java/com/jortage/poolmgr/RivetHandler.java similarity index 93% rename from src/main/java/com/jortage/proxy/RivetHandler.java rename to src/main/java/com/jortage/poolmgr/RivetHandler.java index da8020a..6e63037 100644 --- a/src/main/java/com/jortage/proxy/RivetHandler.java +++ b/src/main/java/com/jortage/poolmgr/RivetHandler.java @@ -1,4 +1,4 @@ -package com.jortage.proxy; +package com.jortage.poolmgr; import static com.google.common.base.Verify.verify; @@ -141,20 +141,20 @@ public final class RivetHandler extends AbstractHandler { hos.close(); HashCode hash = hos.hash(); String hashStr = hash.toString(); - String path = JortageProxy.hashToPath(hashStr); - if (Queries.isMapped(JortageProxy.dataSource, hash)) { + String path = Poolmgr.hashToPath(hashStr); + if (Queries.isMapped(Poolmgr.dataSource, hash)) { results.put(url, new Pair<>(RivetResult.PRESENT, Temperature.COLD)); } else { - Blob blob = JortageProxy.backingBlobStore.blobBuilder(path) + Blob blob = Poolmgr.backingBlobStore.blobBuilder(path) .payload(bss.getSource()) .contentLength(bss.getSource().size()) .contentType(getRes.body().contentType().toString()) .build(); long size = bss.getSource().size(); - JortageProxy.backingBlobStore.putBlob(JortageProxy.bucket, blob, + Poolmgr.backingBlobStore.putBlob(Poolmgr.bucket, blob, new PutOptions().setBlobAccess(BlobAccess.PUBLIC_READ).multipart(size > 8192)); - Queries.putPendingBackup(JortageProxy.dataSource, hash); - Queries.putFilesize(JortageProxy.dataSource, hash, size); + Queries.putPendingBackup(Poolmgr.dataSource, hash); + Queries.putFilesize(Poolmgr.dataSource, hash, size); results.put(url, new Pair<>(RivetResult.ADDED, Temperature.FREEZING)); } return hash; @@ -171,7 +171,7 @@ public final class RivetHandler extends AbstractHandler { } private HashCode checkShortCircuit(String originalUrl, HttpUrl url, Temperature temp) { - String publicHost = JortageProxy.config.getObject("backend").get(String.class, "publicHost").replaceFirst("^https?://", ""); + String publicHost = Poolmgr.config.getObject("backend").get(String.class, "publicHost").replaceFirst("^https?://", ""); String fullHost = url.host(); if (url.port() != (url.scheme().equals("https") ? 443 : 80)) { fullHost = fullHost+":"+url.port(); @@ -183,7 +183,7 @@ public final class RivetHandler extends AbstractHandler { String hashStr = segments.get(3); if (hashStr.startsWith(prelude) && HEX_MATCHER.matchesAllOf(hashStr)) { HashCode hash = HashCode.fromString(hashStr); - if (Queries.isMapped(JortageProxy.dataSource, hash)) { + if (Queries.isMapped(Poolmgr.dataSource, hash)) { results.put(originalUrl, new Pair<>(RivetResult.FOUND, temp)); return hash; } @@ -247,8 +247,8 @@ public final class RivetHandler extends AbstractHandler { public void handle(String target, org.eclipse.jetty.server.Request baseRequest, HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException { baseRequest.setHandled(true); if ("/retrieve".equals(target)) { - JortageProxy.reloadConfigIfChanged(); - if (JortageProxy.readOnly) { + Poolmgr.reloadConfigIfChanged(); + if (Poolmgr.readOnly) { jsonError(res, 503, "Currently in read-only maintenance mode; try again later"); return; } @@ -308,7 +308,7 @@ public final class RivetHandler extends AbstractHandler { } } try { - Queries.putMap(JortageProxy.dataSource, rreq.identity, destinationPath, hash); + Queries.putMap(Poolmgr.dataSource, rreq.identity, destinationPath, hash); res.setStatus(200); JsonObject obj = new JsonObject(); JsonObject result = new JsonObject(); @@ -322,8 +322,8 @@ public final class RivetHandler extends AbstractHandler { return; } } else if (target.startsWith("/upload/")) { - JortageProxy.reloadConfigIfChanged(); - if (JortageProxy.readOnly) { + Poolmgr.reloadConfigIfChanged(); + if (Poolmgr.readOnly) { jsonError(res, 503, "Currently in read-only maintenance mode; try again later"); return; } @@ -344,7 +344,7 @@ public final class RivetHandler extends AbstractHandler { HashCode hash = HashCode.fromString(hashStr); RivetResult rres; Temperature temp; - if (Queries.isMapped(JortageProxy.dataSource, hash)) { + if (Queries.isMapped(Poolmgr.dataSource, hash)) { rres = RivetResult.FOUND; temp = Temperature.HOT; } else { @@ -368,23 +368,23 @@ public final class RivetHandler extends AbstractHandler { jsonError(res, 400, "Hash of body ("+realHash+") did not match hash in query ("+hash+")"); return; } - Blob blob = JortageProxy.backingBlobStore.blobBuilder(JortageProxy.hashToPath(hash.toString())) + Blob blob = Poolmgr.backingBlobStore.blobBuilder(Poolmgr.hashToPath(hash.toString())) .payload(bss.getSource()) .contentLength(bss.getSource().size()) .contentType(req.getContentType()) .build(); long size = bss.getSource().size(); - JortageProxy.backingBlobStore.putBlob(JortageProxy.bucket, blob, + Poolmgr.backingBlobStore.putBlob(Poolmgr.bucket, blob, new PutOptions().setBlobAccess(BlobAccess.PUBLIC_READ).multipart(size > 8192)); - Queries.putPendingBackup(JortageProxy.dataSource, hash); - Queries.putFilesize(JortageProxy.dataSource, hash, size); + Queries.putPendingBackup(Poolmgr.dataSource, hash); + Queries.putFilesize(Poolmgr.dataSource, hash, size); rres = RivetResult.ADDED; temp = Temperature.FREEZING; } finally { if (bss != null) bss.close(); } } - Queries.putMap(JortageProxy.dataSource, rreq.identity, path, hash); + Queries.putMap(Poolmgr.dataSource, rreq.identity, path, hash); res.setStatus(200); JsonObject obj = new JsonObject(); JsonObject result = new JsonObject(); @@ -457,7 +457,7 @@ public final class RivetHandler extends AbstractHandler { return null; } - if (!JortageProxy.config.containsKey("users") || !JortageProxy.config.getObject("users").containsKey(identity)) { + if (!Poolmgr.config.containsKey("users") || !Poolmgr.config.getObject("users").containsKey(identity)) { jsonError(res, 401, "Rivet-Auth header invalid (Bad access ID)"); return null; } @@ -490,7 +490,7 @@ public final class RivetHandler extends AbstractHandler { } String payloadStr = new String(payload, Charsets.UTF_8); - String key = JortageProxy.config.getObject("users").get(String.class, identity); + String key = Poolmgr.config.getObject("users").get(String.class, identity); assertSuccess(() -> mac.init(new SecretKeySpec(key.getBytes(Charsets.UTF_8), "RAW"))); String query; if (req.getQueryString() == null) { diff --git a/src/main/java/com/jortage/proxy/RivetTest.java b/src/main/java/com/jortage/poolmgr/RivetTest.java similarity index 99% rename from src/main/java/com/jortage/proxy/RivetTest.java rename to src/main/java/com/jortage/poolmgr/RivetTest.java index 3a153e7..e2cd563 100644 --- a/src/main/java/com/jortage/proxy/RivetTest.java +++ b/src/main/java/com/jortage/poolmgr/RivetTest.java @@ -1,4 +1,4 @@ -package com.jortage.proxy; +package com.jortage.poolmgr; import java.io.ByteArrayOutputStream; import java.io.File;