mirror
/
poolmgr
kopia lustrzana https://github.com/jortage/poolmgr
1
0
Forkuj 0
Kod Zgłoszenia Packages Projekty Wydania Wiki Aktywność

Rename to poolmgr

trunk
Una Thompson 2020-06-20 14:33:35 -07:00
rodzic 019d5fa22f
commit 0ee3368f9b
13 zmienionych plików z 64 dodań i 64 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

2
README.md
Wyświetl plik

@ -1,4 +1,4 @@
# Jortage Storage Pool Proxy
# Jortage Storage Pool Manager
A miniature S3-to-S3 proxy that accepts incoming requests, hashes the uploaded
file, stores their desired path in a MariaDB database, along with the hash, and
then uploads the original file to a backing S3 server named after the

2
build.gradle
Wyświetl plik

@ -48,5 +48,5 @@ project.configurations.implementation.setCanBeResolved(true)
task capsule(type: FatCapsule) {
embedConfiguration configurations.implementation
applicationClass 'com.jortage.proxy.JortageProxy'
applicationClass 'com.jortage.poolmgr.Poolmgr'
}

2
src/main/java/com/jortage/proxy/ByteSinkSource.java → src/main/java/com/jortage/poolmgr/ByteSinkSource.java
Wyświetl plik

@ -1,4 +1,4 @@
package com.jortage.proxy;
package com.jortage.poolmgr;
import java.io.Closeable;
import com.google.common.io.ByteSink;

2
src/main/java/com/jortage/proxy/FileByteSinkSource.java → src/main/java/com/jortage/poolmgr/FileByteSinkSource.java
Wyświetl plik

@ -1,4 +1,4 @@
package com.jortage.proxy;
package com.jortage.poolmgr;
import java.io.File;

42
src/main/java/com/jortage/proxy/JortageBlobStore.java → src/main/java/com/jortage/poolmgr/JortageBlobStore.java
Wyświetl plik

@ -1,4 +1,4 @@
package com.jortage.proxy;
package com.jortage.poolmgr;
import java.io.File;
import java.io.FileOutputStream;
@ -69,7 +69,7 @@ public class JortageBlobStore extends ForwardingBlobStore {
private String getMapPath(String container, String name) {
checkContainer(container);
return JortageProxy.hashToPath(Queries.getMap(dataSource, container, name).toString());
return Poolmgr.hashToPath(Queries.getMap(dataSource, container, name).toString());
}
private boolean isDump(String name) {
@ -195,7 +195,7 @@ public class JortageBlobStore extends ForwardingBlobStore {
@Override
public String putBlob(String container, Blob blob) {
JortageProxy.checkReadOnly();
Poolmgr.checkReadOnly();
checkContainer(container);
String blobName = blob.getMetadata().getName();
if (isDump(blobName)) {
@ -203,8 +203,8 @@ public class JortageBlobStore extends ForwardingBlobStore {
}
File tempFile = null;
Object mutex = new Object();
synchronized (JortageProxy.provisionalMaps) {
JortageProxy.provisionalMaps.put(identity, blobName, mutex);
synchronized (Poolmgr.provisionalMaps) {
Poolmgr.provisionalMaps.put(identity, blobName, mutex);
}
try {
File f = File.createTempFile("jortage-proxy-", ".dat");
@ -220,12 +220,12 @@ public class JortageBlobStore extends ForwardingBlobStore {
String hashString = hash.toString();
try (Payload payload = new FilePayload(f)) {
payload.getContentMetadata().setContentType(contentType);
if (delegate().blobExists(bucket, JortageProxy.hashToPath(hashString))) {
String etag = delegate().blobMetadata(bucket, JortageProxy.hashToPath(hashString)).getETag();
if (delegate().blobExists(bucket, Poolmgr.hashToPath(hashString))) {
String etag = delegate().blobMetadata(bucket, Poolmgr.hashToPath(hashString)).getETag();
Queries.putMap(dataSource, identity, blobName, hash);
return etag;
}
Blob blob2 = blobBuilder(JortageProxy.hashToPath(hashString))
Blob blob2 = blobBuilder(Poolmgr.hashToPath(hashString))
.payload(payload)
.userMetadata(blob.getMetadata().getUserMetadata())
.build();
@ -239,8 +239,8 @@ public class JortageBlobStore extends ForwardingBlobStore {
throw new UncheckedIOException(e);
} finally {
if (tempFile != null) tempFile.delete();
synchronized (JortageProxy.provisionalMaps) {
JortageProxy.provisionalMaps.remove(identity, blobName);
synchronized (Poolmgr.provisionalMaps) {
Poolmgr.provisionalMaps.remove(identity, blobName);
}
synchronized (mutex) {
mutex.notifyAll();
@ -250,7 +250,7 @@ public class JortageBlobStore extends ForwardingBlobStore {
@Override
public String copyBlob(String fromContainer, String fromName, String toContainer, String toName, CopyOptions options) {
JortageProxy.checkReadOnly();
Poolmgr.checkReadOnly();
checkContainer(fromContainer);
checkContainer(toContainer);
if (isDump(fromName)) {
@ -260,12 +260,12 @@ public class JortageBlobStore extends ForwardingBlobStore {
// javadoc says options are ignored, so we ignore them too
HashCode hash = Queries.getMap(dataSource, identity, fromName);
Queries.putMap(dataSource, identity, toName, hash);
return blobMetadata(bucket, JortageProxy.hashToPath(hash.toString())).getETag();
return blobMetadata(bucket, Poolmgr.hashToPath(hash.toString())).getETag();
}
@Override
public MultipartUpload initiateMultipartUpload(String container, BlobMetadata blobMetadata, PutOptions options) {
JortageProxy.checkReadOnly();
Poolmgr.checkReadOnly();
checkContainer(container);
if (isDump(blobMetadata.getName())) {
return dumpsStore.initiateMultipartUpload(container, blobMetadata, new PutOptions().setBlobAccess(BlobAccess.PUBLIC_READ));
@ -291,7 +291,7 @@ public class JortageBlobStore extends ForwardingBlobStore {
@Override
public void abortMultipartUpload(MultipartUpload mpu) {
JortageProxy.checkReadOnly();
Poolmgr.checkReadOnly();
if (isDump(mpu.blobName())) {
checkContainer(mpu.containerName());
dumpsStore.abortMultipartUpload(mpu);
@ -302,7 +302,7 @@ public class JortageBlobStore extends ForwardingBlobStore {
@Override
public String completeMultipartUpload(MultipartUpload mpu, List<MultipartPart> parts) {
JortageProxy.checkReadOnly();
Poolmgr.checkReadOnly();
if (isDump(mpu.blobName())) {
checkContainer(mpu.containerName());
return dumpsStore.completeMultipartUpload(mpu, parts);
@ -316,7 +316,7 @@ public class JortageBlobStore extends ForwardingBlobStore {
ByteStreams.copy(stream, hos);
HashCode hash = hos.hash();
String hashStr = hash.toString();
String path = JortageProxy.hashToPath(hashStr);
String path = Poolmgr.hashToPath(hashStr);
// we're about to do a bunch of stuff at once
// sleep so we don't fall afoul of request rate limits
// (causes intermittent 429s on at least DigitalOcean)
@ -347,7 +347,7 @@ public class JortageBlobStore extends ForwardingBlobStore {
@Override
public MultipartPart uploadMultipartPart(MultipartUpload mpu, int partNumber, Payload payload) {
JortageProxy.checkReadOnly();
Poolmgr.checkReadOnly();
if (isDump(mpu.blobName())) {
checkContainer(mpu.containerName());
return dumpsStore.uploadMultipartPart(mpu, partNumber, payload);
@ -384,7 +384,7 @@ public class JortageBlobStore extends ForwardingBlobStore {
@Override
public void removeBlob(String container, String name) {
JortageProxy.checkReadOnly();
Poolmgr.checkReadOnly();
checkContainer(container);
if (isDump(name)) {
dumpsStore.removeBlob(container, name);
@ -395,7 +395,7 @@ public class JortageBlobStore extends ForwardingBlobStore {
int rc = Queries.getMapCount(dataSource, hc);
if (rc == 0) {
String hashString = hc.toString();
String path = JortageProxy.hashToPath(hashString);
String path = Poolmgr.hashToPath(hashString);
delegate().removeBlob(bucket, path);
Queries.removeFilesize(dataSource, hc);
Queries.removePendingBackup(dataSource, hc);
@ -417,7 +417,7 @@ public class JortageBlobStore extends ForwardingBlobStore {
@Override
public boolean createContainerInLocation(Location location, String container) {
JortageProxy.checkReadOnly();
Poolmgr.checkReadOnly();
checkContainer(container);
return true;
}
@ -425,7 +425,7 @@ public class JortageBlobStore extends ForwardingBlobStore {
@Override
public boolean createContainerInLocation(Location location,
String container, CreateContainerOptions createContainerOptions) {
JortageProxy.checkReadOnly();
Poolmgr.checkReadOnly();
checkContainer(container);
return true;
}

4
src/main/java/com/jortage/proxy/MastodonHackHandler.java → src/main/java/com/jortage/poolmgr/MastodonHackHandler.java
Wyświetl plik

@ -1,4 +1,4 @@
package com.jortage.proxy;
package com.jortage.poolmgr;
import java.io.IOException;
import java.util.concurrent.Executors;
@ -26,7 +26,7 @@ public class MastodonHackHandler extends HandlerWrapper {
public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
String ua = request.getHeader("User-Agent");
ScheduledFuture<?> shortCircuit = null;
if (!JortageProxy.readOnly && ua != null && ua.contains("aws-sdk-ruby") && request.getHeader("Jortage-Dont202") == null
if (!Poolmgr.readOnly && ua != null && ua.contains("aws-sdk-ruby") && request.getHeader("Jortage-Dont202") == null
&& request.getQueryString() == null && request.getHeader("x-amz-copy-source") == null
&& (request.getMethod().equals("POST") || request.getMethod().equals("PUT"))) {
// Mastodon's uploader has a very short timeout.

2
src/main/java/com/jortage/proxy/MemoryByteSinkSource.java → src/main/java/com/jortage/poolmgr/MemoryByteSinkSource.java
Wyświetl plik

@ -1,4 +1,4 @@
package com.jortage.proxy;
package com.jortage.poolmgr;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;

2
src/main/java/com/jortage/proxy/OuterHandler.java → src/main/java/com/jortage/poolmgr/OuterHandler.java
Wyświetl plik

@ -1,4 +1,4 @@
package com.jortage.proxy;
package com.jortage.poolmgr;
import java.io.IOException;

6
src/main/java/com/jortage/proxy/JortageProxy.java → src/main/java/com/jortage/poolmgr/Poolmgr.java
Wyświetl plik

@ -1,4 +1,4 @@
package com.jortage.proxy;
package com.jortage.poolmgr;
import java.io.File;
import java.lang.reflect.Field;
@ -40,7 +40,7 @@ import blue.endless.jankson.Jankson;
import blue.endless.jankson.JsonObject;
import blue.endless.jankson.JsonPrimitive;
public class JortageProxy {
public class Poolmgr {
private static final File configFile = new File("config.jkson");
public static JsonObject config;
@ -185,7 +185,7 @@ public class JortageProxy {
} catch (Exception e) {
System.err.println("failed");
}
System.err.println("This proxy has Super Denim Powers. (Done in "+initSw+")");
System.err.println("This Poolmgr has Super Denim Powers. (Done in "+initSw+")");
} catch (Throwable t) {
System.err.println(" failed");
t.printStackTrace();

2
src/main/java/com/jortage/proxy/Queries.java → src/main/java/com/jortage/poolmgr/Queries.java
Wyświetl plik

@ -1,4 +1,4 @@
package com.jortage.proxy;
package com.jortage.poolmgr;
import java.sql.Connection;
import java.sql.PreparedStatement;

16
src/main/java/com/jortage/proxy/RedirHandler.java → src/main/java/com/jortage/poolmgr/RedirHandler.java
Wyświetl plik

@ -1,4 +1,4 @@
package com.jortage.proxy;
package com.jortage.poolmgr;
import java.io.IOException;
import java.util.List;
@ -49,13 +49,13 @@ public final class RedirHandler extends AbstractHandler {
}
return;
}
JortageProxy.reloadConfigIfChanged();
Poolmgr.reloadConfigIfChanged();
try {
boolean waited = false;
while (true) {
Object mutex = null;
synchronized (JortageProxy.provisionalMaps) {
mutex = JortageProxy.provisionalMaps.get(identity, name);
synchronized (Poolmgr.provisionalMaps) {
mutex = Poolmgr.provisionalMaps.get(identity, name);
}
if (mutex == null) break;
waited = true;
@ -68,13 +68,13 @@ public final class RedirHandler extends AbstractHandler {
if (waited) {
response.setHeader("Jortage-Waited", "true");
}
String hash = Queries.getMap(JortageProxy.dataSource, identity, name).toString();
BlobAccess ba = JortageProxy.backingBlobStore.getBlobAccess(JortageProxy.bucket, JortageProxy.hashToPath(hash));
String hash = Queries.getMap(Poolmgr.dataSource, identity, name).toString();
BlobAccess ba = Poolmgr.backingBlobStore.getBlobAccess(Poolmgr.bucket, Poolmgr.hashToPath(hash));
if (ba != BlobAccess.PUBLIC_READ) {
JortageProxy.backingBlobStore.setBlobAccess(JortageProxy.bucket, JortageProxy.hashToPath(hash), BlobAccess.PUBLIC_READ);
Poolmgr.backingBlobStore.setBlobAccess(Poolmgr.bucket, Poolmgr.hashToPath(hash), BlobAccess.PUBLIC_READ);
}
response.setHeader("Cache-Control", "public");
response.setHeader("Location", JortageProxy.publicHost+"/"+JortageProxy.hashToPath(hash));
response.setHeader("Location", Poolmgr.publicHost+"/"+Poolmgr.hashToPath(hash));
response.setStatus(301);
} catch (IllegalArgumentException e) {
response.sendError(404);

44
src/main/java/com/jortage/proxy/RivetHandler.java → src/main/java/com/jortage/poolmgr/RivetHandler.java
Wyświetl plik

@ -1,4 +1,4 @@
package com.jortage.proxy;
package com.jortage.poolmgr;
import static com.google.common.base.Verify.verify;
@ -141,20 +141,20 @@ public final class RivetHandler extends AbstractHandler {
hos.close();
HashCode hash = hos.hash();
String hashStr = hash.toString();
String path = JortageProxy.hashToPath(hashStr);
if (Queries.isMapped(JortageProxy.dataSource, hash)) {
String path = Poolmgr.hashToPath(hashStr);
if (Queries.isMapped(Poolmgr.dataSource, hash)) {
results.put(url, new Pair<>(RivetResult.PRESENT, Temperature.COLD));
} else {
Blob blob = JortageProxy.backingBlobStore.blobBuilder(path)
Blob blob = Poolmgr.backingBlobStore.blobBuilder(path)
.payload(bss.getSource())
.contentLength(bss.getSource().size())
.contentType(getRes.body().contentType().toString())
.build();
long size = bss.getSource().size();
JortageProxy.backingBlobStore.putBlob(JortageProxy.bucket, blob,
Poolmgr.backingBlobStore.putBlob(Poolmgr.bucket, blob,
new PutOptions().setBlobAccess(BlobAccess.PUBLIC_READ).multipart(size > 8192));
Queries.putPendingBackup(JortageProxy.dataSource, hash);
Queries.putFilesize(JortageProxy.dataSource, hash, size);
Queries.putPendingBackup(Poolmgr.dataSource, hash);
Queries.putFilesize(Poolmgr.dataSource, hash, size);
results.put(url, new Pair<>(RivetResult.ADDED, Temperature.FREEZING));
}
return hash;
@ -171,7 +171,7 @@ public final class RivetHandler extends AbstractHandler {
}
private HashCode checkShortCircuit(String originalUrl, HttpUrl url, Temperature temp) {
String publicHost = JortageProxy.config.getObject("backend").get(String.class, "publicHost").replaceFirst("^https?://", "");
String publicHost = Poolmgr.config.getObject("backend").get(String.class, "publicHost").replaceFirst("^https?://", "");
String fullHost = url.host();
if (url.port() != (url.scheme().equals("https") ? 443 : 80)) {
fullHost = fullHost+":"+url.port();
@ -183,7 +183,7 @@ public final class RivetHandler extends AbstractHandler {
String hashStr = segments.get(3);
if (hashStr.startsWith(prelude) && HEX_MATCHER.matchesAllOf(hashStr)) {
HashCode hash = HashCode.fromString(hashStr);
if (Queries.isMapped(JortageProxy.dataSource, hash)) {
if (Queries.isMapped(Poolmgr.dataSource, hash)) {
results.put(originalUrl, new Pair<>(RivetResult.FOUND, temp));
return hash;
}
@ -247,8 +247,8 @@ public final class RivetHandler extends AbstractHandler {
public void handle(String target, org.eclipse.jetty.server.Request baseRequest, HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException {
baseRequest.setHandled(true);
if ("/retrieve".equals(target)) {
JortageProxy.reloadConfigIfChanged();
if (JortageProxy.readOnly) {
Poolmgr.reloadConfigIfChanged();
if (Poolmgr.readOnly) {
jsonError(res, 503, "Currently in read-only maintenance mode; try again later");
return;
}
@ -308,7 +308,7 @@ public final class RivetHandler extends AbstractHandler {
}
}
try {
Queries.putMap(JortageProxy.dataSource, rreq.identity, destinationPath, hash);
Queries.putMap(Poolmgr.dataSource, rreq.identity, destinationPath, hash);
res.setStatus(200);
JsonObject obj = new JsonObject();
JsonObject result = new JsonObject();
@ -322,8 +322,8 @@ public final class RivetHandler extends AbstractHandler {
return;
}
} else if (target.startsWith("/upload/")) {
JortageProxy.reloadConfigIfChanged();
if (JortageProxy.readOnly) {
Poolmgr.reloadConfigIfChanged();
if (Poolmgr.readOnly) {
jsonError(res, 503, "Currently in read-only maintenance mode; try again later");
return;
}
@ -344,7 +344,7 @@ public final class RivetHandler extends AbstractHandler {
HashCode hash = HashCode.fromString(hashStr);
RivetResult rres;
Temperature temp;
if (Queries.isMapped(JortageProxy.dataSource, hash)) {
if (Queries.isMapped(Poolmgr.dataSource, hash)) {
rres = RivetResult.FOUND;
temp = Temperature.HOT;
} else {
@ -368,23 +368,23 @@ public final class RivetHandler extends AbstractHandler {
jsonError(res, 400, "Hash of body ("+realHash+") did not match hash in query ("+hash+")");
return;
}
Blob blob = JortageProxy.backingBlobStore.blobBuilder(JortageProxy.hashToPath(hash.toString()))
Blob blob = Poolmgr.backingBlobStore.blobBuilder(Poolmgr.hashToPath(hash.toString()))
.payload(bss.getSource())
.contentLength(bss.getSource().size())
.contentType(req.getContentType())
.build();
long size = bss.getSource().size();
JortageProxy.backingBlobStore.putBlob(JortageProxy.bucket, blob,
Poolmgr.backingBlobStore.putBlob(Poolmgr.bucket, blob,
new PutOptions().setBlobAccess(BlobAccess.PUBLIC_READ).multipart(size > 8192));
Queries.putPendingBackup(JortageProxy.dataSource, hash);
Queries.putFilesize(JortageProxy.dataSource, hash, size);
Queries.putPendingBackup(Poolmgr.dataSource, hash);
Queries.putFilesize(Poolmgr.dataSource, hash, size);
rres = RivetResult.ADDED;
temp = Temperature.FREEZING;
} finally {
if (bss != null) bss.close();
}
}
Queries.putMap(JortageProxy.dataSource, rreq.identity, path, hash);
Queries.putMap(Poolmgr.dataSource, rreq.identity, path, hash);
res.setStatus(200);
JsonObject obj = new JsonObject();
JsonObject result = new JsonObject();
@ -457,7 +457,7 @@ public final class RivetHandler extends AbstractHandler {
return null;
}
if (!JortageProxy.config.containsKey("users") || !JortageProxy.config.getObject("users").containsKey(identity)) {
if (!Poolmgr.config.containsKey("users") || !Poolmgr.config.getObject("users").containsKey(identity)) {
jsonError(res, 401, "Rivet-Auth header invalid (Bad access ID)");
return null;
}
@ -490,7 +490,7 @@ public final class RivetHandler extends AbstractHandler {
}
String payloadStr = new String(payload, Charsets.UTF_8);
String key = JortageProxy.config.getObject("users").get(String.class, identity);
String key = Poolmgr.config.getObject("users").get(String.class, identity);
assertSuccess(() -> mac.init(new SecretKeySpec(key.getBytes(Charsets.UTF_8), "RAW")));
String query;
if (req.getQueryString() == null) {

2
src/main/java/com/jortage/proxy/RivetTest.java → src/main/java/com/jortage/poolmgr/RivetTest.java
Wyświetl plik

@ -1,4 +1,4 @@
package com.jortage.proxy;
package com.jortage.poolmgr;
import java.io.ByteArrayOutputStream;
import java.io.File;