diff --git a/build.gradle b/build.gradle index 19ba0ed..1928ba2 100644 --- a/build.gradle +++ b/build.gradle @@ -1,12 +1,16 @@ plugins { + id 'com.github.johnrengelman.shadow' version '8.1.1' id 'java' - id 'us.kirchmeier.capsule' version '1.0.2' + id 'com.github.ben-manes.versions' version '0.47.0' } repositories { mavenCentral() } +archivesBaseName = 'jortage-poolmgr' +version = '1.4.0' + sourceSets { main { java { @@ -16,37 +20,54 @@ sourceSets { } dependencies { - implementation 'blue.endless:jankson:1.1.2' - implementation 'org.mariadb.jdbc:mariadb-java-client:2.4.4' - implementation 'com.squareup.okhttp3:okhttp:4.7.2' - implementation 'com.squareup.okhttp3:okhttp-brotli:4.7.2' + implementation 'blue.endless:jankson:1.2.3' + implementation 'org.mariadb.jdbc:mariadb-java-client:3.1.4' + implementation 'com.squareup.okhttp3:okhttp:4.11.0' + implementation 'com.squareup.okhttp3:okhttp-brotli:4.11.0' - implementation 'org.apache.jclouds:jclouds-blobstore:2.2.1' - implementation 'org.apache.jclouds.provider:aws-s3:2.2.1' - implementation 'org.apache.jclouds.api:filesystem:2.2.1' - implementation 'org.apache.jclouds.driver:jclouds-slf4j:2.2.1' + implementation 'org.apache.jclouds:jclouds-blobstore:2.5.0' + implementation 'org.apache.jclouds.provider:aws-s3:2.5.0' + implementation 'org.apache.jclouds.api:filesystem:2.5.0' + implementation 'org.apache.jclouds.driver:jclouds-slf4j:2.5.0' - implementation 'org.eclipse.jetty:jetty-server:9.4.24.v20191120' + implementation 'org.eclipse.jetty:jetty-server:11.0.15' - implementation 'org.slf4j:slf4j-api:1.7.9' - implementation 'org.slf4j:slf4j-simple:1.7.9' + implementation 'org.slf4j:slf4j-api:1.7.36' + implementation 'org.slf4j:slf4j-simple:1.7.36' implementation 'com.google.code.findbugs:jsr305:3.0.2' implementation 'com.google.code.findbugs:findbugs-annotations:3.0.1' - implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.11.0' - implementation 'com.fasterxml.woodstox:woodstox-core:6.2.1' + implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.15.2' + implementation 'com.fasterxml.woodstox:woodstox-core:6.5.1' - implementation 'commons-fileupload:commons-fileupload:1.4' + implementation 'commons-fileupload:commons-fileupload:1.5' } -// I am *not* pulling in five different dependencies for a couple classes we don't use -file('s3proxy/src/main/java/org/gaul/s3proxy/junit/S3ProxyRule.java').delete(); -file('s3proxy/src/main/java/org/gaul/s3proxy/Main.java').delete(); +// I am *not* pulling in seven different dependencies for classes we don't use +['junit/S3ProxyRule.java', 'junit/S3ProxyExtension.java', 'Main.java', 'EncryptedBlobStore.java'].each { + file('s3proxy/src/main/java/org/gaul/s3proxy/'+it).delete() +} -project.configurations.implementation.setCanBeResolved(true) +jar { + destinationDirectory.set(file('build/tmp')) +} -task capsule(type: FatCapsule) { - embedConfiguration configurations.implementation - applicationClass 'com.jortage.poolmgr.Poolmgr' +shadowJar { + configurations = [project.configurations.compileClasspath] + manifest.attributes ( + 'Main-Class': 'com.jortage.poolmgr.Poolmgr' + ) + archiveClassifier = '' +} + +build.dependsOn shadowJar + +tasks.named("dependencyUpdates").configure { + gradleReleaseChannel = 'current' + revision = 'release' + rejectVersionIf { + it.candidate.version.contains("alpha") || it.candidate.version.contains("beta") + || (it.candidate.group == 'org.slf4j' && it.candidate.version.startsWith("2.")) + } } \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index bf3de21..84a0b92 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.6-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.2.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/s3proxy b/s3proxy index e638111..cf4db28 160000 --- a/s3proxy +++ b/s3proxy @@ -1 +1 @@ -Subproject commit e638111d05b5e7193d2ae8dd1d0e7a63110d878b +Subproject commit cf4db284a61043a0e78175baae7f889f637eee7c diff --git a/src/main/java/com/jortage/poolmgr/JortageBlobStore.java b/src/main/java/com/jortage/poolmgr/JortageBlobStore.java index 8cfe77c..c6928a1 100644 --- a/src/main/java/com/jortage/poolmgr/JortageBlobStore.java +++ b/src/main/java/com/jortage/poolmgr/JortageBlobStore.java @@ -220,8 +220,9 @@ public class JortageBlobStore extends ForwardingBlobStore { String hashString = hash.toString(); try (Payload payload = new FilePayload(f)) { payload.getContentMetadata().setContentType(contentType); - if (delegate().blobExists(bucket, Poolmgr.hashToPath(hashString))) { - String etag = delegate().blobMetadata(bucket, Poolmgr.hashToPath(hashString)).getETag(); + BlobMetadata meta = delegate().blobMetadata(bucket, Poolmgr.hashToPath(hashString)); + if (meta != null) { + String etag = meta.getETag(); Queries.putMap(dataSource, identity, blobName, hash); return etag; } @@ -302,47 +303,52 @@ public class JortageBlobStore extends ForwardingBlobStore { @Override public String completeMultipartUpload(MultipartUpload mpu, List parts) { - Poolmgr.checkReadOnly(); - if (isDump(mpu.blobName())) { - checkContainer(mpu.containerName()); - return dumpsStore.completeMultipartUpload(mpu, parts); - } - mpu = mask(mpu); - // TODO this is a bit of a hack and isn't very efficient - String etag = delegate().completeMultipartUpload(mpu, parts); - try (InputStream stream = delegate().getBlob(mpu.containerName(), mpu.blobName()).getPayload().openStream()) { - CountingOutputStream counter = new CountingOutputStream(ByteStreams.nullOutputStream()); - HashingOutputStream hos = new HashingOutputStream(Hashing.sha512(), counter); - ByteStreams.copy(stream, hos); - HashCode hash = hos.hash(); - String hashStr = hash.toString(); - String path = Poolmgr.hashToPath(hashStr); - // we're about to do a bunch of stuff at once - // sleep so we don't fall afoul of request rate limits - // (causes intermittent 429s on at least DigitalOcean) - Thread.sleep(250); - BlobMetadata meta = delegate().blobMetadata(mpu.containerName(), mpu.blobName()); - if (!delegate().blobExists(bucket, path)) { - Thread.sleep(250); - etag = delegate().copyBlob(mpu.containerName(), mpu.blobName(), bucket, path, CopyOptions.builder().contentMetadata(meta.getContentMetadata()).build()); - Thread.sleep(250); - delegate().setBlobAccess(bucket, path, BlobAccess.PUBLIC_READ); - Queries.putPendingBackup(dataSource, hash); - } else { - Thread.sleep(250); - etag = delegate().blobMetadata(bucket, path).getETag(); + try { + Poolmgr.checkReadOnly(); + if (isDump(mpu.blobName())) { + checkContainer(mpu.containerName()); + return dumpsStore.completeMultipartUpload(mpu, parts); } - Queries.putMap(dataSource, identity, Preconditions.checkNotNull(meta.getUserMetadata().get("jortage-originalname")), hash); - Queries.putFilesize(dataSource, hash, counter.getCount()); - Queries.removeMultipart(dataSource, mpu.blobName()); - Thread.sleep(250); - delegate().removeBlob(mpu.containerName(), mpu.blobName()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } catch (InterruptedException e) { - throw new RuntimeException(e); + mpu = mask(mpu); + // TODO this is a bit of a hack and isn't very efficient + String etag = delegate().completeMultipartUpload(mpu, parts); + try (InputStream stream = delegate().getBlob(mpu.containerName(), mpu.blobName()).getPayload().openStream()) { + CountingOutputStream counter = new CountingOutputStream(ByteStreams.nullOutputStream()); + HashingOutputStream hos = new HashingOutputStream(Hashing.sha512(), counter); + ByteStreams.copy(stream, hos); + HashCode hash = hos.hash(); + String hashStr = hash.toString(); + String path = Poolmgr.hashToPath(hashStr); + // we're about to do a bunch of stuff at once + // sleep so we don't fall afoul of request rate limits + // (causes intermittent 429s on at least DigitalOcean) + Thread.sleep(100); + BlobMetadata meta = delegate().blobMetadata(mpu.containerName(), mpu.blobName()); + if (meta == null) { + Thread.sleep(100); + etag = delegate().copyBlob(mpu.containerName(), mpu.blobName(), bucket, path, CopyOptions.builder().contentMetadata(meta.getContentMetadata()).build()); + Thread.sleep(100); + delegate().setBlobAccess(bucket, path, BlobAccess.PUBLIC_READ); + Queries.putPendingBackup(dataSource, hash); + } else { + Thread.sleep(100); + etag = delegate().blobMetadata(bucket, path).getETag(); + } + Queries.putMap(dataSource, identity, Preconditions.checkNotNull(meta.getUserMetadata().get("jortage-originalname")), hash); + Queries.putFilesize(dataSource, hash, counter.getCount()); + Queries.removeMultipart(dataSource, mpu.blobName()); + Thread.sleep(100); + delegate().removeBlob(mpu.containerName(), mpu.blobName()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return etag; + } catch (Error | RuntimeException e) { + e.printStackTrace(); + throw e; } - return etag; } @Override @@ -437,18 +443,13 @@ public class JortageBlobStore extends ForwardingBlobStore { private static final String NO_DIR_MSG = "Directories are an illusion"; private static final String NO_BULK_MSG = "Bulk operations are not implemented by Jortage for safety and speed"; - private static final String NO_PRIVATE_MSG = "Jortage does not support private objects"; @Override public void setContainerAccess(String container, ContainerAccess containerAccess) { - if (containerAccess != ContainerAccess.PUBLIC_READ) - throw new UnsupportedOperationException(NO_PRIVATE_MSG); } @Override public void setBlobAccess(String container, String name, BlobAccess access) { - if (access != BlobAccess.PUBLIC_READ) - throw new UnsupportedOperationException(NO_PRIVATE_MSG); } @Override diff --git a/src/main/java/com/jortage/poolmgr/MastodonHackHandler.java b/src/main/java/com/jortage/poolmgr/MastodonHackHandler.java index 517d449..3793f3d 100644 --- a/src/main/java/com/jortage/poolmgr/MastodonHackHandler.java +++ b/src/main/java/com/jortage/poolmgr/MastodonHackHandler.java @@ -6,9 +6,9 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Request; @@ -42,10 +42,13 @@ public class MastodonHackHandler extends HandlerWrapper { response.sendError(202); } catch (IOException e) { } - }, 4000, TimeUnit.MILLISECONDS); + }, 2000, TimeUnit.MILLISECONDS); + } + try { + super.handle(target, baseRequest, request, response); + } finally { + if (shortCircuit != null) shortCircuit.cancel(false); } - super.handle(target, baseRequest, request, response); - if (shortCircuit != null) shortCircuit.cancel(false); } } diff --git a/src/main/java/com/jortage/poolmgr/OuterHandler.java b/src/main/java/com/jortage/poolmgr/OuterHandler.java index c3c2e83..6dbafc7 100644 --- a/src/main/java/com/jortage/poolmgr/OuterHandler.java +++ b/src/main/java/com/jortage/poolmgr/OuterHandler.java @@ -2,9 +2,9 @@ package com.jortage.poolmgr; import java.io.IOException; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Request; diff --git a/src/main/java/com/jortage/poolmgr/Poolmgr.java b/src/main/java/com/jortage/poolmgr/Poolmgr.java index 9a45192..41d4772 100644 --- a/src/main/java/com/jortage/poolmgr/Poolmgr.java +++ b/src/main/java/com/jortage/poolmgr/Poolmgr.java @@ -58,7 +58,6 @@ public class Poolmgr { public static final Table provisionalMaps = HashBasedTable.create(); - @SuppressWarnings("restriction") public static void main(String[] args) throws Exception { try { Stopwatch initSw = Stopwatch.createStarted(); @@ -71,6 +70,10 @@ public class Poolmgr { .endpoint(URI.create("http://localhost:23278")) .jettyMaxThreads(24) .v4MaxNonChunkedRequestSize(128L*1024L*1024L) + // S3Proxy will throw if it sees an X-Amz header it doesn't recognize + // Misskey, starting in some recent version (as of July 2023) now sends an X-Amz-User-Agent header + // So without this, Misskey instances can't upload files. Cool thanks + .ignoreUnknownHeaders(true) .build(); // excuse me, this is mine now diff --git a/src/main/java/com/jortage/poolmgr/RedirHandler.java b/src/main/java/com/jortage/poolmgr/RedirHandler.java index 516e99e..a67a998 100644 --- a/src/main/java/com/jortage/poolmgr/RedirHandler.java +++ b/src/main/java/com/jortage/poolmgr/RedirHandler.java @@ -2,9 +2,9 @@ package com.jortage.poolmgr; import java.io.IOException; import java.util.List; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.handler.AbstractHandler; import org.jclouds.blobstore.BlobStore; diff --git a/src/main/java/com/jortage/poolmgr/RivetHandler.java b/src/main/java/com/jortage/poolmgr/RivetHandler.java index 6e63037..d5c904a 100644 --- a/src/main/java/com/jortage/poolmgr/RivetHandler.java +++ b/src/main/java/com/jortage/poolmgr/RivetHandler.java @@ -24,9 +24,9 @@ import kotlin.Pair; import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import org.eclipse.jetty.server.handler.AbstractHandler; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.BlobAccess;