mirror
/
poolmgr
kopia lustrzana https://github.com/jortage/poolmgr
1
0
Forkuj 0
Kod Zgłoszenia Packages Projekty Wydania Wiki Aktywność

Dependency updates, modernize Gradle setup, fix Misskey compat 

Also a few things I forgot to commit and don't know what changed anymore.
Something to do with multipart uploads, and removing some unnecessary error throws.
trunk
Una Thompson 2023-07-23 00:08:22 -07:00
rodzic fc72c27ce1
commit e7b66883d1
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: CD9D524194BE98F5
9 zmienionych plików z 114 dodań i 86 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

65
build.gradle
Wyświetl plik

@ -1,12 +1,16 @@
plugins {
id 'com.github.johnrengelman.shadow' version '8.1.1'
id 'java'
id 'us.kirchmeier.capsule' version '1.0.2'
id 'com.github.ben-manes.versions' version '0.47.0'
}
repositories {
mavenCentral()
}
archivesBaseName = 'jortage-poolmgr'
version = '1.4.0'
sourceSets {
main {
java {
@ -16,37 +20,54 @@ sourceSets {
}
dependencies {
implementation 'blue.endless:jankson:1.1.2'
implementation 'org.mariadb.jdbc:mariadb-java-client:2.4.4'
implementation 'com.squareup.okhttp3:okhttp:4.7.2'
implementation 'com.squareup.okhttp3:okhttp-brotli:4.7.2'
implementation 'blue.endless:jankson:1.2.3'
implementation 'org.mariadb.jdbc:mariadb-java-client:3.1.4'
implementation 'com.squareup.okhttp3:okhttp:4.11.0'
implementation 'com.squareup.okhttp3:okhttp-brotli:4.11.0'
implementation 'org.apache.jclouds:jclouds-blobstore:2.2.1'
implementation 'org.apache.jclouds.provider:aws-s3:2.2.1'
implementation 'org.apache.jclouds.api:filesystem:2.2.1'
implementation 'org.apache.jclouds.driver:jclouds-slf4j:2.2.1'
implementation 'org.apache.jclouds:jclouds-blobstore:2.5.0'
implementation 'org.apache.jclouds.provider:aws-s3:2.5.0'
implementation 'org.apache.jclouds.api:filesystem:2.5.0'
implementation 'org.apache.jclouds.driver:jclouds-slf4j:2.5.0'
implementation 'org.eclipse.jetty:jetty-server:9.4.24.v20191120'
implementation 'org.eclipse.jetty:jetty-server:11.0.15'
implementation 'org.slf4j:slf4j-api:1.7.9'
implementation 'org.slf4j:slf4j-simple:1.7.9'
implementation 'org.slf4j:slf4j-api:1.7.36'
implementation 'org.slf4j:slf4j-simple:1.7.36'
implementation 'com.google.code.findbugs:jsr305:3.0.2'
implementation 'com.google.code.findbugs:findbugs-annotations:3.0.1'
implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.11.0'
implementation 'com.fasterxml.woodstox:woodstox-core:6.2.1'
implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.15.2'
implementation 'com.fasterxml.woodstox:woodstox-core:6.5.1'
implementation 'commons-fileupload:commons-fileupload:1.4'
implementation 'commons-fileupload:commons-fileupload:1.5'
}
// I am *not* pulling in five different dependencies for a couple classes we don't use
file('s3proxy/src/main/java/org/gaul/s3proxy/junit/S3ProxyRule.java').delete();
file('s3proxy/src/main/java/org/gaul/s3proxy/Main.java').delete();
// I am *not* pulling in seven different dependencies for classes we don't use
['junit/S3ProxyRule.java', 'junit/S3ProxyExtension.java', 'Main.java', 'EncryptedBlobStore.java'].each {
file('s3proxy/src/main/java/org/gaul/s3proxy/'+it).delete()
}
project.configurations.implementation.setCanBeResolved(true)
jar {
destinationDirectory.set(file('build/tmp'))
}
task capsule(type: FatCapsule) {
embedConfiguration configurations.implementation
applicationClass 'com.jortage.poolmgr.Poolmgr'
shadowJar {
configurations = [project.configurations.compileClasspath]
manifest.attributes (
'Main-Class': 'com.jortage.poolmgr.Poolmgr'
)
archiveClassifier = ''
}
build.dependsOn shadowJar
tasks.named("dependencyUpdates").configure {
gradleReleaseChannel = 'current'
revision = 'release'
rejectVersionIf {
it.candidate.version.contains("alpha") || it.candidate.version.contains("beta")
|| (it.candidate.group == 'org.slf4j' && it.candidate.version.startsWith("2."))
}
}

2
gradle/wrapper/gradle-wrapper.properties vendored
Wyświetl plik

@ -1,5 +1,5 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-4.6-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-8.2.1-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists

2
s3proxy

@ -1 +1 @@
Subproject commit e638111d05b5e7193d2ae8dd1d0e7a63110d878b
Subproject commit cf4db284a61043a0e78175baae7f889f637eee7c

93
src/main/java/com/jortage/poolmgr/JortageBlobStore.java
Wyświetl plik

@ -220,8 +220,9 @@ public class JortageBlobStore extends ForwardingBlobStore {
String hashString = hash.toString();
try (Payload payload = new FilePayload(f)) {
payload.getContentMetadata().setContentType(contentType);
if (delegate().blobExists(bucket, Poolmgr.hashToPath(hashString))) {
String etag = delegate().blobMetadata(bucket, Poolmgr.hashToPath(hashString)).getETag();
BlobMetadata meta = delegate().blobMetadata(bucket, Poolmgr.hashToPath(hashString));
if (meta != null) {
String etag = meta.getETag();
Queries.putMap(dataSource, identity, blobName, hash);
return etag;
}
@ -302,47 +303,52 @@ public class JortageBlobStore extends ForwardingBlobStore {
@Override
public String completeMultipartUpload(MultipartUpload mpu, List<MultipartPart> parts) {
Poolmgr.checkReadOnly();
if (isDump(mpu.blobName())) {
checkContainer(mpu.containerName());
return dumpsStore.completeMultipartUpload(mpu, parts);
}
mpu = mask(mpu);
// TODO this is a bit of a hack and isn't very efficient
String etag = delegate().completeMultipartUpload(mpu, parts);
try (InputStream stream = delegate().getBlob(mpu.containerName(), mpu.blobName()).getPayload().openStream()) {
CountingOutputStream counter = new CountingOutputStream(ByteStreams.nullOutputStream());
HashingOutputStream hos = new HashingOutputStream(Hashing.sha512(), counter);
ByteStreams.copy(stream, hos);
HashCode hash = hos.hash();
String hashStr = hash.toString();
String path = Poolmgr.hashToPath(hashStr);
// we're about to do a bunch of stuff at once
// sleep so we don't fall afoul of request rate limits
// (causes intermittent 429s on at least DigitalOcean)
Thread.sleep(250);
BlobMetadata meta = delegate().blobMetadata(mpu.containerName(), mpu.blobName());
if (!delegate().blobExists(bucket, path)) {
Thread.sleep(250);
etag = delegate().copyBlob(mpu.containerName(), mpu.blobName(), bucket, path, CopyOptions.builder().contentMetadata(meta.getContentMetadata()).build());
Thread.sleep(250);
delegate().setBlobAccess(bucket, path, BlobAccess.PUBLIC_READ);
Queries.putPendingBackup(dataSource, hash);
} else {
Thread.sleep(250);
etag = delegate().blobMetadata(bucket, path).getETag();
try {
Poolmgr.checkReadOnly();
if (isDump(mpu.blobName())) {
checkContainer(mpu.containerName());
return dumpsStore.completeMultipartUpload(mpu, parts);
}
Queries.putMap(dataSource, identity, Preconditions.checkNotNull(meta.getUserMetadata().get("jortage-originalname")), hash);
Queries.putFilesize(dataSource, hash, counter.getCount());
Queries.removeMultipart(dataSource, mpu.blobName());
Thread.sleep(250);
delegate().removeBlob(mpu.containerName(), mpu.blobName());
} catch (IOException e) {
throw new UncheckedIOException(e);
} catch (InterruptedException e) {
throw new RuntimeException(e);
mpu = mask(mpu);
// TODO this is a bit of a hack and isn't very efficient
String etag = delegate().completeMultipartUpload(mpu, parts);
try (InputStream stream = delegate().getBlob(mpu.containerName(), mpu.blobName()).getPayload().openStream()) {
CountingOutputStream counter = new CountingOutputStream(ByteStreams.nullOutputStream());
HashingOutputStream hos = new HashingOutputStream(Hashing.sha512(), counter);
ByteStreams.copy(stream, hos);
HashCode hash = hos.hash();
String hashStr = hash.toString();
String path = Poolmgr.hashToPath(hashStr);
// we're about to do a bunch of stuff at once
// sleep so we don't fall afoul of request rate limits
// (causes intermittent 429s on at least DigitalOcean)
Thread.sleep(100);
BlobMetadata meta = delegate().blobMetadata(mpu.containerName(), mpu.blobName());
if (meta == null) {
Thread.sleep(100);
etag = delegate().copyBlob(mpu.containerName(), mpu.blobName(), bucket, path, CopyOptions.builder().contentMetadata(meta.getContentMetadata()).build());
Thread.sleep(100);
delegate().setBlobAccess(bucket, path, BlobAccess.PUBLIC_READ);
Queries.putPendingBackup(dataSource, hash);
} else {
Thread.sleep(100);
etag = delegate().blobMetadata(bucket, path).getETag();
}
Queries.putMap(dataSource, identity, Preconditions.checkNotNull(meta.getUserMetadata().get("jortage-originalname")), hash);
Queries.putFilesize(dataSource, hash, counter.getCount());
Queries.removeMultipart(dataSource, mpu.blobName());
Thread.sleep(100);
delegate().removeBlob(mpu.containerName(), mpu.blobName());
} catch (IOException e) {
throw new UncheckedIOException(e);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
return etag;
} catch (Error | RuntimeException e) {
e.printStackTrace();
throw e;
}
return etag;
}
@Override
@ -437,18 +443,13 @@ public class JortageBlobStore extends ForwardingBlobStore {
private static final String NO_DIR_MSG = "Directories are an illusion";
private static final String NO_BULK_MSG = "Bulk operations are not implemented by Jortage for safety and speed";
private static final String NO_PRIVATE_MSG = "Jortage does not support private objects";
@Override
public void setContainerAccess(String container, ContainerAccess containerAccess) {
if (containerAccess != ContainerAccess.PUBLIC_READ)
throw new UnsupportedOperationException(NO_PRIVATE_MSG);
}
@Override
public void setBlobAccess(String container, String name, BlobAccess access) {
if (access != BlobAccess.PUBLIC_READ)
throw new UnsupportedOperationException(NO_PRIVATE_MSG);
}
@Override

15
src/main/java/com/jortage/poolmgr/MastodonHackHandler.java
Wyświetl plik

@ -6,9 +6,9 @@ import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import jakarta.servlet.ServletException;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import org.eclipse.jetty.server.Handler;
import org.eclipse.jetty.server.Request;
@ -42,10 +42,13 @@ public class MastodonHackHandler extends HandlerWrapper {
response.sendError(202);
} catch (IOException e) {
}
}, 4000, TimeUnit.MILLISECONDS);
}, 2000, TimeUnit.MILLISECONDS);
}
try {
super.handle(target, baseRequest, request, response);
} finally {
if (shortCircuit != null) shortCircuit.cancel(false);
}
super.handle(target, baseRequest, request, response);
if (shortCircuit != null) shortCircuit.cancel(false);
}
}

6
src/main/java/com/jortage/poolmgr/OuterHandler.java
Wyświetl plik

@ -2,9 +2,9 @@ package com.jortage.poolmgr;
import java.io.IOException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import jakarta.servlet.ServletException;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import org.eclipse.jetty.server.Handler;
import org.eclipse.jetty.server.Request;

5
src/main/java/com/jortage/poolmgr/Poolmgr.java
Wyświetl plik

@ -58,7 +58,6 @@ public class Poolmgr {
public static final Table<String, String, Object> provisionalMaps = HashBasedTable.create();
@SuppressWarnings("restriction")
public static void main(String[] args) throws Exception {
try {
Stopwatch initSw = Stopwatch.createStarted();
@ -71,6 +70,10 @@ public class Poolmgr {
.endpoint(URI.create("http://localhost:23278"))
.jettyMaxThreads(24)
.v4MaxNonChunkedRequestSize(128L*1024L*1024L)
// S3Proxy will throw if it sees an X-Amz header it doesn't recognize
// Misskey, starting in some recent version (as of July 2023) now sends an X-Amz-User-Agent header
// So without this, Misskey instances can't upload files. Cool thanks
.ignoreUnknownHeaders(true)
.build();
// excuse me, this is mine now

6
src/main/java/com/jortage/poolmgr/RedirHandler.java
Wyświetl plik

@ -2,9 +2,9 @@ package com.jortage.poolmgr;
import java.io.IOException;
import java.util.List;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import jakarta.servlet.ServletException;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.handler.AbstractHandler;
import org.jclouds.blobstore.BlobStore;

6
src/main/java/com/jortage/poolmgr/RivetHandler.java
Wyświetl plik

@ -24,9 +24,9 @@ import kotlin.Pair;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import jakarta.servlet.ServletException;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import org.eclipse.jetty.server.handler.AbstractHandler;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.BlobAccess;