kopia lustrzana https://github.com/jortage/poolmgr
Refactor, implement Rivet, make s3proxy a submodule to avoid shading
rodzic
a76c869a26
commit
168b51b2e8
|
@ -11,3 +11,4 @@ hs_err_pid*.log
|
||||||
.idea/
|
.idea/
|
||||||
data.mv
|
data.mv
|
||||||
/backups
|
/backups
|
||||||
|
/config.jkson
|
||||||
|
|
|
@ -0,0 +1,3 @@
|
||||||
|
[submodule "s3proxy"]
|
||||||
|
path = s3proxy
|
||||||
|
url = https://github.com/gaul/s3proxy.git
|
33
build.gradle
33
build.gradle
|
@ -7,12 +7,43 @@ repositories {
|
||||||
mavenCentral()
|
mavenCentral()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
sourceSets {
|
||||||
|
main {
|
||||||
|
java {
|
||||||
|
srcDirs = ['src/main/java', 'src/s3proxy/java']
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
dependencies {
|
dependencies {
|
||||||
implementation 'blue.endless:jankson:1.1.2'
|
implementation 'blue.endless:jankson:1.1.2'
|
||||||
implementation 'org.mariadb.jdbc:mariadb-java-client:2.4.4'
|
implementation 'org.mariadb.jdbc:mariadb-java-client:2.4.4'
|
||||||
implementation 'org.gaul:s3proxy:1.6.1'
|
implementation 'com.squareup.okhttp3:okhttp:4.7.2'
|
||||||
|
implementation 'com.squareup.okhttp3:okhttp-brotli:4.7.2'
|
||||||
|
|
||||||
|
implementation 'org.apache.jclouds:jclouds-blobstore:2.2.1'
|
||||||
|
implementation 'org.apache.jclouds.provider:aws-s3:2.2.1'
|
||||||
|
implementation 'org.apache.jclouds.api:filesystem:2.2.1'
|
||||||
|
implementation 'org.apache.jclouds.driver:jclouds-slf4j:2.2.1'
|
||||||
|
|
||||||
|
implementation 'org.eclipse.jetty:jetty-server:9.4.24.v20191120'
|
||||||
|
|
||||||
|
implementation 'org.slf4j:slf4j-api:1.7.9'
|
||||||
|
implementation 'org.slf4j:slf4j-nop:1.7.9'
|
||||||
|
|
||||||
|
implementation 'com.google.code.findbugs:jsr305:3.0.2'
|
||||||
|
implementation 'com.google.code.findbugs:findbugs-annotations:3.0.1'
|
||||||
|
|
||||||
|
implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.11.0'
|
||||||
|
implementation 'com.fasterxml.woodstox:woodstox-core:6.2.1'
|
||||||
|
|
||||||
|
implementation 'commons-fileupload:commons-fileupload:1.4'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// I am *not* pulling in five different dependencies for a couple classes we don't use
|
||||||
|
file('s3proxy/src/main/java/org/gaul/s3proxy/junit/S3ProxyRule.java').delete();
|
||||||
|
file('s3proxy/src/main/java/org/gaul/s3proxy/Main.java').delete();
|
||||||
|
|
||||||
project.configurations.implementation.setCanBeResolved(true)
|
project.configurations.implementation.setCanBeResolved(true)
|
||||||
|
|
||||||
task capsule(type: FatCapsule) {
|
task capsule(type: FatCapsule) {
|
||||||
|
|
|
@ -0,0 +1 @@
|
||||||
|
Subproject commit e638111d05b5e7193d2ae8dd1d0e7a63110d878b
|
|
@ -0,0 +1,12 @@
|
||||||
|
package com.jortage.proxy;
|
||||||
|
|
||||||
|
import java.io.Closeable;
|
||||||
|
import com.google.common.io.ByteSink;
|
||||||
|
import com.google.common.io.ByteSource;
|
||||||
|
|
||||||
|
public interface ByteSinkSource extends Closeable {
|
||||||
|
ByteSink getSink();
|
||||||
|
ByteSource getSource();
|
||||||
|
@Override
|
||||||
|
void close();
|
||||||
|
}
|
|
@ -0,0 +1,36 @@
|
||||||
|
package com.jortage.proxy;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
|
||||||
|
import com.google.common.io.ByteSink;
|
||||||
|
import com.google.common.io.ByteSource;
|
||||||
|
import com.google.common.io.Files;
|
||||||
|
|
||||||
|
public class FileByteSinkSource implements ByteSinkSource {
|
||||||
|
|
||||||
|
private final File file;
|
||||||
|
private final boolean deleteOnClose;
|
||||||
|
|
||||||
|
public FileByteSinkSource(File file, boolean deleteOnClose) {
|
||||||
|
this.file = file;
|
||||||
|
this.deleteOnClose = deleteOnClose;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ByteSink getSink() {
|
||||||
|
return Files.asByteSink(file);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ByteSource getSource() {
|
||||||
|
return Files.asByteSource(file);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() {
|
||||||
|
if (deleteOnClose) {
|
||||||
|
file.delete();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -149,22 +149,6 @@ public class JortageBlobStore extends ForwardingBlobStore {
|
||||||
return BlobAccess.PUBLIC_READ;
|
return BlobAccess.PUBLIC_READ;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
|
||||||
public PageSet<? extends StorageMetadata> list() {
|
|
||||||
throw new UnsupportedOperationException();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public PageSet<? extends StorageMetadata> list(String container) {
|
|
||||||
throw new UnsupportedOperationException();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public PageSet<? extends StorageMetadata> list(String container,
|
|
||||||
ListContainerOptions options) {
|
|
||||||
throw new UnsupportedOperationException();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public ContainerAccess getContainerAccess(String container) {
|
public ContainerAccess getContainerAccess(String container) {
|
||||||
checkContainer(container);
|
checkContainer(container);
|
||||||
|
@ -317,23 +301,25 @@ public class JortageBlobStore extends ForwardingBlobStore {
|
||||||
HashCode hash = hos.hash();
|
HashCode hash = hos.hash();
|
||||||
String hashStr = hash.toString();
|
String hashStr = hash.toString();
|
||||||
String path = JortageProxy.hashToPath(hashStr);
|
String path = JortageProxy.hashToPath(hashStr);
|
||||||
// don't fall afoul of request rate limits
|
// we're about to do a bunch of stuff at once
|
||||||
Thread.sleep(500);
|
// sleep so we don't fall afoul of request rate limits
|
||||||
|
// (causes intermittent 429s on at least DigitalOcean)
|
||||||
|
Thread.sleep(250);
|
||||||
BlobMetadata meta = delegate().blobMetadata(mpu.containerName(), mpu.blobName());
|
BlobMetadata meta = delegate().blobMetadata(mpu.containerName(), mpu.blobName());
|
||||||
if (!delegate().blobExists(bucket, path)) {
|
if (!delegate().blobExists(bucket, path)) {
|
||||||
Thread.sleep(500);
|
Thread.sleep(250);
|
||||||
etag = delegate().copyBlob(mpu.containerName(), mpu.blobName(), bucket, path, CopyOptions.builder().contentMetadata(meta.getContentMetadata()).build());
|
etag = delegate().copyBlob(mpu.containerName(), mpu.blobName(), bucket, path, CopyOptions.builder().contentMetadata(meta.getContentMetadata()).build());
|
||||||
Thread.sleep(500);
|
Thread.sleep(250);
|
||||||
delegate().setBlobAccess(bucket, path, BlobAccess.PUBLIC_READ);
|
delegate().setBlobAccess(bucket, path, BlobAccess.PUBLIC_READ);
|
||||||
Queries.putPendingBackup(dataSource, hash);
|
Queries.putPendingBackup(dataSource, hash);
|
||||||
} else {
|
} else {
|
||||||
Thread.sleep(500);
|
Thread.sleep(250);
|
||||||
etag = delegate().blobMetadata(bucket, path).getETag();
|
etag = delegate().blobMetadata(bucket, path).getETag();
|
||||||
}
|
}
|
||||||
Queries.putMap(dataSource, identity, Preconditions.checkNotNull(meta.getUserMetadata().get("jortage-originalname")), hash);
|
Queries.putMap(dataSource, identity, Preconditions.checkNotNull(meta.getUserMetadata().get("jortage-originalname")), hash);
|
||||||
Queries.putFilesize(dataSource, hash, counter.getCount());
|
Queries.putFilesize(dataSource, hash, counter.getCount());
|
||||||
Queries.removeMultipart(dataSource, mpu.blobName());
|
Queries.removeMultipart(dataSource, mpu.blobName());
|
||||||
Thread.sleep(500);
|
Thread.sleep(250);
|
||||||
delegate().removeBlob(mpu.containerName(), mpu.blobName());
|
delegate().removeBlob(mpu.containerName(), mpu.blobName());
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
throw new UncheckedIOException(e);
|
throw new UncheckedIOException(e);
|
||||||
|
@ -387,11 +373,14 @@ public class JortageBlobStore extends ForwardingBlobStore {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
HashCode hc = Queries.getMap(dataSource, identity, name);
|
HashCode hc = Queries.getMap(dataSource, identity, name);
|
||||||
if (Queries.deleteMap(dataSource, identity, name)) {
|
if (Queries.removeMap(dataSource, identity, name)) {
|
||||||
int rc = Queries.getReferenceCount(dataSource, hc);
|
int rc = Queries.getMapCount(dataSource, hc);
|
||||||
if (rc == 0) {
|
if (rc == 0) {
|
||||||
String hashString = hc.toString();
|
String hashString = hc.toString();
|
||||||
delegate().removeBlob(bucket, JortageProxy.hashToPath(hashString));
|
String path = JortageProxy.hashToPath(hashString);
|
||||||
|
delegate().removeBlob(bucket, path);
|
||||||
|
Queries.removeFilesize(dataSource, hc);
|
||||||
|
Queries.removePendingBackup(dataSource, hc);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -426,46 +415,65 @@ public class JortageBlobStore extends ForwardingBlobStore {
|
||||||
return identity.equals(container);
|
return identity.equals(container);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static final String NO_DIR_MSG = "Directories are an illusion";
|
||||||
|
private static final String NO_BULK_MSG = "Bulk operations are not implemented by Jortage for safety and speed";
|
||||||
|
private static final String NO_PRIVATE_MSG = "Jortage does not support private objects";
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void setContainerAccess(String container, ContainerAccess
|
public void setContainerAccess(String container, ContainerAccess containerAccess) {
|
||||||
containerAccess) {
|
if (containerAccess != ContainerAccess.PUBLIC_READ)
|
||||||
throw new UnsupportedOperationException("Read-only BlobStore");
|
throw new UnsupportedOperationException(NO_PRIVATE_MSG);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setBlobAccess(String container, String name, BlobAccess access) {
|
||||||
|
if (access != BlobAccess.PUBLIC_READ)
|
||||||
|
throw new UnsupportedOperationException(NO_PRIVATE_MSG);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void clearContainer(String container) {
|
public void clearContainer(String container) {
|
||||||
throw new UnsupportedOperationException("Read-only BlobStore");
|
throw new UnsupportedOperationException(NO_BULK_MSG);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void clearContainer(String container, ListContainerOptions options) {
|
public void clearContainer(String container, ListContainerOptions options) {
|
||||||
throw new UnsupportedOperationException("Read-only BlobStore");
|
throw new UnsupportedOperationException(NO_BULK_MSG);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void deleteContainer(String container) {
|
public void deleteContainer(String container) {
|
||||||
throw new UnsupportedOperationException("Read-only BlobStore");
|
throw new UnsupportedOperationException(NO_BULK_MSG);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean deleteContainerIfEmpty(String container) {
|
public boolean deleteContainerIfEmpty(String container) {
|
||||||
throw new UnsupportedOperationException("Read-only BlobStore");
|
throw new UnsupportedOperationException(NO_BULK_MSG);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public PageSet<? extends StorageMetadata> list() {
|
||||||
|
throw new UnsupportedOperationException(NO_BULK_MSG);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public PageSet<? extends StorageMetadata> list(String container) {
|
||||||
|
throw new UnsupportedOperationException(NO_BULK_MSG);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public PageSet<? extends StorageMetadata> list(String container, ListContainerOptions options) {
|
||||||
|
throw new UnsupportedOperationException(NO_BULK_MSG);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void createDirectory(String container, String directory) {
|
public void createDirectory(String container, String directory) {
|
||||||
throw new UnsupportedOperationException("Read-only BlobStore");
|
throw new UnsupportedOperationException(NO_DIR_MSG);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void deleteDirectory(String container, String directory) {
|
public void deleteDirectory(String container, String directory) {
|
||||||
throw new UnsupportedOperationException("Read-only BlobStore");
|
throw new UnsupportedOperationException(NO_DIR_MSG);
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void setBlobAccess(String container, String name,
|
|
||||||
BlobAccess access) {
|
|
||||||
throw new UnsupportedOperationException("Read-only BlobStore");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
|
@ -1,29 +1,21 @@
|
||||||
package com.jortage.proxy;
|
package com.jortage.proxy;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.IOException;
|
import java.lang.reflect.Field;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.sql.Connection;
|
import java.sql.Connection;
|
||||||
import java.sql.PreparedStatement;
|
import java.sql.PreparedStatement;
|
||||||
import java.sql.ResultSet;
|
import java.sql.ResultSet;
|
||||||
import java.sql.SQLException;
|
import java.sql.SQLException;
|
||||||
import java.sql.Statement;
|
import java.sql.Statement;
|
||||||
import java.util.List;
|
|
||||||
import java.util.Properties;
|
import java.util.Properties;
|
||||||
import java.util.Map.Entry;
|
|
||||||
import javax.servlet.ServletException;
|
|
||||||
import javax.servlet.http.HttpServletRequest;
|
|
||||||
import javax.servlet.http.HttpServletResponse;
|
|
||||||
|
|
||||||
import sun.misc.Signal;
|
import sun.misc.Signal;
|
||||||
|
|
||||||
import org.eclipse.jetty.server.Request;
|
|
||||||
import org.eclipse.jetty.server.Server;
|
import org.eclipse.jetty.server.Server;
|
||||||
import org.eclipse.jetty.server.ServerConnector;
|
import org.eclipse.jetty.server.ServerConnector;
|
||||||
import org.eclipse.jetty.server.handler.AbstractHandler;
|
|
||||||
import org.eclipse.jetty.util.thread.QueuedThreadPool;
|
import org.eclipse.jetty.util.thread.QueuedThreadPool;
|
||||||
import org.gaul.s3proxy.AuthenticationType;
|
import org.gaul.s3proxy.AuthenticationType;
|
||||||
import org.gaul.s3proxy.BlobStoreLocator;
|
|
||||||
import org.gaul.s3proxy.S3Proxy;
|
import org.gaul.s3proxy.S3Proxy;
|
||||||
import org.jclouds.ContextBuilder;
|
import org.jclouds.ContextBuilder;
|
||||||
import org.jclouds.blobstore.BlobStore;
|
import org.jclouds.blobstore.BlobStore;
|
||||||
|
@ -35,13 +27,11 @@ import org.jclouds.filesystem.reference.FilesystemConstants;
|
||||||
import org.jclouds.logging.slf4j.config.SLF4JLoggingModule;
|
import org.jclouds.logging.slf4j.config.SLF4JLoggingModule;
|
||||||
import org.mariadb.jdbc.MariaDbPoolDataSource;
|
import org.mariadb.jdbc.MariaDbPoolDataSource;
|
||||||
|
|
||||||
import com.google.common.base.Splitter;
|
|
||||||
import com.google.common.base.Stopwatch;
|
import com.google.common.base.Stopwatch;
|
||||||
import com.google.common.collect.ImmutableList;
|
import com.google.common.collect.ImmutableList;
|
||||||
import com.google.common.collect.Maps;
|
import com.google.common.collect.Maps;
|
||||||
import com.google.common.escape.Escaper;
|
import com.google.common.escape.Escaper;
|
||||||
import com.google.common.hash.HashCode;
|
import com.google.common.hash.HashCode;
|
||||||
import com.google.common.io.ByteStreams;
|
|
||||||
import com.google.common.net.UrlEscapers;
|
import com.google.common.net.UrlEscapers;
|
||||||
|
|
||||||
import blue.endless.jankson.Jankson;
|
import blue.endless.jankson.Jankson;
|
||||||
|
@ -50,23 +40,25 @@ import blue.endless.jankson.JsonPrimitive;
|
||||||
|
|
||||||
public class JortageProxy {
|
public class JortageProxy {
|
||||||
|
|
||||||
private static final Splitter SPLITTER = Splitter.on('/').limit(2).omitEmptyStrings();
|
|
||||||
|
|
||||||
private static final File configFile = new File("config.jkson");
|
private static final File configFile = new File("config.jkson");
|
||||||
private static JsonObject config;
|
public static JsonObject config;
|
||||||
private static long configFileLastLoaded;
|
public static long configFileLastLoaded;
|
||||||
private static BlobStore backingBlobStore;
|
public static BlobStore backingBlobStore;
|
||||||
private static BlobStore backingBackupBlobStore;
|
public static BlobStore backingBackupBlobStore;
|
||||||
private static String bucket;
|
public static String bucket;
|
||||||
private static String backupBucket;
|
public static String backupBucket;
|
||||||
private static String publicHost;
|
public static String publicHost;
|
||||||
private static MariaDbPoolDataSource dataSource;
|
public static MariaDbPoolDataSource dataSource;
|
||||||
private static boolean backingUp = false;
|
private static boolean backingUp = false;
|
||||||
|
|
||||||
@SuppressWarnings("restriction")
|
@SuppressWarnings("restriction")
|
||||||
public static void main(String[] args) throws Exception {
|
public static void main(String[] args) throws Exception {
|
||||||
|
try {
|
||||||
|
Stopwatch initSw = Stopwatch.createStarted();
|
||||||
reloadConfig();
|
reloadConfig();
|
||||||
|
|
||||||
|
System.err.print("Starting S3 server... ");
|
||||||
|
System.err.flush();
|
||||||
S3Proxy s3Proxy = S3Proxy.builder()
|
S3Proxy s3Proxy = S3Proxy.builder()
|
||||||
.awsAuthentication(AuthenticationType.AWS_V2_OR_V4, "DUMMY", "DUMMY")
|
.awsAuthentication(AuthenticationType.AWS_V2_OR_V4, "DUMMY", "DUMMY")
|
||||||
.endpoint(URI.create("http://localhost:23278"))
|
.endpoint(URI.create("http://localhost:23278"))
|
||||||
|
@ -74,6 +66,14 @@ public class JortageProxy {
|
||||||
.v4MaxNonChunkedRequestSize(128L*1024L*1024L)
|
.v4MaxNonChunkedRequestSize(128L*1024L*1024L)
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
|
// excuse me, this is mine now
|
||||||
|
Field serverField = S3Proxy.class.getDeclaredField("server");
|
||||||
|
serverField.setAccessible(true);
|
||||||
|
Server s3ProxyServer = (Server) serverField.get(s3Proxy);
|
||||||
|
s3ProxyServer.setHandler(new OuterHandler(s3ProxyServer.getHandler()));
|
||||||
|
QueuedThreadPool pool = (QueuedThreadPool)s3ProxyServer.getThreadPool();
|
||||||
|
pool.setName("Jetty-Common");
|
||||||
|
|
||||||
Properties dumpsProps = new Properties();
|
Properties dumpsProps = new Properties();
|
||||||
dumpsProps.setProperty(FilesystemConstants.PROPERTY_BASEDIR, "dumps");
|
dumpsProps.setProperty(FilesystemConstants.PROPERTY_BASEDIR, "dumps");
|
||||||
BlobStore dumpsStore = ContextBuilder.newBuilder("filesystem")
|
BlobStore dumpsStore = ContextBuilder.newBuilder("filesystem")
|
||||||
|
@ -81,78 +81,44 @@ public class JortageProxy {
|
||||||
.build(BlobStoreContext.class)
|
.build(BlobStoreContext.class)
|
||||||
.getBlobStore();
|
.getBlobStore();
|
||||||
|
|
||||||
s3Proxy.setBlobStoreLocator(new BlobStoreLocator() {
|
s3Proxy.setBlobStoreLocator((identity, container, blob) -> {
|
||||||
|
|
||||||
@Override
|
|
||||||
public Entry<String, BlobStore> locateBlobStore(String identity, String container, String blob) {
|
|
||||||
reloadConfigIfChanged();
|
reloadConfigIfChanged();
|
||||||
if (config.containsKey("users") && config.getObject("users").containsKey(identity)) {
|
if (config.containsKey("users") && config.getObject("users").containsKey(identity)) {
|
||||||
return Maps.immutableEntry(((JsonPrimitive)config.getObject("users").get(identity)).asString(), new JortageBlobStore(backingBlobStore, dumpsStore, bucket, identity, dataSource));
|
return Maps.immutableEntry(((JsonPrimitive)config.getObject("users").get(identity)).asString(),
|
||||||
|
new JortageBlobStore(backingBlobStore, dumpsStore, bucket, identity, dataSource));
|
||||||
} else {
|
} else {
|
||||||
throw new RuntimeException("Access denied");
|
throw new RuntimeException("Access denied");
|
||||||
}
|
}
|
||||||
}
|
|
||||||
});
|
});
|
||||||
|
|
||||||
s3Proxy.start();
|
s3Proxy.start();
|
||||||
System.err.println("S3 listening on localhost:23278");
|
System.err.println("ready on http://localhost:23278");
|
||||||
|
|
||||||
QueuedThreadPool pool = new QueuedThreadPool(24);
|
System.err.print("Starting redirector server... ");
|
||||||
pool.setName("Redir-Jetty");
|
System.err.flush();
|
||||||
Server redir = new Server(pool);
|
Server redir = new Server(pool);
|
||||||
ServerConnector conn = new ServerConnector(redir);
|
ServerConnector redirConn = new ServerConnector(redir);
|
||||||
conn.setHost("localhost");
|
redirConn.setHost("localhost");
|
||||||
conn.setPort(23279);
|
redirConn.setPort(23279);
|
||||||
redir.addConnector(conn);
|
redir.addConnector(redirConn);
|
||||||
redir.setHandler(new AbstractHandler() {
|
redir.setHandler(new OuterHandler(new RedirHandler(dumpsStore)));
|
||||||
|
|
||||||
@Override
|
|
||||||
public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
|
|
||||||
baseRequest.setHandled(true);
|
|
||||||
if ("/".equals(target) || "/index.html".equals(target) || "".equals(target)) {
|
|
||||||
response.setHeader("Location", "https://jortage.com");
|
|
||||||
response.setStatus(301);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
List<String> split = SPLITTER.splitToList(target);
|
|
||||||
if (split.size() != 2) {
|
|
||||||
response.sendError(400);
|
|
||||||
return;
|
|
||||||
} else {
|
|
||||||
String identity = split.get(0);
|
|
||||||
String name = split.get(1);
|
|
||||||
if (name.startsWith("backups/dumps") || name.startsWith("/backups/dumps")) {
|
|
||||||
Blob b = dumpsStore.getBlob(identity, name);
|
|
||||||
if (b != null) {
|
|
||||||
response.setHeader("Cache-Control", "private, no-cache");
|
|
||||||
response.setHeader("Content-Type", b.getMetadata().getContentMetadata().getContentType());
|
|
||||||
if (b.getMetadata().getContentMetadata().getContentLength() != null) {
|
|
||||||
response.setHeader("Content-Length", b.getMetadata().getContentMetadata().getContentLength().toString());
|
|
||||||
}
|
|
||||||
response.setStatus(200);
|
|
||||||
ByteStreams.copy(b.getPayload().openStream(), response.getOutputStream());
|
|
||||||
} else {
|
|
||||||
response.sendError(404);
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
String hash = Queries.getMap(dataSource, identity, name).toString();
|
|
||||||
BlobAccess ba = backingBlobStore.getBlobAccess(bucket, hashToPath(hash));
|
|
||||||
if (ba != BlobAccess.PUBLIC_READ) {
|
|
||||||
backingBlobStore.setBlobAccess(bucket, hashToPath(hash), BlobAccess.PUBLIC_READ);
|
|
||||||
}
|
|
||||||
response.setHeader("Cache-Control", "public");
|
|
||||||
response.setHeader("Location", publicHost+"/"+hashToPath(hash));
|
|
||||||
response.setStatus(301);
|
|
||||||
} catch (IllegalArgumentException e) {
|
|
||||||
response.sendError(404);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
redir.start();
|
redir.start();
|
||||||
System.err.println("Redirector listening on localhost:23279");
|
System.err.println("ready on http://localhost:23279");
|
||||||
|
|
||||||
|
System.err.print("Starting Rivet server... ");
|
||||||
|
System.err.flush();
|
||||||
|
Server rivet = new Server(pool);
|
||||||
|
ServerConnector rivetConn = new ServerConnector(rivet);
|
||||||
|
rivetConn.setHost("localhost");
|
||||||
|
rivetConn.setPort(23280);
|
||||||
|
rivet.addConnector(rivetConn);
|
||||||
|
rivet.setHandler(new OuterHandler(new RivetHandler()));
|
||||||
|
rivet.start();
|
||||||
|
System.err.println("ready on http://localhost:23280");
|
||||||
|
|
||||||
|
System.err.print("Registering SIGALRM handler for backups... ");
|
||||||
|
System.err.flush();
|
||||||
|
try {
|
||||||
Signal.handle(new Signal("ALRM"), (sig) -> {
|
Signal.handle(new Signal("ALRM"), (sig) -> {
|
||||||
reloadConfigIfChanged();
|
reloadConfigIfChanged();
|
||||||
if (backingUp) {
|
if (backingUp) {
|
||||||
|
@ -203,9 +169,18 @@ public class JortageProxy {
|
||||||
}
|
}
|
||||||
}, "Backup thread").start();
|
}, "Backup thread").start();
|
||||||
});
|
});
|
||||||
|
System.err.println("done");
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.err.println("failed");
|
||||||
|
}
|
||||||
|
System.err.println("This proxy has Super Denim Powers. (Done in "+initSw+")");
|
||||||
|
} catch (Throwable t) {
|
||||||
|
System.err.println(" failed");
|
||||||
|
t.printStackTrace();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void reloadConfigIfChanged() {
|
public static void reloadConfigIfChanged() {
|
||||||
if (System.currentTimeMillis()-configFileLastLoaded > 500 && configFile.lastModified() > configFileLastLoaded) reloadConfig();
|
if (System.currentTimeMillis()-configFileLastLoaded > 500 && configFile.lastModified() > configFileLastLoaded) reloadConfig();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -214,17 +189,28 @@ public class JortageProxy {
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void reloadConfig() {
|
private static void reloadConfig() {
|
||||||
|
boolean reloading = config != null;
|
||||||
try {
|
try {
|
||||||
config = Jankson.builder().build().load(configFile);
|
String prelude = "\r"+(reloading ? "Reloading" : "Loading")+" config: ";
|
||||||
configFileLastLoaded = System.currentTimeMillis();
|
System.err.print(prelude+"Parsing...");
|
||||||
bucket = ((JsonPrimitive)config.getObject("backend").get("bucket")).asString();
|
System.err.flush();
|
||||||
publicHost = ((JsonPrimitive)config.getObject("backend").get("publicHost")).asString();
|
JsonObject configTmp = Jankson.builder().build().load(configFile);
|
||||||
backingBlobStore = createBlobStore("backend");
|
long configFileLastLoadedTmp = System.currentTimeMillis();
|
||||||
if (config.containsKey("backupBackend")) {
|
String bucketTmp = ((JsonPrimitive)configTmp.getObject("backend").get("bucket")).asString();
|
||||||
backupBucket = ((JsonPrimitive)config.getObject("backupBackend").get("bucket")).asString();
|
String publicHostTmp = ((JsonPrimitive)configTmp.getObject("backend").get("publicHost")).asString();
|
||||||
backingBackupBlobStore = createBlobStore("backupBackend");
|
System.err.print(prelude+"Constructing blob stores...");
|
||||||
|
System.err.flush();
|
||||||
|
BlobStore backingBlobStoreTmp = createBlobStore(configTmp.getObject("backend"));
|
||||||
|
String backupBucketTmp;
|
||||||
|
BlobStore backingBackupBlobStoreTmp;
|
||||||
|
if (configTmp.containsKey("backupBackend")) {
|
||||||
|
backupBucketTmp = ((JsonPrimitive)configTmp.getObject("backupBackend").get("bucket")).asString();
|
||||||
|
backingBackupBlobStoreTmp = createBlobStore(configTmp.getObject("backupBackend"));
|
||||||
|
} else {
|
||||||
|
backupBucketTmp = null;
|
||||||
|
backingBackupBlobStoreTmp = null;
|
||||||
}
|
}
|
||||||
JsonObject sql = config.getObject("mysql");
|
JsonObject sql = configTmp.getObject("mysql");
|
||||||
String sqlHost = ((JsonPrimitive)sql.get("host")).asString();
|
String sqlHost = ((JsonPrimitive)sql.get("host")).asString();
|
||||||
int sqlPort = ((Number)((JsonPrimitive)sql.get("port")).getValue()).intValue();
|
int sqlPort = ((Number)((JsonPrimitive)sql.get("port")).getValue()).intValue();
|
||||||
String sqlDb = ((JsonPrimitive)sql.get("database")).asString();
|
String sqlDb = ((JsonPrimitive)sql.get("database")).asString();
|
||||||
|
@ -232,11 +218,12 @@ public class JortageProxy {
|
||||||
String sqlPass = ((JsonPrimitive)sql.get("pass")).asString();
|
String sqlPass = ((JsonPrimitive)sql.get("pass")).asString();
|
||||||
Escaper pesc = UrlEscapers.urlPathSegmentEscaper();
|
Escaper pesc = UrlEscapers.urlPathSegmentEscaper();
|
||||||
Escaper esc = UrlEscapers.urlFormParameterEscaper();
|
Escaper esc = UrlEscapers.urlFormParameterEscaper();
|
||||||
if (dataSource != null) {
|
System.err.print(prelude+"Connecting to MariaDB... ");
|
||||||
dataSource.close();
|
System.err.flush();
|
||||||
}
|
MariaDbPoolDataSource dataSourceTmp =
|
||||||
dataSource = new MariaDbPoolDataSource("jdbc:mariadb://"+pesc.escape(sqlHost)+":"+sqlPort+"/"+pesc.escape(sqlDb)+"?user="+esc.escape(sqlUser)+"&password="+esc.escape(sqlPass)+"&autoReconnect=true");
|
new MariaDbPoolDataSource("jdbc:mariadb://"+pesc.escape(sqlHost)+":"+sqlPort+"/"+pesc.escape(sqlDb)
|
||||||
try (Connection c = dataSource.getConnection()) {
|
+"?user="+esc.escape(sqlUser)+"&password="+esc.escape(sqlPass)+"&autoReconnect=true");
|
||||||
|
try (Connection c = dataSourceTmp.getConnection()) {
|
||||||
execOneshot(c, "CREATE TABLE IF NOT EXISTS `name_map` (\n" +
|
execOneshot(c, "CREATE TABLE IF NOT EXISTS `name_map` (\n" +
|
||||||
" `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,\n" +
|
" `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,\n" +
|
||||||
" `identity` VARCHAR(255) NOT NULL,\n" +
|
" `identity` VARCHAR(255) NOT NULL,\n" +
|
||||||
|
@ -265,15 +252,32 @@ public class JortageProxy {
|
||||||
" PRIMARY KEY (`hash`)\n" +
|
" PRIMARY KEY (`hash`)\n" +
|
||||||
") ROW_FORMAT=COMPRESSED;");
|
") ROW_FORMAT=COMPRESSED;");
|
||||||
}
|
}
|
||||||
System.err.println("Config file reloaded.");
|
System.err.println("\r"+(reloading ? "Reloading" : "Loading")+" config... done");
|
||||||
|
MariaDbPoolDataSource oldDataSource = dataSource;
|
||||||
|
config = configTmp;
|
||||||
|
configFileLastLoaded = configFileLastLoadedTmp;
|
||||||
|
bucket = bucketTmp;
|
||||||
|
publicHost = publicHostTmp;
|
||||||
|
backingBlobStore = backingBlobStoreTmp;
|
||||||
|
backupBucket = backupBucketTmp;
|
||||||
|
backingBackupBlobStore = backingBackupBlobStoreTmp;
|
||||||
|
dataSource = dataSourceTmp;
|
||||||
|
if (oldDataSource != null) {
|
||||||
|
oldDataSource.close();
|
||||||
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
|
System.err.println(" failed");
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
|
if (reloading) {
|
||||||
System.err.println("Failed to reload config. Behavior unchanged.");
|
System.err.println("Failed to reload config. Behavior unchanged.");
|
||||||
|
} else {
|
||||||
|
System.err.println("Failed to load config. Exit");
|
||||||
|
System.exit(2);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static BlobStore createBlobStore(String string) {
|
private static BlobStore createBlobStore(JsonObject obj) {
|
||||||
JsonObject obj = config.getObject(string);
|
|
||||||
return ContextBuilder.newBuilder("s3")
|
return ContextBuilder.newBuilder("s3")
|
||||||
.credentials(((JsonPrimitive)obj.get("accessKeyId")).asString(), ((JsonPrimitive)obj.get("secretAccessKey")).asString())
|
.credentials(((JsonPrimitive)obj.get("accessKeyId")).asString(), ((JsonPrimitive)obj.get("secretAccessKey")).asString())
|
||||||
.modules(ImmutableList.of(new SLF4JLoggingModule()))
|
.modules(ImmutableList.of(new SLF4JLoggingModule()))
|
||||||
|
|
|
@ -0,0 +1,62 @@
|
||||||
|
package com.jortage.proxy;
|
||||||
|
|
||||||
|
import java.io.ByteArrayInputStream;
|
||||||
|
import java.io.ByteArrayOutputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.io.OutputStream;
|
||||||
|
|
||||||
|
import com.google.common.io.ByteSink;
|
||||||
|
import com.google.common.io.ByteSource;
|
||||||
|
|
||||||
|
public class MemoryByteSinkSource implements ByteSinkSource {
|
||||||
|
|
||||||
|
private final ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||||
|
|
||||||
|
public MemoryByteSinkSource() {}
|
||||||
|
public MemoryByteSinkSource(byte[] bys) {
|
||||||
|
this(bys, 0, bys.length);
|
||||||
|
}
|
||||||
|
public MemoryByteSinkSource(byte[] bys, int ofs, int len) {
|
||||||
|
baos.write(bys, ofs, len);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ByteSink getSink() {
|
||||||
|
return new ByteSink() {
|
||||||
|
@Override
|
||||||
|
public OutputStream openStream() throws IOException {
|
||||||
|
baos.reset();
|
||||||
|
return baos;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ByteSource getSource() {
|
||||||
|
return new ByteSource() {
|
||||||
|
@Override
|
||||||
|
public InputStream openStream() throws IOException {
|
||||||
|
return new ByteArrayInputStream(baos.toByteArray());
|
||||||
|
}
|
||||||
|
@Override
|
||||||
|
public InputStream openBufferedStream() throws IOException {
|
||||||
|
return openStream();
|
||||||
|
}
|
||||||
|
@Override
|
||||||
|
public byte[] read() throws IOException {
|
||||||
|
return baos.toByteArray();
|
||||||
|
}
|
||||||
|
@Override
|
||||||
|
public long size() throws IOException {
|
||||||
|
return baos.size();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() {
|
||||||
|
baos.reset();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,35 @@
|
||||||
|
package com.jortage.proxy;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import javax.servlet.ServletException;
|
||||||
|
import javax.servlet.http.HttpServletRequest;
|
||||||
|
import javax.servlet.http.HttpServletResponse;
|
||||||
|
|
||||||
|
import org.eclipse.jetty.server.Handler;
|
||||||
|
import org.eclipse.jetty.server.Request;
|
||||||
|
import org.eclipse.jetty.server.handler.HandlerWrapper;
|
||||||
|
import org.eclipse.jetty.util.Jetty;
|
||||||
|
|
||||||
|
public class OuterHandler extends HandlerWrapper {
|
||||||
|
|
||||||
|
public OuterHandler(Handler delegate) {
|
||||||
|
setHandler(delegate);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void handle(String target, Request baseRequest, HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException {
|
||||||
|
res.setHeader("Server", "jortage-proxy");
|
||||||
|
res.setHeader("Powered-By", "Jetty/"+Jetty.VERSION);
|
||||||
|
res.setHeader("Clacks-Overhead", "GNU Natalie Nguyen, Shiina Mota");
|
||||||
|
res.setHeader("Jeans-Teleshorted", Integer.toString((int)(Math.random()*200000)+70));
|
||||||
|
if (target.isEmpty() || target.equals("/") || target.equals("/index.html")) {
|
||||||
|
baseRequest.setHandled(true);
|
||||||
|
res.setHeader("Location", "https://jortage.com");
|
||||||
|
res.setStatus(301);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
super.handle(target, baseRequest, req, res);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -71,7 +71,7 @@ public class Queries {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static boolean deleteMap(DataSource dataSource, String identity, String name) {
|
public static boolean removeMap(DataSource dataSource, String identity, String name) {
|
||||||
name = toSFN(name);
|
name = toSFN(name);
|
||||||
try (Connection c = dataSource.getConnection()) {
|
try (Connection c = dataSource.getConnection()) {
|
||||||
try (PreparedStatement ps = c.prepareStatement("DELETE FROM `name_map` WHERE `identity` = ? AND `name` = ?;")) {
|
try (PreparedStatement ps = c.prepareStatement("DELETE FROM `name_map` WHERE `identity` = ? AND `name` = ?;")) {
|
||||||
|
@ -84,7 +84,7 @@ public class Queries {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static int getReferenceCount(DataSource dataSource, HashCode hash) {
|
public static int getMapCount(DataSource dataSource, HashCode hash) {
|
||||||
try (Connection c = dataSource.getConnection()) {
|
try (Connection c = dataSource.getConnection()) {
|
||||||
try (PreparedStatement ps = c.prepareStatement("SELECT COUNT(1) AS count FROM `name_map` WHERE `hash` = ?;")) {
|
try (PreparedStatement ps = c.prepareStatement("SELECT COUNT(1) AS count FROM `name_map` WHERE `hash` = ?;")) {
|
||||||
ps.setBytes(1, hash.asBytes());
|
ps.setBytes(1, hash.asBytes());
|
||||||
|
@ -113,6 +113,17 @@ public class Queries {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static void removeFilesize(DataSource dataSource, HashCode hash) {
|
||||||
|
try (Connection c = dataSource.getConnection()) {
|
||||||
|
try (PreparedStatement ps = c.prepareStatement("DELETE FROM `filesizes` WHERE `hash` = ?;")) {
|
||||||
|
ps.setBytes(1, hash.asBytes());
|
||||||
|
ps.executeUpdate();
|
||||||
|
}
|
||||||
|
} catch (SQLException e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public static void putPendingBackup(DataSource dataSource, HashCode hash) {
|
public static void putPendingBackup(DataSource dataSource, HashCode hash) {
|
||||||
try (Connection c = dataSource.getConnection()) {
|
try (Connection c = dataSource.getConnection()) {
|
||||||
try (PreparedStatement ps = c.prepareStatement("INSERT IGNORE INTO `pending_backup` (`hash`) VALUES (?);")) {
|
try (PreparedStatement ps = c.prepareStatement("INSERT IGNORE INTO `pending_backup` (`hash`) VALUES (?);")) {
|
||||||
|
@ -124,6 +135,17 @@ public class Queries {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static void removePendingBackup(DataSource dataSource, HashCode hash) {
|
||||||
|
try (Connection c = dataSource.getConnection()) {
|
||||||
|
try (PreparedStatement ps = c.prepareStatement("DELETE FROM `pending_backup` WHERE `hash` = ?;")) {
|
||||||
|
ps.setBytes(1, hash.asBytes());
|
||||||
|
ps.executeUpdate();
|
||||||
|
}
|
||||||
|
} catch (SQLException e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public static void putMultipart(DataSource dataSource, String identity, String name, String tempfile) {
|
public static void putMultipart(DataSource dataSource, String identity, String name, String tempfile) {
|
||||||
name = toSFN(name);
|
name = toSFN(name);
|
||||||
try (Connection c = dataSource.getConnection()) {
|
try (Connection c = dataSource.getConnection()) {
|
||||||
|
|
|
@ -0,0 +1,68 @@
|
||||||
|
package com.jortage.proxy;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import javax.servlet.ServletException;
|
||||||
|
import javax.servlet.http.HttpServletRequest;
|
||||||
|
import javax.servlet.http.HttpServletResponse;
|
||||||
|
import org.eclipse.jetty.server.Request;
|
||||||
|
import org.eclipse.jetty.server.handler.AbstractHandler;
|
||||||
|
import org.jclouds.blobstore.BlobStore;
|
||||||
|
import org.jclouds.blobstore.domain.Blob;
|
||||||
|
import org.jclouds.blobstore.domain.BlobAccess;
|
||||||
|
import com.google.common.base.Splitter;
|
||||||
|
import com.google.common.io.ByteStreams;
|
||||||
|
|
||||||
|
public final class RedirHandler extends AbstractHandler {
|
||||||
|
private static final Splitter REDIR_SPLITTER = Splitter.on('/').limit(2).omitEmptyStrings();
|
||||||
|
|
||||||
|
private final BlobStore dumpsStore;
|
||||||
|
|
||||||
|
public RedirHandler(BlobStore dumpsStore) {
|
||||||
|
this.dumpsStore = dumpsStore;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
|
||||||
|
baseRequest.setHandled(true);
|
||||||
|
List<String> split = REDIR_SPLITTER.splitToList(target);
|
||||||
|
if (split.size() != 2) {
|
||||||
|
response.sendError(400);
|
||||||
|
return;
|
||||||
|
} else {
|
||||||
|
String identity = split.get(0);
|
||||||
|
String name = split.get(1);
|
||||||
|
if (name.startsWith("backups/dumps") || name.startsWith("/backups/dumps")) {
|
||||||
|
Blob b = dumpsStore.getBlob(identity, name);
|
||||||
|
if (b != null) {
|
||||||
|
response.setHeader("Cache-Control", "private, no-cache");
|
||||||
|
response.setHeader("Content-Type", b.getMetadata().getContentMetadata().getContentType());
|
||||||
|
if (b.getMetadata().getContentMetadata().getContentLength() != null) {
|
||||||
|
response.setHeader("Content-Length", b.getMetadata().getContentMetadata().getContentLength().toString());
|
||||||
|
}
|
||||||
|
response.setStatus(200);
|
||||||
|
ByteStreams.copy(b.getPayload().openStream(), response.getOutputStream());
|
||||||
|
} else {
|
||||||
|
response.sendError(404);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
JortageProxy.reloadConfigIfChanged();
|
||||||
|
try {
|
||||||
|
String hash = Queries.getMap(JortageProxy.dataSource, identity, name).toString();
|
||||||
|
BlobAccess ba = JortageProxy.backingBlobStore.getBlobAccess(JortageProxy.bucket, JortageProxy.hashToPath(hash));
|
||||||
|
if (ba != BlobAccess.PUBLIC_READ) {
|
||||||
|
JortageProxy.backingBlobStore.setBlobAccess(JortageProxy.bucket, JortageProxy.hashToPath(hash), BlobAccess.PUBLIC_READ);
|
||||||
|
}
|
||||||
|
response.setHeader("Cache-Control", "public");
|
||||||
|
response.setHeader("Location", JortageProxy.publicHost+"/"+JortageProxy.hashToPath(hash));
|
||||||
|
response.setStatus(301);
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
response.sendError(404);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,507 @@
|
||||||
|
package com.jortage.proxy;
|
||||||
|
|
||||||
|
import static com.google.common.base.Verify.verify;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.OutputStream;
|
||||||
|
import java.net.InetAddress;
|
||||||
|
import java.net.UnknownHostException;
|
||||||
|
import java.security.MessageDigest;
|
||||||
|
import java.time.Instant;
|
||||||
|
import java.time.format.DateTimeFormatter;
|
||||||
|
import java.time.format.DateTimeParseException;
|
||||||
|
import java.time.temporal.ChronoUnit;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Locale;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.ExecutionException;
|
||||||
|
import java.util.concurrent.ThreadLocalRandom;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
import kotlin.Pair;
|
||||||
|
|
||||||
|
import javax.crypto.Mac;
|
||||||
|
import javax.crypto.spec.SecretKeySpec;
|
||||||
|
import javax.servlet.ServletException;
|
||||||
|
import javax.servlet.http.HttpServletRequest;
|
||||||
|
import javax.servlet.http.HttpServletResponse;
|
||||||
|
import org.eclipse.jetty.server.handler.AbstractHandler;
|
||||||
|
import org.jclouds.blobstore.domain.Blob;
|
||||||
|
import org.jclouds.blobstore.domain.BlobAccess;
|
||||||
|
import org.jclouds.blobstore.options.PutOptions;
|
||||||
|
|
||||||
|
import com.google.gson.Gson;
|
||||||
|
import com.google.gson.JsonObject;
|
||||||
|
import com.google.gson.JsonSyntaxException;
|
||||||
|
|
||||||
|
import com.google.common.base.CharMatcher;
|
||||||
|
import com.google.common.base.Charsets;
|
||||||
|
import com.google.common.base.Splitter;
|
||||||
|
import com.google.common.cache.CacheBuilder;
|
||||||
|
import com.google.common.cache.CacheLoader;
|
||||||
|
import com.google.common.cache.LoadingCache;
|
||||||
|
import com.google.common.collect.ImmutableSet;
|
||||||
|
import com.google.common.collect.Maps;
|
||||||
|
import com.google.common.hash.HashCode;
|
||||||
|
import com.google.common.hash.Hashing;
|
||||||
|
import com.google.common.hash.HashingOutputStream;
|
||||||
|
import com.google.common.io.BaseEncoding;
|
||||||
|
import com.google.common.io.ByteStreams;
|
||||||
|
import com.google.common.util.concurrent.UncheckedExecutionException;
|
||||||
|
|
||||||
|
import okhttp3.HttpUrl;
|
||||||
|
import okhttp3.Interceptor;
|
||||||
|
import okhttp3.Interceptor.Chain;
|
||||||
|
import okhttp3.OkHttpClient;
|
||||||
|
import okhttp3.Request;
|
||||||
|
import okhttp3.Response;
|
||||||
|
import okhttp3.brotli.BrotliInterceptor;
|
||||||
|
|
||||||
|
public final class RivetHandler extends AbstractHandler {
|
||||||
|
private static final Splitter RIVET_AUTH_SPLITTER = Splitter.on(':').limit(3);
|
||||||
|
private static final CharMatcher HEX_MATCHER = CharMatcher.anyOf("0123456789abcdef");
|
||||||
|
|
||||||
|
private static final String UA = "Jortage Rivet (+https://jortage.com/rivet.html)";
|
||||||
|
|
||||||
|
private enum Temperature {
|
||||||
|
FREEZING, COLD, WARM, HOT, SCALDING;
|
||||||
|
}
|
||||||
|
|
||||||
|
private enum RetrieveResult {
|
||||||
|
/**
|
||||||
|
* The file was downloaded and added to the pool. Worst case.
|
||||||
|
*/
|
||||||
|
ADDED,
|
||||||
|
/**
|
||||||
|
* The file was downloaded, and after hashing was found to be present in the pool already;
|
||||||
|
* the data was thrown away.
|
||||||
|
*/
|
||||||
|
PRESENT,
|
||||||
|
/**
|
||||||
|
* The file was requested, and a blob redirect was found, so it short-circuited and avoided
|
||||||
|
* a download.
|
||||||
|
*/
|
||||||
|
FOUND,
|
||||||
|
/**
|
||||||
|
* Someone else requested the exact same url within the past 10 minutes, so no requests
|
||||||
|
* were made at all. Best case.
|
||||||
|
*/
|
||||||
|
CACHED,
|
||||||
|
}
|
||||||
|
|
||||||
|
private final Gson gson;
|
||||||
|
// synchronize on a mutex when loading URLs to avoid download races that would waste bandwidth
|
||||||
|
private final Object retrieveMutex = new Object();
|
||||||
|
private final Map<String, Pair<RetrieveResult, Temperature>> results = Maps.newHashMap();
|
||||||
|
private final LoadingCache<String, HashCode> urlCache = CacheBuilder.newBuilder()
|
||||||
|
.concurrencyLevel(1)
|
||||||
|
.expireAfterWrite(10, TimeUnit.MINUTES)
|
||||||
|
.removalListener((n) -> {
|
||||||
|
synchronized (retrieveMutex) {
|
||||||
|
results.remove(n.getKey());
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.build(new CacheLoader<String, HashCode>() {
|
||||||
|
@Override
|
||||||
|
public HashCode load(String url) throws Exception {
|
||||||
|
ByteSinkSource bss = null;
|
||||||
|
HttpUrl parsedUrl = HttpUrl.Companion.parse(url);
|
||||||
|
checkIllegalUrl(null, parsedUrl);
|
||||||
|
HashCode shortCircuit = checkShortCircuit(url, parsedUrl, Temperature.HOT);
|
||||||
|
if (shortCircuit != null) return shortCircuit;
|
||||||
|
try (Response headRes = client.newCall(new Request.Builder()
|
||||||
|
.addHeader("User-Agent", UA)
|
||||||
|
.url(parsedUrl)
|
||||||
|
.head()
|
||||||
|
.build()).execute()) {
|
||||||
|
if (headRes.isSuccessful()) {
|
||||||
|
shortCircuit = checkShortCircuit(url, headRes.request().url(), Temperature.WARM);
|
||||||
|
if (shortCircuit != null) return shortCircuit;
|
||||||
|
shortCircuit = checkShortCircuit(url, headRes.networkResponse().request().url(), Temperature.WARM);
|
||||||
|
if (shortCircuit != null) return shortCircuit;
|
||||||
|
try (Response getRes = client.newCall(new Request.Builder()
|
||||||
|
.addHeader("User-Agent", UA)
|
||||||
|
.url(headRes.request().url())
|
||||||
|
.get()
|
||||||
|
.build()).execute()) {
|
||||||
|
if (getRes.isSuccessful()) {
|
||||||
|
long len = getRes.body().contentLength();
|
||||||
|
if (len == -1 || len > 8192) {
|
||||||
|
bss = new FileByteSinkSource(File.createTempFile("jortage-proxy-", ".dat"), true);
|
||||||
|
} else {
|
||||||
|
bss = new MemoryByteSinkSource();
|
||||||
|
}
|
||||||
|
OutputStream sinkOut = bss.getSink().openStream();
|
||||||
|
HashingOutputStream hos = new HashingOutputStream(Hashing.sha512(), sinkOut);
|
||||||
|
ByteStreams.copy(getRes.body().byteStream(), hos);
|
||||||
|
hos.close();
|
||||||
|
HashCode hash = hos.hash();
|
||||||
|
String hashStr = hash.toString();
|
||||||
|
String path = JortageProxy.hashToPath(hashStr);
|
||||||
|
if (JortageProxy.backingBlobStore.blobExists(JortageProxy.bucket, path)) {
|
||||||
|
results.put(url, new Pair<>(RetrieveResult.PRESENT, Temperature.COLD));
|
||||||
|
} else {
|
||||||
|
Blob blob = JortageProxy.backingBlobStore.blobBuilder(path)
|
||||||
|
.payload(bss.getSource())
|
||||||
|
.contentLength(bss.getSource().size())
|
||||||
|
.contentType(getRes.body().contentType().toString())
|
||||||
|
.build();
|
||||||
|
long size = bss.getSource().size();
|
||||||
|
JortageProxy.backingBlobStore.putBlob(JortageProxy.bucket, blob,
|
||||||
|
new PutOptions().setBlobAccess(BlobAccess.PUBLIC_READ).multipart(size > 8192));
|
||||||
|
Queries.putPendingBackup(JortageProxy.dataSource, hash);
|
||||||
|
Queries.putFilesize(JortageProxy.dataSource, hash, size);
|
||||||
|
results.put(url, new Pair<>(RetrieveResult.ADDED, Temperature.FREEZING));
|
||||||
|
}
|
||||||
|
return hash;
|
||||||
|
} else {
|
||||||
|
throw new IOException("Unsuccessful response code to GET: "+getRes.code());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
throw new IOException("Unsuccessful response code to HEAD: "+headRes.code());
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
if (bss != null) bss.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private HashCode checkShortCircuit(String originalUrl, HttpUrl url, Temperature temp) {
|
||||||
|
String publicHost = JortageProxy.config.getObject("backend").get(String.class, "publicHost").replaceFirst("^https?://", "");
|
||||||
|
String fullHost = url.host();
|
||||||
|
if (url.port() != (url.scheme().equals("https") ? 443 : 80)) {
|
||||||
|
fullHost = fullHost+":"+url.port();
|
||||||
|
}
|
||||||
|
if (fullHost.equals(publicHost)) {
|
||||||
|
List<String> segments = url.pathSegments();
|
||||||
|
if (segments.size() == 4 && segments.get(0).equals("blobs")) {
|
||||||
|
String prelude = segments.get(1)+segments.get(2);
|
||||||
|
String hashStr = segments.get(3);
|
||||||
|
if (hashStr.startsWith(prelude) && HEX_MATCHER.matchesAllOf(hashStr)) {
|
||||||
|
if (JortageProxy.backingBlobStore.blobExists(JortageProxy.bucket, JortageProxy.hashToPath(hashStr))) {
|
||||||
|
HashCode hash = HashCode.fromString(hashStr);
|
||||||
|
results.put(originalUrl, new Pair<>(RetrieveResult.FOUND, temp));
|
||||||
|
return hash;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
private OkHttpClient client;
|
||||||
|
|
||||||
|
public RivetHandler() {
|
||||||
|
this.gson = new Gson();
|
||||||
|
Interceptor urlChecker = (chain) -> {
|
||||||
|
Request req = chain.request();
|
||||||
|
checkIllegalUrl(chain, req.url());
|
||||||
|
Response resp = chain.proceed(req);
|
||||||
|
if (resp.isRedirect() && resp.header("Location") != null) {
|
||||||
|
String location = resp.header("Location");
|
||||||
|
HttpUrl url = HttpUrl.Companion.parse(location);
|
||||||
|
checkIllegalUrl(chain, url);
|
||||||
|
}
|
||||||
|
return resp;
|
||||||
|
};
|
||||||
|
this.client = new OkHttpClient.Builder()
|
||||||
|
.addInterceptor(BrotliInterceptor.INSTANCE)
|
||||||
|
.addInterceptor(urlChecker)
|
||||||
|
.addNetworkInterceptor(urlChecker)
|
||||||
|
.connectTimeout(8, TimeUnit.SECONDS)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private void checkIllegalUrl(Chain chain, HttpUrl url) throws UnknownHostException, IOException {
|
||||||
|
if (url.port() <= 0 || url.port() > 65535 || illegalPorts.contains(url.port())) {
|
||||||
|
if (chain != null) chain.call().cancel();
|
||||||
|
throw new IOException("Illegal host: Illegal port "+url.port());
|
||||||
|
}
|
||||||
|
String host = url.host();
|
||||||
|
for (InetAddress inet : client.dns().lookup(host)) {
|
||||||
|
if (inet.isAnyLocalAddress() || inet.isLinkLocalAddress() || inet.isLoopbackAddress()
|
||||||
|
|| inet.isMulticastAddress() || inet.isSiteLocalAddress()) {
|
||||||
|
if (chain != null) chain.call().cancel();
|
||||||
|
throw new IOException("Illegal host: Illegal address "+inet.getHostAddress()+" ("+host+")");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void handle(String target, org.eclipse.jetty.server.Request baseRequest, HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException {
|
||||||
|
baseRequest.setHandled(true);
|
||||||
|
if ("/retrieve".equals(target)) {
|
||||||
|
try {
|
||||||
|
if ("OPTIONS".equals(req.getMethod())) {
|
||||||
|
res.setHeader("Allow", "POST");
|
||||||
|
res.setHeader("Accept", "application/json;charset=utf-8");
|
||||||
|
res.setStatus(204);
|
||||||
|
res.getOutputStream().close();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!"POST".equals(req.getMethod())) {
|
||||||
|
res.setHeader("Allow", "POST");
|
||||||
|
jsonError(res, 405, "/retrieve only accepts POST");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
String authHeader = req.getHeader("Rivet-Auth");
|
||||||
|
if (authHeader == null) {
|
||||||
|
jsonError(res, 401, "Rivet-Auth header missing");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Iterator<String> iter = RIVET_AUTH_SPLITTER.split(authHeader).iterator();
|
||||||
|
if (!iter.hasNext()) {
|
||||||
|
jsonError(res, 401, "Rivet-Auth header invalid (Not enough fields)");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
String identity = iter.next();
|
||||||
|
if (!iter.hasNext()) {
|
||||||
|
jsonError(res, 401, "Rivet-Auth header invalid (Not enough fields)");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
String macStr = iter.next();
|
||||||
|
if (!iter.hasNext()) {
|
||||||
|
jsonError(res, 401, "Rivet-Auth header invalid (Not enough fields)");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
String dateStr = iter.next();
|
||||||
|
verify(!iter.hasNext());
|
||||||
|
|
||||||
|
Instant date;
|
||||||
|
try {
|
||||||
|
date = Instant.from(DateTimeFormatter.ISO_INSTANT.parse(dateStr));
|
||||||
|
} catch (DateTimeParseException e) {
|
||||||
|
jsonError(res, 401, "Rivet-Auth header invalid (Could not parse date)");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (date.isBefore(Instant.now().minus(5, ChronoUnit.MINUTES))) {
|
||||||
|
jsonError(res, 401, "Rivet-Auth header invalid (Too old)");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
JortageProxy.reloadConfigIfChanged();
|
||||||
|
if (!JortageProxy.config.containsKey("users") || !JortageProxy.config.getObject("users").containsKey(identity)) {
|
||||||
|
jsonError(res, 401, "Rivet-Auth header invalid (Bad access ID)");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (req.getContentLength() == -1) {
|
||||||
|
jsonError(res, 411, "Length required");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (req.getContentLength() > 8192) {
|
||||||
|
jsonError(res, 413, "Payload too large");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
String contentType = req.getHeader("Content-Type");
|
||||||
|
if (contentType == null || !"application/json;charset=utf-8".equals(contentType.replace(" ", "").toLowerCase(Locale.ROOT))) {
|
||||||
|
res.setHeader("Accept", "application/json;charset=utf-8");
|
||||||
|
jsonError(res, 415, "Content-Type must be application/json; charset=utf-8");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
byte[] theirMac = BaseEncoding.base64().decode(macStr);
|
||||||
|
Mac mac = assertSuccess(() -> Mac.getInstance("HmacSHA512"));
|
||||||
|
byte[] payload = ByteStreams.toByteArray(ByteStreams.limit(req.getInputStream(), req.getContentLength()));
|
||||||
|
req.getInputStream().close();
|
||||||
|
String payloadStr = new String(payload, Charsets.UTF_8);
|
||||||
|
|
||||||
|
String key = JortageProxy.config.getObject("users").get(String.class, identity);
|
||||||
|
assertSuccess(() -> mac.init(new SecretKeySpec(key.getBytes(Charsets.UTF_8), "RAW")));
|
||||||
|
mac.update((identity+":"+dateStr+":"+payloadStr).getBytes(Charsets.UTF_8));
|
||||||
|
byte[] ourMac = mac.doFinal();
|
||||||
|
if (!MessageDigest.isEqual(theirMac, ourMac)) {
|
||||||
|
jsonError(res, 401, "Rivet-Auth header invalid (Bad MAC)");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// phew. now that all of *that* is out of the way... what is it they want?
|
||||||
|
JsonObject json;
|
||||||
|
try {
|
||||||
|
json = gson.fromJson(payloadStr, JsonObject.class);
|
||||||
|
} catch (JsonSyntaxException e) {
|
||||||
|
jsonError(res, 400, "Syntax error in payload");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!json.has("sourceUrl")) {
|
||||||
|
jsonError(res, 400, "Must specify sourceUrl");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!json.has("destinationPath")) {
|
||||||
|
jsonError(res, 400, "Must specify destinationPath");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
String sourceUrl = json.get("sourceUrl").getAsString();
|
||||||
|
if (!sourceUrl.startsWith("https://") && !sourceUrl.startsWith("http://")) {
|
||||||
|
jsonError(res, 400, "sourceUrl must be http or https");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
String destinationPath = json.get("destinationPath").getAsString();
|
||||||
|
synchronized (retrieveMutex) {
|
||||||
|
RetrieveResult retRes = null;
|
||||||
|
Temperature temp = null;
|
||||||
|
HashCode hash;
|
||||||
|
try {
|
||||||
|
if (urlCache.getIfPresent(sourceUrl) != null) {
|
||||||
|
retRes = RetrieveResult.CACHED;
|
||||||
|
temp = Temperature.SCALDING;
|
||||||
|
}
|
||||||
|
hash = urlCache.get(sourceUrl);
|
||||||
|
if (retRes == null || temp == null) {
|
||||||
|
Pair<RetrieveResult, Temperature> pair = results.get(sourceUrl);
|
||||||
|
retRes = pair.getFirst();
|
||||||
|
temp = pair.getSecond();
|
||||||
|
}
|
||||||
|
} catch (ExecutionException | UncheckedExecutionException e) {
|
||||||
|
if (e.getMessage() != null) {
|
||||||
|
if (e.getMessage().contains("Illegal host")) {
|
||||||
|
jsonError(res, 400, "Illegal host");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (e.getMessage().contains("Unsuccessful response")) {
|
||||||
|
jsonError(res, 502, "Upstream error "+(e.getMessage().substring(e.getMessage().lastIndexOf(':')+1).trim()));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (e.getMessage().contains("connect timed out")) {
|
||||||
|
jsonError(res, 504, "Upstream timeout");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
jsonExceptionError(res, e, "sourceUrl: "+sourceUrl, "identity: "+identity);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Queries.putMap(JortageProxy.dataSource, identity, destinationPath, hash);
|
||||||
|
res.setStatus(200);
|
||||||
|
res.setHeader("Content-Type", "application/json; charset=utf-8");
|
||||||
|
JsonObject obj = new JsonObject();
|
||||||
|
JsonObject result = new JsonObject();
|
||||||
|
result.addProperty("name", retRes.name());
|
||||||
|
result.addProperty("temperature", temp.name());
|
||||||
|
obj.add("result", result);
|
||||||
|
obj.addProperty("hash", hash.toString());
|
||||||
|
res.getOutputStream().write(obj.toString().getBytes(Charsets.UTF_8));
|
||||||
|
res.getOutputStream().close();
|
||||||
|
}
|
||||||
|
} catch (Throwable t) {
|
||||||
|
jsonExceptionError(res, t);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
res.sendError(404);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void jsonExceptionError(HttpServletResponse res, Throwable t, String... extra) throws IOException {
|
||||||
|
byte[] tokenBys = new byte[8];
|
||||||
|
ThreadLocalRandom.current().nextBytes(tokenBys);
|
||||||
|
String token = BaseEncoding.base16().lowerCase().encode(tokenBys);
|
||||||
|
System.err.println("== BEGIN "+token+" ==");
|
||||||
|
t.printStackTrace();
|
||||||
|
if (extra.length > 0) {
|
||||||
|
System.err.println("Extra information:");
|
||||||
|
for (String s : extra) {
|
||||||
|
System.err.println(s);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
System.err.println("== END "+token+" ==");
|
||||||
|
jsonError(res, 500, "Internal error "+token);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private void jsonError(HttpServletResponse res, int code, String msg) throws IOException {
|
||||||
|
res.setStatus(code);
|
||||||
|
res.setHeader("Content-Type", "application/json; charset=utf-8");
|
||||||
|
JsonObject obj = new JsonObject();
|
||||||
|
obj.addProperty("error", msg);
|
||||||
|
res.getOutputStream().write(obj.toString().getBytes(Charsets.UTF_8));
|
||||||
|
res.getOutputStream().close();
|
||||||
|
}
|
||||||
|
|
||||||
|
private interface ExceptableRunnable { void run() throws Exception; }
|
||||||
|
private interface ExceptableSupplier<T> { T get() throws Exception; }
|
||||||
|
|
||||||
|
private static void assertSuccess(ExceptableRunnable r) {
|
||||||
|
try {
|
||||||
|
r.run();
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new AssertionError(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
private static <T> T assertSuccess(ExceptableSupplier<T> s) {
|
||||||
|
try {
|
||||||
|
return s.get();
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new AssertionError(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// https://chromium.googlesource.com/chromium/chromium/+/master/net/base/net_util.cc#92
|
||||||
|
private static final ImmutableSet<Integer> illegalPorts = ImmutableSet.of(
|
||||||
|
1, // tcpmux
|
||||||
|
7, // echo
|
||||||
|
9, // discard
|
||||||
|
11, // systat
|
||||||
|
13, // daytime
|
||||||
|
15, // netstat
|
||||||
|
17, // qotd
|
||||||
|
19, // chargen
|
||||||
|
20, // ftp data
|
||||||
|
21, // ftp access
|
||||||
|
22, // ssh
|
||||||
|
23, // telnet
|
||||||
|
25, // smtp
|
||||||
|
37, // time
|
||||||
|
42, // name
|
||||||
|
43, // nicname
|
||||||
|
53, // domain
|
||||||
|
77, // priv-rjs
|
||||||
|
79, // finger
|
||||||
|
87, // ttylink
|
||||||
|
95, // supdup
|
||||||
|
101, // hostriame
|
||||||
|
102, // iso-tsap
|
||||||
|
103, // gppitnp
|
||||||
|
104, // acr-nema
|
||||||
|
109, // pop2
|
||||||
|
110, // pop3
|
||||||
|
111, // sunrpc
|
||||||
|
113, // auth
|
||||||
|
115, // sftp
|
||||||
|
117, // uucp-path
|
||||||
|
119, // nntp
|
||||||
|
123, // NTP
|
||||||
|
135, // loc-srv /epmap
|
||||||
|
139, // netbios
|
||||||
|
143, // imap2
|
||||||
|
179, // BGP
|
||||||
|
389, // ldap
|
||||||
|
465, // smtp+ssl
|
||||||
|
512, // print / exec
|
||||||
|
513, // login
|
||||||
|
514, // shell
|
||||||
|
515, // printer
|
||||||
|
526, // tempo
|
||||||
|
530, // courier
|
||||||
|
531, // chat
|
||||||
|
532, // netnews
|
||||||
|
540, // uucp
|
||||||
|
556, // remotefs
|
||||||
|
563, // nntp+ssl
|
||||||
|
587, // stmp?
|
||||||
|
601, // ??
|
||||||
|
636, // ldap+ssl
|
||||||
|
993, // ldap+ssl
|
||||||
|
995, // pop3+ssl
|
||||||
|
2049, // nfs
|
||||||
|
3659, // apple-sasl / PasswordServer
|
||||||
|
4045, // lockd
|
||||||
|
6000, // X11
|
||||||
|
6665, // Alternate IRC [Apple addition]
|
||||||
|
6666, // Alternate IRC [Apple addition]
|
||||||
|
6667, // Standard IRC [Apple addition]
|
||||||
|
6668, // Alternate IRC [Apple addition]
|
||||||
|
6669 // Alternate IRC [Apple addition]
|
||||||
|
);
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,85 @@
|
||||||
|
package com.jortage.proxy;
|
||||||
|
|
||||||
|
import java.time.Instant;
|
||||||
|
import java.time.format.DateTimeFormatter;
|
||||||
|
|
||||||
|
import javax.crypto.Mac;
|
||||||
|
import javax.crypto.spec.SecretKeySpec;
|
||||||
|
|
||||||
|
import com.google.gson.JsonObject;
|
||||||
|
|
||||||
|
import com.google.common.base.Charsets;
|
||||||
|
import com.google.common.io.BaseEncoding;
|
||||||
|
import com.google.common.io.ByteStreams;
|
||||||
|
|
||||||
|
import kotlin.Pair;
|
||||||
|
import okhttp3.MediaType;
|
||||||
|
import okhttp3.OkHttpClient;
|
||||||
|
import okhttp3.Request;
|
||||||
|
import okhttp3.RequestBody;
|
||||||
|
import okhttp3.Response;
|
||||||
|
import okhttp3.brotli.BrotliInterceptor;
|
||||||
|
|
||||||
|
public class RivetTest {
|
||||||
|
|
||||||
|
public static void main(String[] args) throws Exception {
|
||||||
|
OkHttpClient client = new OkHttpClient.Builder()
|
||||||
|
.addInterceptor(BrotliInterceptor.INSTANCE)
|
||||||
|
.build();
|
||||||
|
JsonObject obj = new JsonObject();
|
||||||
|
obj.addProperty("sourceUrl", "http://example.com/nothing.png");
|
||||||
|
obj.addProperty("destinationPath", "test.png");
|
||||||
|
String payload = obj.toString();
|
||||||
|
String accessKey = "test";
|
||||||
|
String secretKey = "test";
|
||||||
|
String date = DateTimeFormatter.ISO_INSTANT.format(Instant.now());
|
||||||
|
Mac mac = assertSuccess(() -> Mac.getInstance("HmacSHA512"));
|
||||||
|
byte[] payloadBytes = payload.getBytes(Charsets.UTF_8);
|
||||||
|
|
||||||
|
assertSuccess(() -> mac.init(new SecretKeySpec(secretKey.getBytes(Charsets.UTF_8), "RAW")));
|
||||||
|
mac.update((accessKey+":"+date+":"+payload).getBytes(Charsets.UTF_8));
|
||||||
|
byte[] macBys = mac.doFinal();
|
||||||
|
String auth = accessKey+":"+BaseEncoding.base64().encode(macBys)+":"+date;
|
||||||
|
try (Response res = client.newCall(new Request.Builder()
|
||||||
|
.url("http://localhost:23280/retrieve")
|
||||||
|
.post(RequestBody.create(payloadBytes, MediaType.parse("application/json; charset=utf-8")))
|
||||||
|
.header("Rivet-Auth", auth)
|
||||||
|
.header("User-Agent", "Jortage Rivet Test")
|
||||||
|
.build()).execute()) {
|
||||||
|
Request req = res.networkResponse().request();
|
||||||
|
System.out.println(req.method()+" "+req.url());
|
||||||
|
for (Pair<? extends String, ? extends String> pair : req.headers()) {
|
||||||
|
System.out.println(pair.getFirst()+": "+pair.getSecond());
|
||||||
|
}
|
||||||
|
System.out.println();
|
||||||
|
System.out.println(payload);
|
||||||
|
System.out.println();
|
||||||
|
System.out.println();
|
||||||
|
System.out.println(res.code()+" "+res.message());
|
||||||
|
for (Pair<? extends String, ? extends String> pair : res.headers()) {
|
||||||
|
System.out.println(pair.getFirst()+": "+pair.getSecond());
|
||||||
|
}
|
||||||
|
System.out.println();
|
||||||
|
ByteStreams.copy(res.body().byteStream(), System.out);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private interface ExceptableRunnable { void run() throws Exception; }
|
||||||
|
private interface ExceptableSupplier<T> { T get() throws Exception; }
|
||||||
|
|
||||||
|
private static void assertSuccess(ExceptableRunnable r) {
|
||||||
|
try {
|
||||||
|
r.run();
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new AssertionError(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
private static <T> T assertSuccess(ExceptableSupplier<T> s) {
|
||||||
|
try {
|
||||||
|
return s.get();
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new AssertionError(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1 @@
|
||||||
|
../../s3proxy/src/main/java/
|
Ładowanie…
Reference in New Issue