kopia lustrzana https://github.com/jortage/poolmgr
Replace built-in trash with HikariCP, enable perf options
Also don't keep around the entire config JSON in memory for no reasontrunk
rodzic
fc79bb51f4
commit
6cc1cc7fae
|
@ -10,7 +10,7 @@ repositories {
|
||||||
|
|
||||||
base {
|
base {
|
||||||
archivesName = 'jortage-poolmgr'
|
archivesName = 'jortage-poolmgr'
|
||||||
version = '1.4.1'
|
version = '1.4.2'
|
||||||
}
|
}
|
||||||
|
|
||||||
compileJava {
|
compileJava {
|
||||||
|
@ -33,10 +33,12 @@ configurations.all {
|
||||||
|
|
||||||
dependencies {
|
dependencies {
|
||||||
implementation 'blue.endless:jankson:1.2.3'
|
implementation 'blue.endless:jankson:1.2.3'
|
||||||
implementation 'org.mariadb.jdbc:mariadb-java-client:3.1.4'
|
|
||||||
implementation 'com.squareup.okhttp3:okhttp:4.11.0'
|
implementation 'com.squareup.okhttp3:okhttp:4.11.0'
|
||||||
implementation 'com.squareup.okhttp3:okhttp-brotli:4.11.0'
|
implementation 'com.squareup.okhttp3:okhttp-brotli:4.11.0'
|
||||||
|
|
||||||
|
implementation 'org.mariadb.jdbc:mariadb-java-client:3.1.4'
|
||||||
|
implementation 'com.zaxxer:HikariCP:5.0.1'
|
||||||
|
|
||||||
implementation 'org.apache.jclouds:jclouds-blobstore:2.5.0'
|
implementation 'org.apache.jclouds:jclouds-blobstore:2.5.0'
|
||||||
implementation 'org.apache.jclouds.provider:aws-s3:2.5.0'
|
implementation 'org.apache.jclouds.provider:aws-s3:2.5.0'
|
||||||
implementation 'org.apache.jclouds.api:filesystem:2.5.0'
|
implementation 'org.apache.jclouds.api:filesystem:2.5.0'
|
||||||
|
|
|
@ -55,7 +55,6 @@ public class JortageBlobStore extends ForwardingBlobStore {
|
||||||
public JortageBlobStore(BlobStore blobStore, BlobStore dumpsStore, String bucket, String identity, DataSource dataSource) {
|
public JortageBlobStore(BlobStore blobStore, BlobStore dumpsStore, String bucket, String identity, DataSource dataSource) {
|
||||||
super(blobStore);
|
super(blobStore);
|
||||||
this.dumpsStore = dumpsStore;
|
this.dumpsStore = dumpsStore;
|
||||||
dumpsStore.createContainerInLocation(null, identity);
|
|
||||||
this.bucket = bucket;
|
this.bucket = bucket;
|
||||||
this.identity = identity;
|
this.identity = identity;
|
||||||
this.dataSource = dataSource;
|
this.dataSource = dataSource;
|
||||||
|
|
|
@ -8,7 +8,11 @@ import java.sql.PreparedStatement;
|
||||||
import java.sql.ResultSet;
|
import java.sql.ResultSet;
|
||||||
import java.sql.SQLException;
|
import java.sql.SQLException;
|
||||||
import java.sql.Statement;
|
import java.sql.Statement;
|
||||||
|
import java.util.Map;
|
||||||
import java.util.Properties;
|
import java.util.Properties;
|
||||||
|
|
||||||
|
import javax.sql.DataSource;
|
||||||
|
|
||||||
import sun.misc.Signal;
|
import sun.misc.Signal;
|
||||||
|
|
||||||
import org.eclipse.jetty.server.Server;
|
import org.eclipse.jetty.server.Server;
|
||||||
|
@ -24,12 +28,13 @@ import org.jclouds.blobstore.domain.BlobAccess;
|
||||||
import org.jclouds.blobstore.options.PutOptions;
|
import org.jclouds.blobstore.options.PutOptions;
|
||||||
import org.jclouds.filesystem.reference.FilesystemConstants;
|
import org.jclouds.filesystem.reference.FilesystemConstants;
|
||||||
import org.jclouds.logging.slf4j.config.SLF4JLoggingModule;
|
import org.jclouds.logging.slf4j.config.SLF4JLoggingModule;
|
||||||
import org.mariadb.jdbc.MariaDbPoolDataSource;
|
import com.zaxxer.hikari.HikariDataSource;
|
||||||
|
|
||||||
import com.google.common.base.MoreObjects;
|
import com.google.common.base.MoreObjects;
|
||||||
import com.google.common.base.Stopwatch;
|
import com.google.common.base.Stopwatch;
|
||||||
import com.google.common.collect.HashBasedTable;
|
import com.google.common.collect.HashBasedTable;
|
||||||
import com.google.common.collect.ImmutableList;
|
import com.google.common.collect.ImmutableList;
|
||||||
|
import com.google.common.collect.ImmutableMap;
|
||||||
import com.google.common.collect.Maps;
|
import com.google.common.collect.Maps;
|
||||||
import com.google.common.collect.Table;
|
import com.google.common.collect.Table;
|
||||||
import com.google.common.escape.Escaper;
|
import com.google.common.escape.Escaper;
|
||||||
|
@ -37,20 +42,19 @@ import com.google.common.hash.HashCode;
|
||||||
import com.google.common.net.UrlEscapers;
|
import com.google.common.net.UrlEscapers;
|
||||||
|
|
||||||
import blue.endless.jankson.Jankson;
|
import blue.endless.jankson.Jankson;
|
||||||
|
import blue.endless.jankson.JsonElement;
|
||||||
import blue.endless.jankson.JsonObject;
|
import blue.endless.jankson.JsonObject;
|
||||||
import blue.endless.jankson.JsonPrimitive;
|
import blue.endless.jankson.JsonPrimitive;
|
||||||
|
|
||||||
public class Poolmgr {
|
public class Poolmgr {
|
||||||
|
|
||||||
private static final File configFile = new File("config.jkson");
|
private static final File configFile = new File("config.jkson");
|
||||||
public static JsonObject config;
|
|
||||||
public static long configFileLastLoaded;
|
public static long configFileLastLoaded;
|
||||||
public static BlobStore backingBlobStore;
|
public static BlobStore backingBlobStore, backingBackupBlobStore, dumpsStore;
|
||||||
public static BlobStore backingBackupBlobStore;
|
public static String bucket, backupBucket;
|
||||||
public static String bucket;
|
|
||||||
public static String backupBucket;
|
|
||||||
public static String publicHost;
|
public static String publicHost;
|
||||||
public static MariaDbPoolDataSource dataSource;
|
public static DataSource dataSource;
|
||||||
|
public static Map<String, String> users;
|
||||||
public static volatile boolean readOnly = false;
|
public static volatile boolean readOnly = false;
|
||||||
private static boolean backingUp = false;
|
private static boolean backingUp = false;
|
||||||
private static boolean rivetEnabled;
|
private static boolean rivetEnabled;
|
||||||
|
@ -60,6 +64,13 @@ public class Poolmgr {
|
||||||
|
|
||||||
public static void main(String[] args) throws Exception {
|
public static void main(String[] args) throws Exception {
|
||||||
try {
|
try {
|
||||||
|
Properties dumpsProps = new Properties();
|
||||||
|
dumpsProps.setProperty(FilesystemConstants.PROPERTY_BASEDIR, "dumps");
|
||||||
|
dumpsStore = ContextBuilder.newBuilder("filesystem")
|
||||||
|
.overrides(dumpsProps)
|
||||||
|
.build(BlobStoreContext.class)
|
||||||
|
.getBlobStore();
|
||||||
|
|
||||||
Stopwatch initSw = Stopwatch.createStarted();
|
Stopwatch initSw = Stopwatch.createStarted();
|
||||||
reloadConfig();
|
reloadConfig();
|
||||||
|
|
||||||
|
@ -84,17 +95,11 @@ public class Poolmgr {
|
||||||
QueuedThreadPool pool = (QueuedThreadPool)s3ProxyServer.getThreadPool();
|
QueuedThreadPool pool = (QueuedThreadPool)s3ProxyServer.getThreadPool();
|
||||||
pool.setName("Jetty-Common");
|
pool.setName("Jetty-Common");
|
||||||
|
|
||||||
Properties dumpsProps = new Properties();
|
|
||||||
dumpsProps.setProperty(FilesystemConstants.PROPERTY_BASEDIR, "dumps");
|
|
||||||
BlobStore dumpsStore = ContextBuilder.newBuilder("filesystem")
|
|
||||||
.overrides(dumpsProps)
|
|
||||||
.build(BlobStoreContext.class)
|
|
||||||
.getBlobStore();
|
|
||||||
|
|
||||||
s3Proxy.setBlobStoreLocator((identity, container, blob) -> {
|
s3Proxy.setBlobStoreLocator((identity, container, blob) -> {
|
||||||
reloadConfigIfChanged();
|
reloadConfigIfChanged();
|
||||||
if (config.containsKey("users") && config.getObject("users").containsKey(identity)) {
|
String secret = users.get(identity);
|
||||||
return Maps.immutableEntry(((JsonPrimitive)config.getObject("users").get(identity)).asString(),
|
if (secret != null) {
|
||||||
|
return Maps.immutableEntry(secret,
|
||||||
new JortageBlobStore(backingBlobStore, dumpsStore, bucket, identity, dataSource));
|
new JortageBlobStore(backingBlobStore, dumpsStore, bucket, identity, dataSource));
|
||||||
} else {
|
} else {
|
||||||
throw new RuntimeException("Access denied");
|
throw new RuntimeException("Access denied");
|
||||||
|
@ -196,7 +201,7 @@ public class Poolmgr {
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void reloadConfigIfChanged() {
|
public static void reloadConfigIfChanged() {
|
||||||
if (System.currentTimeMillis()-configFileLastLoaded > 500 && configFile.lastModified() > configFileLastLoaded) reloadConfig();
|
// if (System.currentTimeMillis()-configFileLastLoaded > 500 && configFile.lastModified() > configFileLastLoaded) reloadConfig();
|
||||||
}
|
}
|
||||||
|
|
||||||
private static String s(int i) {
|
private static String s(int i) {
|
||||||
|
@ -204,7 +209,7 @@ public class Poolmgr {
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void reloadConfig() {
|
private static void reloadConfig() {
|
||||||
boolean reloading = config != null;
|
boolean reloading = dataSource != null;
|
||||||
try {
|
try {
|
||||||
String prelude = "\r"+(reloading ? "Reloading" : "Loading")+" config: ";
|
String prelude = "\r"+(reloading ? "Reloading" : "Loading")+" config: ";
|
||||||
System.err.print(prelude+"Parsing...");
|
System.err.print(prelude+"Parsing...");
|
||||||
|
@ -237,9 +242,20 @@ public class Poolmgr {
|
||||||
Escaper esc = UrlEscapers.urlFormParameterEscaper();
|
Escaper esc = UrlEscapers.urlFormParameterEscaper();
|
||||||
System.err.print(prelude+"Connecting to MariaDB... ");
|
System.err.print(prelude+"Connecting to MariaDB... ");
|
||||||
System.err.flush();
|
System.err.flush();
|
||||||
MariaDbPoolDataSource dataSourceTmp =
|
HikariDataSource dataSourceTmp = new HikariDataSource();
|
||||||
new MariaDbPoolDataSource("jdbc:mariadb://"+pesc.escape(sqlHost)+":"+sqlPort+"/"+pesc.escape(sqlDb)
|
dataSourceTmp.setJdbcUrl("jdbc:mariadb://"+pesc.escape(sqlHost)+":"+sqlPort+"/"+pesc.escape(sqlDb));
|
||||||
+"?user="+esc.escape(sqlUser)+"&password="+esc.escape(sqlPass)+"&autoReconnect=true");
|
dataSourceTmp.setUsername(sqlUser);
|
||||||
|
dataSourceTmp.setPassword(sqlPass);
|
||||||
|
dataSourceTmp.addDataSourceProperty("cachePrepStmts", "true");
|
||||||
|
dataSourceTmp.addDataSourceProperty("prepStmtCacheSize", "100");
|
||||||
|
dataSourceTmp.addDataSourceProperty("prepStmtCacheSqlLimit", "3000");
|
||||||
|
dataSourceTmp.addDataSourceProperty("useServerPrepStmts", "true");
|
||||||
|
dataSourceTmp.addDataSourceProperty("useLocalSessionState", "true");
|
||||||
|
dataSourceTmp.addDataSourceProperty("rewriteBatchedStatements", "true");
|
||||||
|
dataSourceTmp.addDataSourceProperty("cacheResultSetMetadata", "true");
|
||||||
|
dataSourceTmp.addDataSourceProperty("cacheServerConfiguration", "true");
|
||||||
|
dataSourceTmp.addDataSourceProperty("elideSetAutoCommits", "true");
|
||||||
|
dataSourceTmp.addDataSourceProperty("maintainTimeStats", "false");
|
||||||
try (Connection c = dataSourceTmp.getConnection()) {
|
try (Connection c = dataSourceTmp.getConnection()) {
|
||||||
execOneshot(c, "CREATE TABLE IF NOT EXISTS `name_map` (\n" +
|
execOneshot(c, "CREATE TABLE IF NOT EXISTS `name_map` (\n" +
|
||||||
" `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,\n" +
|
" `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,\n" +
|
||||||
|
@ -269,9 +285,14 @@ public class Poolmgr {
|
||||||
" PRIMARY KEY (`hash`)\n" +
|
" PRIMARY KEY (`hash`)\n" +
|
||||||
") ROW_FORMAT=COMPRESSED;");
|
") ROW_FORMAT=COMPRESSED;");
|
||||||
}
|
}
|
||||||
|
ImmutableMap.Builder<String, String> usersTmp = ImmutableMap.builder();
|
||||||
|
for (Map.Entry<String, JsonElement> en : configTmp.getObject("users").entrySet()) {
|
||||||
|
usersTmp.put(en.getKey(), ((JsonPrimitive)en.getValue()).asString());
|
||||||
|
dumpsStore.createContainerInLocation(null, en.getKey());
|
||||||
|
}
|
||||||
|
users = usersTmp.build();
|
||||||
System.err.println("\r"+(reloading ? "Reloading" : "Loading")+" config... done ");
|
System.err.println("\r"+(reloading ? "Reloading" : "Loading")+" config... done ");
|
||||||
MariaDbPoolDataSource oldDataSource = dataSource;
|
HikariDataSource oldDataSource = (HikariDataSource)dataSource;
|
||||||
config = configTmp;
|
|
||||||
configFileLastLoaded = configFileLastLoadedTmp;
|
configFileLastLoaded = configFileLastLoadedTmp;
|
||||||
bucket = bucketTmp;
|
bucket = bucketTmp;
|
||||||
publicHost = publicHostTmp;
|
publicHost = publicHostTmp;
|
||||||
|
|
|
@ -98,7 +98,7 @@ public final class RivetHandler extends AbstractHandler {
|
||||||
private final LoadingCache<String, HashCode> urlCache = CacheBuilder.newBuilder()
|
private final LoadingCache<String, HashCode> urlCache = CacheBuilder.newBuilder()
|
||||||
.concurrencyLevel(1)
|
.concurrencyLevel(1)
|
||||||
.expireAfterWrite(10, TimeUnit.MINUTES)
|
.expireAfterWrite(10, TimeUnit.MINUTES)
|
||||||
.removalListener((n) -> {
|
.<String, HashCode>removalListener((n) -> {
|
||||||
synchronized (retrieveMutex) {
|
synchronized (retrieveMutex) {
|
||||||
results.remove(n.getKey());
|
results.remove(n.getKey());
|
||||||
}
|
}
|
||||||
|
@ -171,7 +171,7 @@ public final class RivetHandler extends AbstractHandler {
|
||||||
}
|
}
|
||||||
|
|
||||||
private HashCode checkShortCircuit(String originalUrl, HttpUrl url, Temperature temp) {
|
private HashCode checkShortCircuit(String originalUrl, HttpUrl url, Temperature temp) {
|
||||||
String publicHost = Poolmgr.config.getObject("backend").get(String.class, "publicHost").replaceFirst("^https?://", "");
|
String publicHost = Poolmgr.publicHost.replaceFirst("^https?://", "");
|
||||||
String fullHost = url.host();
|
String fullHost = url.host();
|
||||||
if (url.port() != (url.scheme().equals("https") ? 443 : 80)) {
|
if (url.port() != (url.scheme().equals("https") ? 443 : 80)) {
|
||||||
fullHost = fullHost+":"+url.port();
|
fullHost = fullHost+":"+url.port();
|
||||||
|
@ -457,7 +457,7 @@ public final class RivetHandler extends AbstractHandler {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!Poolmgr.config.containsKey("users") || !Poolmgr.config.getObject("users").containsKey(identity)) {
|
if (!Poolmgr.users.containsKey(identity)) {
|
||||||
jsonError(res, 401, "Rivet-Auth header invalid (Bad access ID)");
|
jsonError(res, 401, "Rivet-Auth header invalid (Bad access ID)");
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
@ -490,7 +490,7 @@ public final class RivetHandler extends AbstractHandler {
|
||||||
}
|
}
|
||||||
String payloadStr = new String(payload, Charsets.UTF_8);
|
String payloadStr = new String(payload, Charsets.UTF_8);
|
||||||
|
|
||||||
String key = Poolmgr.config.getObject("users").get(String.class, identity);
|
String key = Poolmgr.users.get(identity);
|
||||||
assertSuccess(() -> mac.init(new SecretKeySpec(key.getBytes(Charsets.UTF_8), "RAW")));
|
assertSuccess(() -> mac.init(new SecretKeySpec(key.getBytes(Charsets.UTF_8), "RAW")));
|
||||||
String query;
|
String query;
|
||||||
if (req.getQueryString() == null) {
|
if (req.getQueryString() == null) {
|
||||||
|
|
Ładowanie…
Reference in New Issue