From a76c869a262660d2ac6a382c178119a3bba7459a Mon Sep 17 00:00:00 2001 From: Una Thompson Date: Fri, 12 Jun 2020 19:03:33 -0700 Subject: [PATCH] Fix dump permissions, add LFN->SFN conversion, misc --- .../com/jortage/proxy/JortageBlobStore.java | 15 +++-- .../java/com/jortage/proxy/JortageProxy.java | 32 +++++++++- src/main/java/com/jortage/proxy/Queries.java | 60 +++++++++++++++++++ 3 files changed, 100 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/jortage/proxy/JortageBlobStore.java b/src/main/java/com/jortage/proxy/JortageBlobStore.java index 5bc8ac6..4e9efd3 100644 --- a/src/main/java/com/jortage/proxy/JortageBlobStore.java +++ b/src/main/java/com/jortage/proxy/JortageBlobStore.java @@ -213,7 +213,7 @@ public class JortageBlobStore extends ForwardingBlobStore { public String putBlob(String container, Blob blob) { checkContainer(container); if (isDump(blob.getMetadata().getName())) { - return dumpsStore.putBlob(container, blob); + return dumpsStore.putBlob(container, blob, new PutOptions().setBlobAccess(BlobAccess.PUBLIC_READ)); } File tempFile = null; try { @@ -270,7 +270,7 @@ public class JortageBlobStore extends ForwardingBlobStore { public MultipartUpload initiateMultipartUpload(String container, BlobMetadata blobMetadata, PutOptions options) { checkContainer(container); if (isDump(blobMetadata.getName())) { - return dumpsStore.initiateMultipartUpload(container, blobMetadata, options); + return dumpsStore.initiateMultipartUpload(container, blobMetadata, new PutOptions().setBlobAccess(BlobAccess.PUBLIC_READ)); } MutableBlobMetadata mbm = new MutableBlobMetadataImpl(blobMetadata); String tempfile = "multitmp/"+identity+"-"+System.currentTimeMillis()+"-"+System.nanoTime(); @@ -381,12 +381,19 @@ public class JortageBlobStore extends ForwardingBlobStore { @Override public void removeBlob(String container, String name) { + checkContainer(container); if (isDump(name)) { - checkContainer(container); dumpsStore.removeBlob(container, name); return; } - throw new UnsupportedOperationException("Read-only BlobStore"); + HashCode hc = Queries.getMap(dataSource, identity, name); + if (Queries.deleteMap(dataSource, identity, name)) { + int rc = Queries.getReferenceCount(dataSource, hc); + if (rc == 0) { + String hashString = hc.toString(); + delegate().removeBlob(bucket, JortageProxy.hashToPath(hashString)); + } + } } @Override diff --git a/src/main/java/com/jortage/proxy/JortageProxy.java b/src/main/java/com/jortage/proxy/JortageProxy.java index ed658b8..54359b3 100644 --- a/src/main/java/com/jortage/proxy/JortageProxy.java +++ b/src/main/java/com/jortage/proxy/JortageProxy.java @@ -29,6 +29,8 @@ import org.jclouds.ContextBuilder; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.domain.BlobAccess; +import org.jclouds.blobstore.options.PutOptions; import org.jclouds.filesystem.reference.FilesystemConstants; import org.jclouds.logging.slf4j.config.SLF4JLoggingModule; import org.mariadb.jdbc.MariaDbPoolDataSource; @@ -124,6 +126,9 @@ public class JortageProxy { if (b != null) { response.setHeader("Cache-Control", "private, no-cache"); response.setHeader("Content-Type", b.getMetadata().getContentMetadata().getContentType()); + if (b.getMetadata().getContentMetadata().getContentLength() != null) { + response.setHeader("Content-Length", b.getMetadata().getContentMetadata().getContentLength().toString()); + } response.setStatus(200); ByteStreams.copy(b.getPayload().openStream(), response.getOutputStream()); } else { @@ -133,6 +138,10 @@ public class JortageProxy { } try { String hash = Queries.getMap(dataSource, identity, name).toString(); + BlobAccess ba = backingBlobStore.getBlobAccess(bucket, hashToPath(hash)); + if (ba != BlobAccess.PUBLIC_READ) { + backingBlobStore.setBlobAccess(bucket, hashToPath(hash), BlobAccess.PUBLIC_READ); + } response.setHeader("Cache-Control", "public"); response.setHeader("Location", publicHost+"/"+hashToPath(hash)); response.setStatus(301); @@ -150,6 +159,10 @@ public class JortageProxy { System.err.println("Ignoring SIGALRM, backup already in progress"); return; } + if (backupBucket == null) { + System.err.println("Ignoring SIGALRM, nowhere to backup to"); + return; + } new Thread(() -> { int count = 0; Stopwatch sw = Stopwatch.createStarted(); @@ -162,7 +175,18 @@ public class JortageProxy { byte[] bys = rs.getBytes("hash"); String path = hashToPath(HashCode.fromBytes(bys).toString()); Blob src = backingBlobStore.getBlob(bucket, path); - backingBackupBlobStore.putBlob(backupBucket, src); + if (src == null) { + Blob actualSrc = backingBackupBlobStore.getBlob(backupBucket, path); + if (actualSrc == null) { + System.err.println("Can't find blob "+path+" in source or destination?"); + continue; + } else { + System.err.println("Copying "+path+" from \"backup\" to current - this is a little odd"); + backingBlobStore.putBlob(bucket, actualSrc, new PutOptions().setBlobAccess(BlobAccess.PUBLIC_READ)); + } + } else { + backingBackupBlobStore.putBlob(backupBucket, src, new PutOptions().setBlobAccess(BlobAccess.PUBLIC_READ)); + } delete.setBytes(1, bys); delete.executeUpdate(); count++; @@ -194,10 +218,12 @@ public class JortageProxy { config = Jankson.builder().build().load(configFile); configFileLastLoaded = System.currentTimeMillis(); bucket = ((JsonPrimitive)config.getObject("backend").get("bucket")).asString(); - backupBucket = ((JsonPrimitive)config.getObject("backupBackend").get("bucket")).asString(); publicHost = ((JsonPrimitive)config.getObject("backend").get("publicHost")).asString(); backingBlobStore = createBlobStore("backend"); - backingBackupBlobStore = createBlobStore("backupBackend"); + if (config.containsKey("backupBackend")) { + backupBucket = ((JsonPrimitive)config.getObject("backupBackend").get("bucket")).asString(); + backingBackupBlobStore = createBlobStore("backupBackend"); + } JsonObject sql = config.getObject("mysql"); String sqlHost = ((JsonPrimitive)sql.get("host")).asString(); int sqlPort = ((Number)((JsonPrimitive)sql.get("port")).getValue()).intValue(); diff --git a/src/main/java/com/jortage/proxy/Queries.java b/src/main/java/com/jortage/proxy/Queries.java index 3e82f0b..43308d4 100644 --- a/src/main/java/com/jortage/proxy/Queries.java +++ b/src/main/java/com/jortage/proxy/Queries.java @@ -7,11 +7,38 @@ import java.sql.SQLException; import javax.sql.DataSource; +import com.google.common.base.Charsets; import com.google.common.hash.HashCode; +import com.google.common.hash.Hashing; public class Queries { + /** + * Convert a potential LFN (Long File Name, >255 chars) to a SFN (Short File Name, <=255 chars), + * by truncating the LFN and appending a SHA-256 hash and remainder length to the end. + *

+ * All names that will be touching the DB must go through here. All the other public methods in + * Queries will handle this for you. + *

+ * XXX If the SHA-256 hash collides (VERY UNLIKELY!) then the conflicting names will be + * entangled; we already depend on the collision resistance of SHA-2 for other (more + * important) things, so this is not a huge problem... but, UGH. Very Bad Hack. + *

+ * (Why do this at all? You can't index on a TEXT column in MySQL, and VARCHAR only goes + * up to 255. A VARCHAR that is >255 is secretly a TEXT.) + */ + public static String toSFN(String lfn) { + if (lfn.length() <= 255) return lfn; + int remainder = lfn.length()-255; + String remainderStr = Integer.toString(remainder); + int remainderLength = remainderStr.length(); + String hash = Hashing.sha256().hashString(lfn, Charsets.UTF_8).toString(); + String sfn = lfn.substring(0, 255-1-64-1-remainderLength)+"~"+hash+"$"+remainderStr; + return sfn; + } + public static HashCode getMap(DataSource dataSource, String identity, String name) { + name = toSFN(name); try (Connection c = dataSource.getConnection()) { try (PreparedStatement ps = c.prepareStatement("SELECT `hash` FROM `name_map` WHERE `identity` = ? AND `name` = ?;")) { ps.setString(1, identity); @@ -30,6 +57,7 @@ public class Queries { } public static void putMap(DataSource dataSource, String identity, String name, HashCode hash) { + name = toSFN(name); try (Connection c = dataSource.getConnection()) { try (PreparedStatement ps = c.prepareStatement("INSERT INTO `name_map` (`identity`, `name`, `hash`) VALUES (?, ?, ?) ON DUPLICATE KEY UPDATE `hash` = ?;")) { ps.setString(1, identity); @@ -42,6 +70,36 @@ public class Queries { throw new RuntimeException(e); } } + + public static boolean deleteMap(DataSource dataSource, String identity, String name) { + name = toSFN(name); + try (Connection c = dataSource.getConnection()) { + try (PreparedStatement ps = c.prepareStatement("DELETE FROM `name_map` WHERE `identity` = ? AND `name` = ?;")) { + ps.setString(1, identity); + ps.setString(2, name); + return ps.executeUpdate() > 0; + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + public static int getReferenceCount(DataSource dataSource, HashCode hash) { + try (Connection c = dataSource.getConnection()) { + try (PreparedStatement ps = c.prepareStatement("SELECT COUNT(1) AS count FROM `name_map` WHERE `hash` = ?;")) { + ps.setBytes(1, hash.asBytes()); + try (ResultSet rs = ps.executeQuery()) { + if (rs.first()) { + return rs.getInt("count"); + } else { + return 0; + } + } + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + } public static void putFilesize(DataSource dataSource, HashCode hash, long size) { try (Connection c = dataSource.getConnection()) { @@ -67,6 +125,7 @@ public class Queries { } public static void putMultipart(DataSource dataSource, String identity, String name, String tempfile) { + name = toSFN(name); try (Connection c = dataSource.getConnection()) { try (PreparedStatement ps = c.prepareStatement("INSERT INTO `multipart_uploads` (`identity`, `name`, `tempfile`) VALUES (?, ?, ?) ON DUPLICATE KEY UPDATE `tempfile` = ?;")) { ps.setString(1, identity); @@ -81,6 +140,7 @@ public class Queries { } public static String getMultipart(DataSource dataSource, String identity, String name) { + name = toSFN(name); try (Connection c = dataSource.getConnection()) { try (PreparedStatement ps = c.prepareStatement("SELECT `tempfile` FROM `multipart_uploads` WHERE `identity` = ? AND `name` = ?;")) { ps.setString(1, identity);