Fix dump permissions, add LFN->SFN conversion, misc

src/main/java/com/jortage/proxy/JortageBlobStore.java

@@ -213,7 +213,7 @@ public class JortageBlobStore extends ForwardingBlobStore {
 	public String putBlob(String container, Blob blob) {
 		checkContainer(container);
 		if (isDump(blob.getMetadata().getName())) {
-			return dumpsStore.putBlob(container, blob);
+			return dumpsStore.putBlob(container, blob, new PutOptions().setBlobAccess(BlobAccess.PUBLIC_READ));
 		}
 		File tempFile = null;
 		try {
@@ -270,7 +270,7 @@ public class JortageBlobStore extends ForwardingBlobStore {
 	public MultipartUpload initiateMultipartUpload(String container, BlobMetadata blobMetadata, PutOptions options) {
 		checkContainer(container);
 		if (isDump(blobMetadata.getName())) {
-			return dumpsStore.initiateMultipartUpload(container, blobMetadata, options);
+			return dumpsStore.initiateMultipartUpload(container, blobMetadata, new PutOptions().setBlobAccess(BlobAccess.PUBLIC_READ));
 		}
 		MutableBlobMetadata mbm = new MutableBlobMetadataImpl(blobMetadata);
 		String tempfile = "multitmp/"+identity+"-"+System.currentTimeMillis()+"-"+System.nanoTime();
@@ -381,12 +381,19 @@ public class JortageBlobStore extends ForwardingBlobStore {
 
 	@Override
 	public void removeBlob(String container, String name) {
+		checkContainer(container);
 		if (isDump(name)) {
-			checkContainer(container);
 			dumpsStore.removeBlob(container, name);
 			return;
 		}
-		throw new UnsupportedOperationException("Read-only BlobStore");
+		HashCode hc = Queries.getMap(dataSource, identity, name);
+		if (Queries.deleteMap(dataSource, identity, name)) {
+			int rc = Queries.getReferenceCount(dataSource, hc);
+			if (rc == 0) {
+				String hashString = hc.toString();
+				delegate().removeBlob(bucket, JortageProxy.hashToPath(hashString));
+			}
+		}
 	}
 
 	@Override

src/main/java/com/jortage/proxy/JortageProxy.java

@@ -29,6 +29,8 @@ import org.jclouds.ContextBuilder;
 import org.jclouds.blobstore.BlobStore;
 import org.jclouds.blobstore.BlobStoreContext;
 import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.blobstore.domain.BlobAccess;
+import org.jclouds.blobstore.options.PutOptions;
 import org.jclouds.filesystem.reference.FilesystemConstants;
 import org.jclouds.logging.slf4j.config.SLF4JLoggingModule;
 import org.mariadb.jdbc.MariaDbPoolDataSource;
@@ -124,6 +126,9 @@ public class JortageProxy {
 						if (b != null) {
 							response.setHeader("Cache-Control", "private, no-cache");
 							response.setHeader("Content-Type", b.getMetadata().getContentMetadata().getContentType());
+							if (b.getMetadata().getContentMetadata().getContentLength() != null) {
+								response.setHeader("Content-Length", b.getMetadata().getContentMetadata().getContentLength().toString());
+							}
 							response.setStatus(200);
 							ByteStreams.copy(b.getPayload().openStream(), response.getOutputStream());
 						} else {
@@ -133,6 +138,10 @@ public class JortageProxy {
 					}
 					try {
 						String hash = Queries.getMap(dataSource, identity, name).toString();
+						BlobAccess ba = backingBlobStore.getBlobAccess(bucket, hashToPath(hash));
+						if (ba != BlobAccess.PUBLIC_READ) {
+							backingBlobStore.setBlobAccess(bucket, hashToPath(hash), BlobAccess.PUBLIC_READ);
+						}
 						response.setHeader("Cache-Control", "public");
 						response.setHeader("Location", publicHost+"/"+hashToPath(hash));
 						response.setStatus(301);
@@ -150,6 +159,10 @@ public class JortageProxy {
 				System.err.println("Ignoring SIGALRM, backup already in progress");
 				return;
 			}
+			if (backupBucket == null) {
+				System.err.println("Ignoring SIGALRM, nowhere to backup to");
+				return;
+			}
 			new Thread(() -> {
 				int count = 0;
 				Stopwatch sw = Stopwatch.createStarted();
@@ -162,7 +175,18 @@ public class JortageProxy {
 									byte[] bys = rs.getBytes("hash");
 									String path = hashToPath(HashCode.fromBytes(bys).toString());
 									Blob src = backingBlobStore.getBlob(bucket, path);
-									backingBackupBlobStore.putBlob(backupBucket, src);
+									if (src == null) {
+										Blob actualSrc = backingBackupBlobStore.getBlob(backupBucket, path);
+										if (actualSrc == null) {
+											System.err.println("Can't find blob "+path+" in source or destination?");
+											continue;
+										}  else {
+											System.err.println("Copying "+path+" from \"backup\" to current - this is a little odd");
+											backingBlobStore.putBlob(bucket, actualSrc, new PutOptions().setBlobAccess(BlobAccess.PUBLIC_READ));
+										}
+									} else {
+										backingBackupBlobStore.putBlob(backupBucket, src, new PutOptions().setBlobAccess(BlobAccess.PUBLIC_READ));
+									}
 									delete.setBytes(1, bys);
 									delete.executeUpdate();
 									count++;
@@ -194,10 +218,12 @@ public class JortageProxy {
 			config = Jankson.builder().build().load(configFile);
 			configFileLastLoaded = System.currentTimeMillis();
 			bucket = ((JsonPrimitive)config.getObject("backend").get("bucket")).asString();
-			backupBucket = ((JsonPrimitive)config.getObject("backupBackend").get("bucket")).asString();
 			publicHost = ((JsonPrimitive)config.getObject("backend").get("publicHost")).asString();
 			backingBlobStore = createBlobStore("backend");
-			backingBackupBlobStore = createBlobStore("backupBackend");
+			if (config.containsKey("backupBackend")) {
+				backupBucket = ((JsonPrimitive)config.getObject("backupBackend").get("bucket")).asString();
+				backingBackupBlobStore = createBlobStore("backupBackend");
+			}
 			JsonObject sql = config.getObject("mysql");
 			String sqlHost = ((JsonPrimitive)sql.get("host")).asString();
 			int sqlPort = ((Number)((JsonPrimitive)sql.get("port")).getValue()).intValue();

src/main/java/com/jortage/proxy/Queries.java

@@ -7,11 +7,38 @@ import java.sql.SQLException;
 
 import javax.sql.DataSource;
 
+import com.google.common.base.Charsets;
 import com.google.common.hash.HashCode;
+import com.google.common.hash.Hashing;
 
 public class Queries {
 
+	/**
+	 * Convert a potential LFN (Long File Name, >255 chars) to a SFN (Short File Name, <=255 chars),
+	 * by truncating the LFN and appending a SHA-256 hash and remainder length to the end.
+	 * <p>
+	 * All names that will be touching the DB must go through here. All the other public methods in
+	 * Queries will handle this for you.
+	 * <p>
+	 * XXX If the SHA-256 hash collides (VERY UNLIKELY!) then the conflicting names will be
+	 * entangled; we already depend on the collision resistance of SHA-2 for other (more
+	 * important) things, so this is not a huge problem... but, UGH. Very Bad Hack.
+	 * <p>
+	 * (Why do this at all? You can't index on a TEXT column in MySQL, and VARCHAR only goes
+	 * up to 255. A VARCHAR that is >255 is secretly a TEXT.)
+	 */
+	public static String toSFN(String lfn) {
+		if (lfn.length() <= 255) return lfn;
+		int remainder = lfn.length()-255;
+		String remainderStr = Integer.toString(remainder);
+		int remainderLength = remainderStr.length();
+		String hash = Hashing.sha256().hashString(lfn, Charsets.UTF_8).toString();
+		String sfn = lfn.substring(0, 255-1-64-1-remainderLength)+"~"+hash+"$"+remainderStr;
+		return sfn;
+	}
+	
 	public static HashCode getMap(DataSource dataSource, String identity, String name) {
+		name = toSFN(name);
 		try (Connection c = dataSource.getConnection()) {
 			try (PreparedStatement ps = c.prepareStatement("SELECT `hash` FROM `name_map` WHERE `identity` = ? AND `name` = ?;")) {
 				ps.setString(1, identity);
@@ -30,6 +57,7 @@ public class Queries {
 	}
 
 	public static void putMap(DataSource dataSource, String identity, String name, HashCode hash) {
+		name = toSFN(name);
 		try (Connection c = dataSource.getConnection()) {
 			try (PreparedStatement ps = c.prepareStatement("INSERT INTO `name_map` (`identity`, `name`, `hash`) VALUES (?, ?, ?) ON DUPLICATE KEY UPDATE `hash` = ?;")) {
 				ps.setString(1, identity);
@@ -42,6 +70,36 @@ public class Queries {
 			throw new RuntimeException(e);
 		}
 	}
+	
+	public static boolean deleteMap(DataSource dataSource, String identity, String name) {
+		name = toSFN(name);
+		try (Connection c = dataSource.getConnection()) {
+			try (PreparedStatement ps = c.prepareStatement("DELETE FROM `name_map` WHERE `identity` = ? AND `name` = ?;")) {
+				ps.setString(1, identity);
+				ps.setString(2, name);
+				return ps.executeUpdate() > 0;
+			}
+		} catch (SQLException e) {
+			throw new RuntimeException(e);
+		}
+	}
+	
+	public static int getReferenceCount(DataSource dataSource, HashCode hash) {
+		try (Connection c = dataSource.getConnection()) {
+			try (PreparedStatement ps = c.prepareStatement("SELECT COUNT(1) AS count FROM `name_map` WHERE `hash` = ?;")) {
+				ps.setBytes(1, hash.asBytes());
+				try (ResultSet rs = ps.executeQuery()) {
+					if (rs.first()) {
+						return rs.getInt("count");
+					} else {
+						return 0;
+					}
+				}
+			}
+		} catch (SQLException e) {
+			throw new RuntimeException(e);
+		}
+	}
 
 	public static void putFilesize(DataSource dataSource, HashCode hash, long size) {
 		try (Connection c = dataSource.getConnection()) {
@@ -67,6 +125,7 @@ public class Queries {
 	}
 
 	public static void putMultipart(DataSource dataSource, String identity, String name, String tempfile) {
+		name = toSFN(name);
 		try (Connection c = dataSource.getConnection()) {
 			try (PreparedStatement ps = c.prepareStatement("INSERT INTO `multipart_uploads` (`identity`, `name`, `tempfile`) VALUES (?, ?, ?) ON DUPLICATE KEY UPDATE `tempfile` = ?;")) {
 				ps.setString(1, identity);
@@ -81,6 +140,7 @@ public class Queries {
 	}
 
 	public static String getMultipart(DataSource dataSource, String identity, String name) {
+		name = toSFN(name);
 		try (Connection c = dataSource.getConnection()) {
 			try (PreparedStatement ps = c.prepareStatement("SELECT `tempfile` FROM `multipart_uploads` WHERE `identity` = ? AND `name` = ?;")) {
 				ps.setString(1, identity);