mirror
/
s3proxy
kopia lustrzana https://github.com/gaul/s3proxy
1
0
Forkuj 0
Kod Zgłoszenia Packages Projekty Wydania Wiki Aktywność

Emit XML via StAX 

This addresses issues with object names containing special characters.
References #25.
pull/36/head
Andrew Gaul 2015-01-10 16:31:03 -08:00
rodzic e955b998fb
commit f3221cb136
1 zmienionych plików z 252 dodań i 138 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

390
src/main/java/org/gaul/s3proxy/S3ProxyHandler.java
Wyświetl plik

@ -40,6 +40,9 @@ import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec; import javax.crypto.spec.SecretKeySpec;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import javax.xml.stream.XMLOutputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamWriter;
import com.google.common.base.Optional; import com.google.common.base.Optional;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
@ -82,11 +85,8 @@ import org.slf4j.LoggerFactory;
final class S3ProxyHandler extends AbstractHandler { final class S3ProxyHandler extends AbstractHandler {
private static final Logger logger = LoggerFactory.getLogger( private static final Logger logger = LoggerFactory.getLogger(
S3ProxyHandler.class); S3ProxyHandler.class);
// Note that this excludes a trailing \r\n which the AWS SDK rejects.
private static final String XML_PROLOG =
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>";
private static final String AWS_XMLNS = private static final String AWS_XMLNS =
"xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\""; "http://s3.amazonaws.com/doc/2006-03-01/";
// TODO: support configurable metadata prefix // TODO: support configurable metadata prefix
private static final String USER_METADATA_PREFIX = "x-amz-meta-"; private static final String USER_METADATA_PREFIX = "x-amz-meta-";
// TODO: fake owner // TODO: fake owner
@ -112,6 +112,8 @@ final class S3ProxyHandler extends AbstractHandler {
private final String credential; private final String credential;
private final boolean forceMultiPartUpload; private final boolean forceMultiPartUpload;
private final Optional<String> virtualHost; private final Optional<String> virtualHost;
private final XMLOutputFactory xmlOutputFactory =
XMLOutputFactory.newInstance();
S3ProxyHandler(BlobStore blobStore, String identity, String credential, S3ProxyHandler(BlobStore blobStore, String identity, String credential,
boolean forceMultiPartUpload, Optional<String> virtualHost) { boolean forceMultiPartUpload, Optional<String> virtualHost) {
@ -120,6 +122,8 @@ final class S3ProxyHandler extends AbstractHandler {
this.credential = credential; this.credential = credential;
this.forceMultiPartUpload = forceMultiPartUpload; this.forceMultiPartUpload = forceMultiPartUpload;
this.virtualHost = Preconditions.checkNotNull(virtualHost); this.virtualHost = Preconditions.checkNotNull(virtualHost);
xmlOutputFactory.setProperty("javax.xml.stream.isRepairingNamespaces",
Boolean.FALSE);
} }
@Override @Override
@ -358,70 +362,116 @@ final class S3ProxyHandler extends AbstractHandler {
private void handleContainerOrBlobAcl(HttpServletResponse response, private void handleContainerOrBlobAcl(HttpServletResponse response,
String... containerName) throws IOException { String... containerName) throws IOException {
try (Writer writer = response.getWriter()) { try (Writer writer = response.getWriter()) {
writer.write(XML_PROLOG + XMLStreamWriter xml = xmlOutputFactory.createXMLStreamWriter(
"<AccessControlPolicy " + AWS_XMLNS + ">\r\n" + writer);
" <Owner>\r\n" + xml.writeStartDocument();
" <ID>" + FAKE_OWNER_ID + "</ID>\r\n" + xml.writeStartElement("AccessControlPolicy");
" <DisplayName>" + FAKE_OWNER_DISPLAY_NAME + xml.writeDefaultNamespace(AWS_XMLNS);
"</DisplayName>\r\n" +
" </Owner>\r\n" + xml.writeStartElement("Owner");
" <AccessControlList>\r\n" +
" <Grant>\r\n" + xml.writeStartElement("ID");
" <Grantee xmlns:xsi=" + xml.writeCharacters(FAKE_OWNER_ID);
"\"http://www.w3.org/2001/XMLSchema-instance\"\r\n" + xml.writeEndElement();
" xsi:type=\"CanonicalUser\">\r\n" +
" <ID>" + FAKE_OWNER_ID + "</ID>\r\n" + xml.writeStartElement("DisplayName");
" <DisplayName>" + FAKE_OWNER_DISPLAY_NAME + xml.writeCharacters(FAKE_OWNER_DISPLAY_NAME);
"</DisplayName>\r\n" + xml.writeEndElement();
" </Grantee>\r\n" +
" <Permission>FULL_CONTROL</Permission>\r\n" + xml.writeEndElement();
" </Grant>\r\n" +
" </AccessControlList>\r\n" + xml.writeStartElement("AccessControlList");
"</AccessControlPolicy>"); xml.writeStartElement("Grant");
writer.flush();
xml.writeStartElement("Grantee");
xml.writeNamespace("xsi",
"http://www.w3.org/2001/XMLSchema-instance");
xml.writeAttribute("xsi:type", "CanonicalUser");
xml.writeStartElement("ID");
xml.writeCharacters(FAKE_OWNER_ID);
xml.writeEndElement();
xml.writeStartElement("DisplayName");
xml.writeCharacters(FAKE_OWNER_DISPLAY_NAME);
xml.writeEndElement();
xml.writeEndElement();
xml.writeStartElement("Permission");
xml.writeCharacters("FULL_CONTROL");
xml.writeEndElement();
xml.writeEndElement();
xml.writeEndElement();
xml.writeEndElement();
xml.flush();
} catch (XMLStreamException xse) {
throw new IOException(xse);
} }
} }
private void handleContainerList(HttpServletResponse response) private void handleContainerList(HttpServletResponse response)
throws IOException { throws IOException {
try (Writer writer = response.getWriter()) { try (Writer writer = response.getWriter()) {
writer.write(XML_PROLOG + XMLStreamWriter xml = xmlOutputFactory.createXMLStreamWriter(
"<ListAllMyBucketsResult " + AWS_XMLNS + ">\r\n" + writer);
" <Owner>\r\n" + xml.writeStartDocument();
" <ID>" + FAKE_OWNER_ID + "</ID>\r\n" + xml.writeStartElement("ListAllMyBucketsResult");
" <DisplayName>" + FAKE_OWNER_DISPLAY_NAME + xml.writeDefaultNamespace(AWS_XMLNS);
"</DisplayName>\r\n" +
" </Owner>\r\n" +
" <Buckets>\r\n");
xml.writeStartElement("Owner");
xml.writeStartElement("ID");
xml.writeCharacters(FAKE_OWNER_ID);
xml.writeEndElement();
xml.writeStartElement("DisplayName");
xml.writeCharacters(FAKE_OWNER_DISPLAY_NAME);
xml.writeEndElement();
xml.writeEndElement();
xml.writeStartElement("Buckets");
for (StorageMetadata metadata : blobStore.list()) { for (StorageMetadata metadata : blobStore.list()) {
writer.write(" <Bucket>\r\n" + xml.writeStartElement("Bucket");
" <Name>");
writer.write(metadata.getName()); xml.writeStartElement("Name");
writer.write("</Name>\r\n"); xml.writeCharacters(metadata.getName());
xml.writeEndElement();
Date creationDate = metadata.getCreationDate(); Date creationDate = metadata.getCreationDate();
if (creationDate != null) { if (creationDate != null) {
writer.write(" <CreationDate>"); xml.writeStartElement("CreationDate");
writer.write(blobStore.getContext().utils().date() xml.writeCharacters(blobStore.getContext().utils().date()
.iso8601DateFormat(creationDate).trim()); .iso8601DateFormat(creationDate).trim());
writer.write("</CreationDate>\r\n"); xml.writeEndElement();
} }
writer.write(" </Bucket>\r\n"); xml.writeEndElement();
} }
xml.writeEndElement();
writer.write(" </Buckets>\r\n" + xml.writeEndElement();
"</ListAllMyBucketsResult>"); xml.flush();
writer.flush(); } catch (XMLStreamException xse) {
throw new IOException(xse);
} }
} }
private void handleContainerLocation(HttpServletResponse response, private void handleContainerLocation(HttpServletResponse response,
String containerName) throws IOException { String containerName) throws IOException {
try (Writer writer = response.getWriter()) { try (Writer writer = response.getWriter()) {
XMLStreamWriter xml = xmlOutputFactory.createXMLStreamWriter(
writer);
xml.writeStartDocument();
// TODO: using us-standard semantics but could emit actual location // TODO: using us-standard semantics but could emit actual location
writer.write(XML_PROLOG + xml.writeStartElement("LocationConstraint");
"<LocationConstraint " + AWS_XMLNS + "/>"); xml.writeDefaultNamespace(AWS_XMLNS);
writer.flush(); xml.writeEndElement();
xml.flush();
} catch (XMLStreamException xse) {
throw new IOException(xse);
} }
} }
@ -494,9 +544,8 @@ final class S3ProxyHandler extends AbstractHandler {
return; return;
} }
sendSimpleErrorResponse(response, sendSimpleErrorResponse(response,
S3ErrorCode.BUCKET_ALREADY_OWNED_BY_YOU, S3ErrorCode.BUCKET_ALREADY_OWNED_BY_YOU, "BucketName",
Optional.of(" <BucketName>" + containerName + containerName);
"</BucketName>\r\n"));
} catch (AuthorizationException ae) { } catch (AuthorizationException ae) {
sendSimpleErrorResponse(response, sendSimpleErrorResponse(response,
S3ErrorCode.BUCKET_ALREADY_EXISTS); S3ErrorCode.BUCKET_ALREADY_EXISTS);
@ -555,41 +604,55 @@ final class S3ProxyHandler extends AbstractHandler {
try (Writer writer = response.getWriter()) { try (Writer writer = response.getWriter()) {
response.setStatus(HttpServletResponse.SC_OK); response.setStatus(HttpServletResponse.SC_OK);
writer.write(XML_PROLOG + XMLStreamWriter xml = xmlOutputFactory.createXMLStreamWriter(
"<ListBucketResult " + AWS_XMLNS + ">\r\n" + writer);
" <Name>"); xml.writeStartDocument();
writer.write(containerName); xml.writeStartElement("ListBucketResult");
writer.write("</Name>\r\n"); xml.writeDefaultNamespace(AWS_XMLNS);
xml.writeStartElement("Name");
xml.writeCharacters(containerName);
xml.writeEndElement();
if (prefix == null) { if (prefix == null) {
writer.write(" <Prefix/>\r\n"); xml.writeEmptyElement("Prefix");
} else { } else {
writer.write(" <Prefix>"); xml.writeStartElement("Prefix");
writer.write(prefix); xml.writeCharacters(prefix);
writer.write("</Prefix>\r\n"); xml.writeEndElement();
} }
writer.write(" <MaxKeys>");
writer.write(String.valueOf(maxKeys)); xml.writeStartElement("MaxKeys");
writer.write("</MaxKeys>\r\n"); xml.writeCharacters(String.valueOf(maxKeys));
xml.writeEndElement();
if (marker == null) { if (marker == null) {
writer.write(" <Marker/>\r\n"); xml.writeEmptyElement("Marker");
} else { } else {
writer.write(" <Marker>"); xml.writeStartElement("Marker");
writer.write(marker); xml.writeCharacters(marker);
writer.write("</Marker>\r\n"); xml.writeEndElement();
} }
if (delimiter != null) { if (delimiter != null) {
writer.write(" <Delimiter>"); xml.writeStartElement("Delimiter");
writer.write(delimiter); xml.writeCharacters(delimiter);
writer.write("</Delimiter>\r\n"); xml.writeEndElement();
} }
String nextMarker = set.getNextMarker(); String nextMarker = set.getNextMarker();
if (nextMarker != null) { if (nextMarker != null) {
writer.write(" <IsTruncated>true</IsTruncated>\r\n" + xml.writeStartElement("IsTruncated");
" <NextMarker>"); xml.writeCharacters("true");
writer.write(nextMarker); xml.writeEndElement();
writer.write("</NextMarker>\r\n");
xml.writeStartElement("NextMarker");
xml.writeCharacters(nextMarker);
xml.writeEndElement();
} else { } else {
writer.write(" <IsTruncated>false</IsTruncated>\r\n"); xml.writeStartElement("IsTruncated");
xml.writeCharacters("false");
xml.writeEndElement();
} }
Set<String> commonPrefixes = new TreeSet<>(); Set<String> commonPrefixes = new TreeSet<>();
@ -598,54 +661,74 @@ final class S3ProxyHandler extends AbstractHandler {
commonPrefixes.add(metadata.getName()); commonPrefixes.add(metadata.getName());
continue; continue;
} }
writer.write(" <Contents>\r\n" + xml.writeStartElement("Contents");
" <Key>");
writer.write(metadata.getName()); xml.writeStartElement("Key");
writer.write("</Key>\r\n"); xml.writeCharacters(metadata.getName());
xml.writeEndElement();
Date lastModified = metadata.getLastModified(); Date lastModified = metadata.getLastModified();
if (lastModified != null) { if (lastModified != null) {
writer.write(" <LastModified>"); xml.writeStartElement("LastModified");
writer.write(blobStore.getContext().utils().date() xml.writeCharacters(blobStore.getContext().utils().date()
.iso8601DateFormat(lastModified)); .iso8601DateFormat(lastModified));
writer.write("</LastModified>\r\n"); xml.writeEndElement();
} }
String eTag = metadata.getETag(); String eTag = metadata.getETag();
if (eTag != null) { if (eTag != null) {
String id = blobStore.getContext().unwrap() String id = blobStore.getContext().unwrap()
.getProviderMetadata().getId(); .getProviderMetadata().getId();
writer.write(" <ETag>&quot;"); xml.writeStartElement("ETag");
if (id.equals("google-cloud-storage")) { if (id.equals("google-cloud-storage")) {
eTag = BaseEncoding.base16().lowerCase().encode( eTag = BaseEncoding.base16().lowerCase().encode(
BaseEncoding.base64().decode(eTag)); BaseEncoding.base64().decode(eTag));
} }
writer.write(eTag); xml.writeCharacters("\"" + eTag + "\"");
writer.write("&quot;</ETag>\r\n"); xml.writeEndElement();
} }
writer.write(
// TODO: StorageMetadata does not contain size // TODO: StorageMetadata does not contain size
" <Size>0</Size>\r\n" + xml.writeStartElement("Size");
" <StorageClass>STANDARD</StorageClass>\r\n" + xml.writeCharacters("0");
" <Owner>\r\n" + xml.writeEndElement();
" <ID>" + FAKE_OWNER_ID + "</ID>\r\n" +
" <DisplayName>" + FAKE_OWNER_DISPLAY_NAME + xml.writeStartElement("StorageClass");
"</DisplayName>\r\n" + xml.writeCharacters("STANDARD");
" </Owner>\r\n" + xml.writeEndElement();
" </Contents>\r\n");
xml.writeStartElement("Owner");
xml.writeStartElement("ID");
xml.writeCharacters(FAKE_OWNER_ID);
xml.writeEndElement();
xml.writeStartElement("DisplayName");
xml.writeCharacters(FAKE_OWNER_DISPLAY_NAME);
xml.writeEndElement();
xml.writeEndElement();
xml.writeEndElement();
} }
for (String commonPrefix : commonPrefixes) { for (String commonPrefix : commonPrefixes) {
writer.write(" <CommonPrefixes>\r\n" + xml.writeStartElement("CommonPrefixes");
" <Prefix>");
writer.write(commonPrefix); xml.writeStartElement("Prefix");
xml.writeCharacters(commonPrefix);
if (delimiter != null) { if (delimiter != null) {
writer.write(delimiter); xml.writeCharacters(delimiter);
} }
writer.write("</Prefix>\r\n" + xml.writeEndElement();
" </CommonPrefixes>\r\n");
xml.writeEndElement();
} }
writer.write("</ListBucketResult>"); xml.writeEndElement();
writer.flush(); xml.flush();
} catch (XMLStreamException xse) {
throw new IOException(xse);
} }
} }
@ -664,8 +747,11 @@ final class S3ProxyHandler extends AbstractHandler {
HttpServletResponse response, String containerName) HttpServletResponse response, String containerName)
throws IOException { throws IOException {
try (Writer writer = response.getWriter()) { try (Writer writer = response.getWriter()) {
writer.write(XML_PROLOG); XMLStreamWriter xml = xmlOutputFactory.createXMLStreamWriter(
writer.write("<DeleteResult " + AWS_XMLNS + ">\r\n"); writer);
xml.writeStartDocument();
xml.writeStartElement("DeleteResult");
xml.writeDefaultNamespace(AWS_XMLNS);
// TODO: more robust XML parsing // TODO: more robust XML parsing
Matcher matcher = MULTI_DELETE_KEY_PATTERN.matcher( Matcher matcher = MULTI_DELETE_KEY_PATTERN.matcher(
Strings2.toStringAndClose(request.getInputStream())); Strings2.toStringAndClose(request.getInputStream()));
@ -673,13 +759,17 @@ final class S3ProxyHandler extends AbstractHandler {
String blobName = matcher.group(1); String blobName = matcher.group(1);
blobStore.removeBlob(containerName, blobName); blobStore.removeBlob(containerName, blobName);
writer.write("<Deleted><Key>"); xml.writeStartElement("Deleted");
writer.write(blobName); xml.writeStartElement("Key");
writer.write("</Key></Deleted>\r\n"); xml.writeCharacters(blobName);
xml.writeEndElement();
xml.writeEndElement();
} }
// TODO: emit error stanza // TODO: emit error stanza
writer.write("</DeleteResult>"); xml.writeEndElement();
writer.flush(); xml.flush();
} catch (XMLStreamException xse) {
throw new IOException(xse);
} }
} }
@ -800,17 +890,25 @@ final class S3ProxyHandler extends AbstractHandler {
builder.build()); builder.build());
Date lastModified = blob.getMetadata().getLastModified(); Date lastModified = blob.getMetadata().getLastModified();
try (Writer writer = response.getWriter()) { try (Writer writer = response.getWriter()) {
writer.write(XML_PROLOG + XMLStreamWriter xml = xmlOutputFactory.createXMLStreamWriter(
"<CopyObjectResult " + AWS_XMLNS + ">\r\n"); writer);
writer.write(" <LastModified>"); xml.writeStartDocument();
writer.write(blobStore.getContext().utils().date() xml.writeStartElement("CopyObjectResult");
xml.writeDefaultNamespace(AWS_XMLNS);
xml.writeStartElement("LastModified");
xml.writeCharacters(blobStore.getContext().utils().date()
.iso8601DateFormat(lastModified)); .iso8601DateFormat(lastModified));
writer.write("</LastModified>\r\n"); xml.writeEndElement();
writer.write(" <ETag>&quot;");
writer.write(eTag); xml.writeStartElement("ETag");
writer.write("&quot;</ETag>\r\n"); xml.writeCharacters("\"" + eTag + "\"");
writer.write("</CopyObjectResult>"); xml.writeEndElement();
writer.flush();
xml.writeEndElement();
xml.flush();
} catch (XMLStreamException xse) {
throw new IOException(xse);
} }
} }
} }
@ -970,31 +1068,47 @@ final class S3ProxyHandler extends AbstractHandler {
} }
} }
private static void sendSimpleErrorResponse(HttpServletResponse response, private void sendSimpleErrorResponse(HttpServletResponse response,
S3ErrorCode code) throws IOException { S3ErrorCode code) throws IOException {
sendSimpleErrorResponse(response, code, Optional.<String>absent()); sendSimpleErrorResponse(response, code, null, null);
} }
private static void sendSimpleErrorResponse(HttpServletResponse response, private void sendSimpleErrorResponse(HttpServletResponse response,
S3ErrorCode code, Optional<String> extra) throws IOException { S3ErrorCode code, String element, String characters)
logger.debug("{} {}", code, extra); throws IOException {
Preconditions.checkArgument(!(element == null ^ characters == null),
"Must specify neither or both element and characters");
logger.debug("{} {} {}", code, element, characters);
try (Writer writer = response.getWriter()) { try (Writer writer = response.getWriter()) {
XMLStreamWriter xml = xmlOutputFactory.createXMLStreamWriter(
writer);
response.setStatus(code.getHttpStatusCode()); response.setStatus(code.getHttpStatusCode());
writer.write(XML_PROLOG + xml.writeStartDocument();
"<Error>\r\n" + xml.writeStartElement("Error");
" <Code>");
writer.write(code.getErrorCode()); xml.writeStartElement("Code");
writer.write("</Code>\r\n" + xml.writeCharacters(code.getErrorCode());
" <Message>"); xml.writeEndElement();
writer.write(code.getMessage());
writer.write("</Message>\r\n"); xml.writeStartElement("Message");
if (extra.isPresent()) { xml.writeCharacters(code.getMessage());
writer.write(extra.get()); xml.writeEndElement();
if (element != null) {
xml.writeStartElement(element);
xml.writeCharacters(characters);
xml.writeEndElement();
} }
writer.write(" <RequestId>" + FAKE_REQUEST_ID +
"</RequestId>\r\n" + xml.writeStartElement("RequestId");
"</Error>"); xml.writeCharacters(FAKE_REQUEST_ID);
writer.flush(); xml.writeEndElement();
xml.writeEndElement();
xml.flush();
} catch (XMLStreamException xse) {
throw new IOException(xse);
} }
} }