kopia lustrzana https://github.com/gaul/s3proxy
Initial commit of S3Proxy
Andrew Gaul
commit
ff84ccf19e
|
|
@ -0,0 +1 @@
|
|||
target/
|
||||
|
|
@ -0,0 +1,202 @@
|
|||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
S3Proxy
|
||||
=======
|
||||
S3Proxy allows applications using the S3 API to interface with a variety of
|
||||
object stores, e.g., EMC Atmos, Microsoft Azure, OpenStack Swift. It runs a
|
||||
local HTTP server which translates S3 operations into provider-specific
|
||||
operations. S3Proxy also allows developers to test against S3 without the cost
|
||||
or latency associated with using AWS by using the local file system.
|
||||
|
||||
Features
|
||||
--------
|
||||
* create, remove, list containers
|
||||
* put, get, remove, list blobs (up to 2 GB in size)
|
||||
* store and retrieve user metadata
|
||||
|
||||
Supported object stores:
|
||||
|
||||
* atmos
|
||||
* aws-s3
|
||||
* azureblob
|
||||
* cloudfiles-uk and cloudfiles-us
|
||||
* filesystem (on-disk storage)
|
||||
* hpcloud-objectstorage
|
||||
* s3
|
||||
* swift and swift-keystone
|
||||
* transient (in-memory storage)
|
||||
|
||||
Installation
|
||||
------------
|
||||
S3Proxy requires Java 7 to run. Presently there is no binary release but
|
||||
[Bintray](https://bintray.com/) will host releases in the future.
|
||||
|
||||
One can build the project by running `mvn package` which produces a binary at
|
||||
`target/s3proxy`.
|
||||
|
||||
Examples
|
||||
--------
|
||||
Linux and Mac OS users can run S3Proxy either via the executable jar or by
|
||||
explicitly invoking java:
|
||||
|
||||
```
|
||||
s3proxy --properties s3proxy.conf
|
||||
java -jar s3proxy --properties s3proxy.conf
|
||||
```
|
||||
|
||||
Windows users must explicitly invoke java.
|
||||
|
||||
Configuration
|
||||
-------------
|
||||
Users can configure S3Proxy via a properties file. An example:
|
||||
|
||||
```
|
||||
jclouds.provider=transient
|
||||
jclouds.identity=identity
|
||||
jclouds.credential=credential
|
||||
#jclouds.endpoint=http://127.0.0.1:8081 # optional for some providers
|
||||
jclouds.filesystem.basedir=/tmp/blobstore
|
||||
s3proxy.endpoint=http://127.0.0.1:8080
|
||||
```
|
||||
|
||||
Users can also set a variety of Java and
|
||||
[jclouds properties](https://github.com/jclouds/jclouds/blob/master/core/src/main/java/org/jclouds/Constants.java).
|
||||
|
||||
Limitations
|
||||
-----------
|
||||
S3Proxy does not support:
|
||||
|
||||
* single-part uploads larger than 2 GB ([upstream issue](https://github.com/jclouds/jclouds/pull/426))
|
||||
* multi-part uploads
|
||||
* authorization of clients
|
||||
* bucket ACLs
|
||||
* server-side copy
|
||||
* URL signing
|
||||
* metadata with filesystem provider ([upstream issue](https://github.com/jclouds/jclouds/pull/443))
|
||||
* listening on HTTPS
|
||||
|
||||
References
|
||||
----------
|
||||
[jclouds](http://jclouds.apache.org/) provides object store support for
|
||||
S3Proxy. [s3fs-fuse](https://github.com/s3fs-fuse/s3fs-fuse) provides
|
||||
file system access to S3 and S3Proxy allows use of other providers.
|
||||
|
||||
License
|
||||
-------
|
||||
Copyright (C) 2014 Andrew Gaul
|
||||
|
||||
Licensed under the Apache License, Version 2.0
|
||||
|
|
@ -0,0 +1,199 @@
|
|||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<groupId>org.sonatype.oss</groupId>
|
||||
<artifactId>oss-parent</artifactId>
|
||||
<version>7</version>
|
||||
</parent>
|
||||
|
||||
<groupId>org.gaul</groupId>
|
||||
<artifactId>s3proxy</artifactId>
|
||||
<version>0.0.0-SNAPSHOT</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>S3 Proxy</name>
|
||||
<url>https://github.com/andrewgaul/s3proxy</url>
|
||||
<description>S3Proxy allows applications using the S3 API to interface with a variety of object stores</description>
|
||||
|
||||
<licenses>
|
||||
<license>
|
||||
<name>The Apache Software License, Version 2.0</name>
|
||||
<url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
|
||||
<distribution>repo</distribution>
|
||||
</license>
|
||||
</licenses>
|
||||
|
||||
<scm>
|
||||
<connection>scm:git:git@github.com:andrewgaul/s3proxy.git</connection>
|
||||
<developerConnection>scm:git:git@github.com:andrewgaul/s3proxy.git</developerConnection>
|
||||
<url>git@github.com:andrewgaul/s3proxy.git</url>
|
||||
</scm>
|
||||
|
||||
<developers>
|
||||
<developer>
|
||||
<name>Andrew Gaul</name>
|
||||
<id>gaul</id>
|
||||
<email>andrew@gaul.org</email>
|
||||
</developer>
|
||||
</developers>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.1</version>
|
||||
<configuration>
|
||||
<source>1.7</source>
|
||||
<target>1.7</target>
|
||||
<showDeprecation>true</showDeprecation>
|
||||
<showWarnings>true</showWarnings>
|
||||
<compilerArguments>
|
||||
<Xlint />
|
||||
</compilerArguments>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-shade-plugin</artifactId>
|
||||
<version>2.3</version>
|
||||
<configuration>
|
||||
<createDependencyReducedPom>false</createDependencyReducedPom>
|
||||
<filters>
|
||||
<filter>
|
||||
<artifact>*:*</artifact>
|
||||
<excludes>
|
||||
<exclude>META-INF/*.SF</exclude>
|
||||
<exclude>META-INF/*.DSA</exclude>
|
||||
<exclude>META-INF/*.RSA</exclude>
|
||||
<exclude>about.html</exclude>
|
||||
</excludes>
|
||||
</filter>
|
||||
</filters>
|
||||
<shadedArtifactAttached>true</shadedArtifactAttached>
|
||||
<transformers>
|
||||
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
|
||||
<manifestEntries>
|
||||
<Main-Class>org.gaul.s3proxy.S3Proxy</Main-Class>
|
||||
<Implementation-Build>${buildNumber}</Implementation-Build>
|
||||
<Implementation-Version>${project.version}</Implementation-Version>
|
||||
<Build-Date>${timestamp}</Build-Date>
|
||||
</manifestEntries>
|
||||
</transformer>
|
||||
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
|
||||
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
|
||||
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer"/>
|
||||
</transformers>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>jar-with-dependencies</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>shade</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<shadedClassifierName>jar-with-dependencies</shadedClassifierName>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.skife.maven</groupId>
|
||||
<artifactId>really-executable-jar-maven-plugin</artifactId>
|
||||
<version>1.3.0</version>
|
||||
<configuration>
|
||||
<programFile>s3proxy</programFile>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>really-executable-jar</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<version>2.15</version>
|
||||
<configuration>
|
||||
<parallel>classes</parallel>
|
||||
<threadCount>1</threadCount>
|
||||
<argLine>-Xmx256m</argLine>
|
||||
<redirectTestOutputToFile>true</redirectTestOutputToFile>
|
||||
<forkedProcessTimeoutInSeconds>300</forkedProcessTimeoutInSeconds>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-checkstyle-plugin</artifactId>
|
||||
<version>2.12.1</version>
|
||||
<configuration>
|
||||
<configLocation>src/main/resources/checkstyle.xml</configLocation>
|
||||
<headerLocation>src/main/resources/copyright_header.txt</headerLocation>
|
||||
<includeTestSourceDirectory>true</includeTestSourceDirectory>
|
||||
<violationSeverity>warning</violationSeverity>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<!-- TODO: change to release version -->
|
||||
<jclouds.version>1.8.0-SNAPSHOT</jclouds.version>
|
||||
<slf4j.version>1.7.7</slf4j.version>
|
||||
</properties>
|
||||
|
||||
<prerequisites>
|
||||
<maven>3.0.5</maven>
|
||||
</prerequisites>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>ch.qos.logback</groupId>
|
||||
<artifactId>logback-classic</artifactId>
|
||||
<version>1.1.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<version>4.11</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.jclouds</groupId>
|
||||
<artifactId>jclouds-allblobstore</artifactId>
|
||||
<version>${jclouds.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.jclouds.labs</groupId>
|
||||
<artifactId>glacier</artifactId>
|
||||
<version>${jclouds.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.assertj</groupId>
|
||||
<artifactId>assertj-core</artifactId>
|
||||
<scope>test</scope>
|
||||
<version>1.6.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-servlet</artifactId>
|
||||
<version>9.2.2.v20140723</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
<version>${slf4j.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-simple</artifactId>
|
||||
<version>${slf4j.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
|
|
@ -0,0 +1,111 @@
|
|||
/*
|
||||
* Copyright 2014 Andrew Gaul <andrew@gaul.org>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.gaul.s3proxy;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.net.URI;
|
||||
import java.util.Properties;
|
||||
|
||||
import com.google.common.base.Preconditions;
|
||||
|
||||
import org.eclipse.jetty.server.HttpConnectionFactory;
|
||||
import org.eclipse.jetty.server.Server;
|
||||
import org.eclipse.jetty.server.ServerConnector;
|
||||
import org.jclouds.Constants;
|
||||
import org.jclouds.ContextBuilder;
|
||||
import org.jclouds.blobstore.BlobStore;
|
||||
import org.jclouds.blobstore.BlobStoreContext;
|
||||
|
||||
/**
|
||||
* S3Proxy translates S3 HTTP operations into jclouds provider-agnostic
|
||||
* operations. This allows applications using the S3 API to interface with any
|
||||
* provider that jclouds supports, e.g., EMC Atmos, Microsoft Azure,
|
||||
* OpenStack Swift.
|
||||
*/
|
||||
public final class S3Proxy {
|
||||
private static final String PROPERTY_S3PROXY_ENDPOINT = "s3proxy.endpoint";
|
||||
|
||||
private final Server server;
|
||||
|
||||
public S3Proxy(BlobStore blobStore, URI endpoint) {
|
||||
Preconditions.checkNotNull(blobStore);
|
||||
Preconditions.checkNotNull(endpoint);
|
||||
// TODO: allow service paths?
|
||||
Preconditions.checkArgument(endpoint.getPath().isEmpty(),
|
||||
"endpoint path must be empty, was: " + endpoint.getPath());
|
||||
|
||||
server = new Server();
|
||||
HttpConnectionFactory httpConnectionFactory =
|
||||
new HttpConnectionFactory();
|
||||
ServerConnector connector = new ServerConnector(server,
|
||||
httpConnectionFactory);
|
||||
connector.setHost(endpoint.getHost());
|
||||
connector.setPort(endpoint.getPort());
|
||||
server.addConnector(connector);
|
||||
server.setHandler(new S3ProxyHandler(blobStore));
|
||||
}
|
||||
|
||||
public void start() throws Exception {
|
||||
server.start();
|
||||
}
|
||||
|
||||
public void stop() throws Exception {
|
||||
server.stop();
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
if (args.length != 2) {
|
||||
System.err.println("Usage: s3proxy --properties FILE");
|
||||
System.exit(1);
|
||||
}
|
||||
Properties properties = new Properties();
|
||||
try (InputStream is = new FileInputStream(new File(args[1]))) {
|
||||
properties.load(is);
|
||||
}
|
||||
|
||||
String provider = properties.getProperty(Constants.PROPERTY_PROVIDER);
|
||||
String identity = properties.getProperty(Constants.PROPERTY_IDENTITY);
|
||||
String credential = properties.getProperty(
|
||||
Constants.PROPERTY_CREDENTIAL);
|
||||
String endpoint = properties.getProperty(Constants.PROPERTY_ENDPOINT);
|
||||
String s3ProxyEndpointString = properties.getProperty(
|
||||
PROPERTY_S3PROXY_ENDPOINT);
|
||||
if (provider == null || identity == null || credential == null
|
||||
|| s3ProxyEndpointString == null) {
|
||||
System.err.println("Properties file must contain:\n" +
|
||||
Constants.PROPERTY_PROVIDER + "\n" +
|
||||
Constants.PROPERTY_IDENTITY + "\n" +
|
||||
Constants.PROPERTY_CREDENTIAL + "\n" +
|
||||
PROPERTY_S3PROXY_ENDPOINT);
|
||||
System.exit(1);
|
||||
}
|
||||
|
||||
ContextBuilder builder = ContextBuilder
|
||||
.newBuilder(provider)
|
||||
.credentials(identity, credential)
|
||||
.overrides(properties);
|
||||
if (endpoint != null) {
|
||||
builder = builder.endpoint(endpoint);
|
||||
}
|
||||
BlobStoreContext context = builder.build(BlobStoreContext.class);
|
||||
URI s3ProxyEndpoint = new URI(s3ProxyEndpointString);
|
||||
S3Proxy s3Proxy = new S3Proxy(context.getBlobStore(), s3ProxyEndpoint);
|
||||
s3Proxy.start();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,551 @@
|
|||
/*
|
||||
* Copyright 2014 Andrew Gaul <andrew@gaul.org>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.gaul.s3proxy;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.io.Writer;
|
||||
import java.util.Date;
|
||||
import java.util.Enumeration;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import com.google.common.base.Preconditions;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.hash.HashCode;
|
||||
import com.google.common.io.BaseEncoding;
|
||||
import com.google.common.io.ByteStreams;
|
||||
import com.google.common.net.HttpHeaders;
|
||||
|
||||
import org.eclipse.jetty.server.Request;
|
||||
import org.eclipse.jetty.server.handler.AbstractHandler;
|
||||
import org.jclouds.blobstore.BlobStore;
|
||||
import org.jclouds.blobstore.ContainerNotFoundException;
|
||||
import org.jclouds.blobstore.domain.Blob;
|
||||
import org.jclouds.blobstore.domain.BlobBuilder;
|
||||
import org.jclouds.blobstore.domain.BlobMetadata;
|
||||
import org.jclouds.blobstore.domain.PageSet;
|
||||
import org.jclouds.blobstore.domain.StorageMetadata;
|
||||
import org.jclouds.blobstore.domain.StorageType;
|
||||
import org.jclouds.blobstore.options.GetOptions;
|
||||
import org.jclouds.blobstore.options.ListContainerOptions;
|
||||
import org.jclouds.domain.Location;
|
||||
import org.jclouds.http.HttpResponseException;
|
||||
import org.jclouds.io.ContentMetadata;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
final class S3ProxyHandler extends AbstractHandler {
|
||||
private static final Logger logger = LoggerFactory.getLogger(
|
||||
S3ProxyHandler.class);
|
||||
// TODO: support configurable metadata prefix
|
||||
private static final String USER_METADATA_PREFIX = "x-amz-meta-";
|
||||
// TODO: fake owner
|
||||
private static final String FAKE_OWNER_ID =
|
||||
"75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a";
|
||||
private static final String FAKE_OWNER_DISPLAY_NAME =
|
||||
"CustomersName@amazon.com";
|
||||
private final BlobStore blobStore;
|
||||
|
||||
S3ProxyHandler(BlobStore blobStore) {
|
||||
this.blobStore = Preconditions.checkNotNull(blobStore);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void handle(String target, Request baseRequest,
|
||||
HttpServletRequest request, HttpServletResponse response)
|
||||
throws IOException {
|
||||
int errorCode;
|
||||
String method = request.getMethod();
|
||||
String uri = request.getRequestURI();
|
||||
logger.debug("request: {}", request);
|
||||
|
||||
switch (method) {
|
||||
case "DELETE":
|
||||
if (uri.lastIndexOf("/") == 0) {
|
||||
errorCode = handleContainerDelete(uri.substring(1));
|
||||
if (errorCode != HttpServletResponse.SC_OK) {
|
||||
response.sendError(errorCode);
|
||||
}
|
||||
baseRequest.setHandled(true);
|
||||
return;
|
||||
} else {
|
||||
String[] path = uri.split("/", 3);
|
||||
errorCode = handleBlobRemove(response, path[1], path[2]);
|
||||
if (errorCode != HttpServletResponse.SC_OK) {
|
||||
response.sendError(errorCode);
|
||||
}
|
||||
baseRequest.setHandled(true);
|
||||
return;
|
||||
}
|
||||
case "GET":
|
||||
if (uri.equals("/")) {
|
||||
errorCode = handleContainerList(response);
|
||||
if (errorCode != HttpServletResponse.SC_OK) {
|
||||
response.sendError(errorCode);
|
||||
}
|
||||
baseRequest.setHandled(true);
|
||||
return;
|
||||
} else if (uri.lastIndexOf("/") == 0 &&
|
||||
"".equals(request.getParameter("acl"))) {
|
||||
handleContainerAcl(response, uri.substring(1));
|
||||
baseRequest.setHandled(true);
|
||||
return;
|
||||
} else if (uri.lastIndexOf("/") == 0 &&
|
||||
"0".equals(request.getParameter("max-keys"))) {
|
||||
errorCode = handleContainerExists(uri.substring(1));
|
||||
if (errorCode != HttpServletResponse.SC_OK) {
|
||||
response.sendError(errorCode);
|
||||
}
|
||||
baseRequest.setHandled(true);
|
||||
return;
|
||||
} else if (uri.lastIndexOf("/") == 0) {
|
||||
handleBlobList(request, response, uri.substring(1));
|
||||
baseRequest.setHandled(true);
|
||||
return;
|
||||
} else if (uri.lastIndexOf("/") == uri.length() - 1) {
|
||||
handleBlobList(request, response, uri.substring(1,
|
||||
uri.length() - 1));
|
||||
baseRequest.setHandled(true);
|
||||
return;
|
||||
} else {
|
||||
String[] path = uri.split("/", 3);
|
||||
errorCode = handleGetBlob(request, response, path[1], path[2]);
|
||||
if (errorCode != HttpServletResponse.SC_OK) {
|
||||
response.sendError(errorCode);
|
||||
}
|
||||
baseRequest.setHandled(true);
|
||||
return;
|
||||
}
|
||||
case "HEAD":
|
||||
if (uri.lastIndexOf("/") == 0) {
|
||||
errorCode = handleContainerExists(uri.substring(1));
|
||||
if (errorCode != HttpServletResponse.SC_OK) {
|
||||
response.sendError(errorCode);
|
||||
}
|
||||
baseRequest.setHandled(true);
|
||||
return;
|
||||
} else {
|
||||
String[] path = uri.split("/", 3);
|
||||
handleBlobMetadata(response, path[1], path[2]);
|
||||
baseRequest.setHandled(true);
|
||||
return;
|
||||
}
|
||||
case "PUT":
|
||||
if (uri.lastIndexOf("/") == 0) {
|
||||
handleContainerCreate(response, uri.substring(1));
|
||||
baseRequest.setHandled(true);
|
||||
return;
|
||||
} else {
|
||||
String[] path = uri.split("/", 3);
|
||||
errorCode = handlePutBlob(request, response, path[1], path[2]);
|
||||
if (errorCode != HttpServletResponse.SC_OK) {
|
||||
response.sendError(errorCode);
|
||||
}
|
||||
baseRequest.setHandled(true);
|
||||
return;
|
||||
}
|
||||
default:
|
||||
logger.error("Unknown method {} with URI {}",
|
||||
method, request.getRequestURI());
|
||||
response.sendError(HttpServletResponse.SC_NOT_IMPLEMENTED);
|
||||
baseRequest.setHandled(true);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
private int handleContainerAcl(HttpServletResponse response,
|
||||
String containerName) {
|
||||
try (Writer writer = response.getWriter()) {
|
||||
writer.write("<AccessControlPolicy>\r\n" +
|
||||
" <Owner>\r\n" +
|
||||
" <ID>" + FAKE_OWNER_ID + "</ID>\r\n" +
|
||||
" <DisplayName>" + FAKE_OWNER_DISPLAY_NAME + "</DisplayName>\r\n" +
|
||||
" </Owner>\r\n" +
|
||||
" <AccessControlList>\r\n" +
|
||||
" <Grant>\r\n" +
|
||||
" <Grantee xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\r\n" +
|
||||
" xsi:type=\"CanonicalUser\">\r\n" +
|
||||
" <ID>" + FAKE_OWNER_ID + "</ID>\r\n" +
|
||||
" <DisplayName>" + FAKE_OWNER_DISPLAY_NAME + "</DisplayName>\r\n" +
|
||||
" </Grantee>\r\n" +
|
||||
" <Permission>FULL_CONTROL</Permission>\r\n" +
|
||||
" </Grant>\r\n" +
|
||||
" </AccessControlList>\r\n" +
|
||||
"</AccessControlPolicy>\r\n");
|
||||
writer.flush();
|
||||
} catch (IOException ioe) {
|
||||
logger.error("Error writing to client: {}", ioe.getMessage());
|
||||
return HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
|
||||
}
|
||||
return HttpServletResponse.SC_OK;
|
||||
}
|
||||
|
||||
private int handleContainerList(HttpServletResponse response) {
|
||||
try (Writer writer = response.getWriter()) {
|
||||
writer.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n" +
|
||||
"<ListAllMyBucketsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\r\n" +
|
||||
" <Owner>\r\n" +
|
||||
" <ID>" + FAKE_OWNER_ID + "</ID>\r\n" +
|
||||
" <DisplayName>" + FAKE_OWNER_DISPLAY_NAME + "</DisplayName>\r\n" +
|
||||
" </Owner>\r\n" +
|
||||
" <Buckets>");
|
||||
|
||||
for (StorageMetadata metadata : blobStore.list()) {
|
||||
writer.write("<Bucket>\r\n" +
|
||||
" <Name>");
|
||||
writer.write(metadata.getName());
|
||||
writer.write("</Name>\r\n");
|
||||
Date creationDate = metadata.getCreationDate();
|
||||
if (creationDate != null) {
|
||||
writer.write(" <CreationDate>");
|
||||
writer.write(blobStore.getContext().utils().date()
|
||||
.iso8601DateFormat(creationDate));
|
||||
writer.write("</CreationDate>\r\n");
|
||||
}
|
||||
writer.write("</Bucket>\r\n");
|
||||
}
|
||||
|
||||
writer.write(" </Buckets>\r\n" +
|
||||
"</ListAllMyBucketsResult>\r\n");
|
||||
writer.flush();
|
||||
} catch (IOException ioe) {
|
||||
logger.error("Error writing to client: {}", ioe.getMessage());
|
||||
return HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
|
||||
}
|
||||
return HttpServletResponse.SC_OK;
|
||||
}
|
||||
|
||||
private int handleContainerExists(String containerName) {
|
||||
try {
|
||||
return blobStore.containerExists(containerName)
|
||||
? HttpServletResponse.SC_OK
|
||||
: HttpServletResponse.SC_NOT_FOUND;
|
||||
} catch (RuntimeException re) {
|
||||
logger.error("Error determining container existence: {}",
|
||||
re.getMessage());
|
||||
return HttpServletResponse.SC_FORBIDDEN;
|
||||
}
|
||||
}
|
||||
|
||||
private void handleContainerCreate(HttpServletResponse response,
|
||||
String containerName) {
|
||||
try {
|
||||
// TODO: how to support locations?
|
||||
Location location = null;
|
||||
if (blobStore.createContainerInLocation(location, containerName)) {
|
||||
return;
|
||||
}
|
||||
try (Writer writer = response.getWriter()) {
|
||||
response.setStatus(HttpServletResponse.SC_CONFLICT);
|
||||
writer.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n" +
|
||||
"<Error>\r\n" +
|
||||
" <Code>BucketAlreadyOwnedByYou</Code>\r\n" +
|
||||
" <Message>Your previous request to create the named bucket succeeded and you already own it.</Message>\r\n" +
|
||||
" <BucketName>");
|
||||
writer.write(containerName);
|
||||
writer.write("</BucketName>\r\n" +
|
||||
// TODO: RequestId
|
||||
" <RequestId>4442587FB7D0A2F9</RequestId>\r\n" +
|
||||
"</Error>\r\n");
|
||||
writer.flush();
|
||||
} catch (IOException ioe) {
|
||||
logger.error("Error writing to client: {}",
|
||||
ioe.getMessage());
|
||||
response.setStatus(
|
||||
HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
|
||||
}
|
||||
} catch (RuntimeException re) {
|
||||
logger.error("Error creating container: {}", re.getMessage());
|
||||
response.setStatus(HttpServletResponse.SC_FORBIDDEN);
|
||||
}
|
||||
}
|
||||
|
||||
private int handleContainerDelete(String containerName) {
|
||||
try {
|
||||
return blobStore.deleteContainerIfEmpty(containerName)
|
||||
? HttpServletResponse.SC_OK
|
||||
// TODO: emit BucketNotEmpty error code?
|
||||
: HttpServletResponse.SC_CONFLICT;
|
||||
} catch (RuntimeException re) {
|
||||
logger.error("Error deleting container {}: {}", containerName,
|
||||
re.getMessage());
|
||||
return HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
private void handleBlobList(HttpServletRequest request,
|
||||
HttpServletResponse response, String containerName) {
|
||||
ListContainerOptions options = new ListContainerOptions();
|
||||
String delimiter = request.getParameter("delimiter");
|
||||
if (!(delimiter != null && delimiter.equals("/"))) {
|
||||
options = options.recursive();
|
||||
}
|
||||
String prefix = request.getParameter("prefix");
|
||||
if (prefix != null) {
|
||||
options = options.inDirectory(prefix);
|
||||
}
|
||||
String marker = request.getParameter("marker");
|
||||
if (marker != null) {
|
||||
options = options.afterMarker(request.getParameter("marker"));
|
||||
}
|
||||
String maxKeys = request.getParameter("max-keys");
|
||||
if (maxKeys != null) {
|
||||
options = options.maxResults(Integer.valueOf(maxKeys));
|
||||
}
|
||||
|
||||
PageSet<? extends StorageMetadata> set;
|
||||
try {
|
||||
set = blobStore.list(containerName, options);
|
||||
} catch (ContainerNotFoundException cnfe) {
|
||||
response.setStatus(HttpServletResponse.SC_NOT_FOUND);
|
||||
return;
|
||||
}
|
||||
|
||||
try (Writer writer = response.getWriter()) {
|
||||
response.setStatus(HttpServletResponse.SC_OK);
|
||||
writer.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n" +
|
||||
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\r\n" +
|
||||
" <Name>");
|
||||
writer.write(containerName);
|
||||
writer.write("</Name>\r\n");
|
||||
if (prefix == null) {
|
||||
writer.write(" <Prefix/>\r\n");
|
||||
} else {
|
||||
writer.write(" <Prefix>");
|
||||
writer.write(prefix);
|
||||
writer.write("</Prefix>\r\n");
|
||||
}
|
||||
writer.write(" <MaxKeys>");
|
||||
writer.write(String.valueOf(set.size()));
|
||||
writer.write("</MaxKeys>\r\n");
|
||||
if (marker == null) {
|
||||
writer.write(" <Marker/>\r\n");
|
||||
} else {
|
||||
writer.write(" <Marker>");
|
||||
writer.write(marker);
|
||||
writer.write("</Marker>\r\n");
|
||||
}
|
||||
if (delimiter != null) {
|
||||
writer.write(" <Delimiter>");
|
||||
writer.write(delimiter);
|
||||
writer.write("</Delimiter>\r\n");
|
||||
}
|
||||
String nextMarker = set.getNextMarker();
|
||||
if (nextMarker != null) {
|
||||
writer.write(" <IsTruncated>true</IsTruncated>\r\n" +
|
||||
" <NextMarker>");
|
||||
writer.write(nextMarker);
|
||||
writer.write("</NextMarker>\r\n");
|
||||
} else {
|
||||
writer.write(" <IsTruncated>false</IsTruncated>\r\n");
|
||||
}
|
||||
|
||||
Set<String> commonPrefixes = new HashSet<>();
|
||||
for (StorageMetadata metadata : set) {
|
||||
if (metadata.getType() != StorageType.BLOB) {
|
||||
commonPrefixes.add(metadata.getName());
|
||||
continue;
|
||||
}
|
||||
writer.write(" <Contents>\r\n" +
|
||||
" <Key>");
|
||||
writer.write(metadata.getName());
|
||||
writer.write("</Key>\r\n");
|
||||
Date lastModified = metadata.getLastModified();
|
||||
if (lastModified != null) {
|
||||
writer.write(" <LastModified>");
|
||||
writer.write(blobStore.getContext().utils().date()
|
||||
.iso8601DateFormat(lastModified));
|
||||
writer.write("</LastModified>\r\n");
|
||||
}
|
||||
String eTag = metadata.getETag();
|
||||
if (eTag != null) {
|
||||
writer.write(" <ETag>"");
|
||||
writer.write(metadata.getETag());
|
||||
writer.write(""</ETag>\r\n");
|
||||
}
|
||||
writer.write(
|
||||
// TODO: StorageMetadata does not contain size
|
||||
" <Size>0</Size>\r\n" +
|
||||
" <StorageClass>STANDARD</StorageClass>\r\n" +
|
||||
" <Owner>\r\n" +
|
||||
" <ID>" + FAKE_OWNER_ID + "</ID>\r\n" +
|
||||
" <DisplayName>" + FAKE_OWNER_DISPLAY_NAME + "</DisplayName>\r\n" +
|
||||
" </Owner>\r\n" +
|
||||
" </Contents>\r\n");
|
||||
}
|
||||
|
||||
for (String commonPrefix : commonPrefixes) {
|
||||
writer.write(" <CommonPrefixes>\r\n" +
|
||||
" <Prefix>");
|
||||
writer.write(commonPrefix);
|
||||
writer.write("</Prefix>\r\n" +
|
||||
" </CommonPrefixes>\r\n");
|
||||
}
|
||||
|
||||
writer.write("</ListBucketResult>");
|
||||
writer.flush();
|
||||
} catch (IOException ioe) {
|
||||
logger.error("Error writing to client: {}",
|
||||
ioe.getMessage());
|
||||
response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
private int handleBlobRemove(HttpServletResponse response,
|
||||
String containerName, String blobName) {
|
||||
try {
|
||||
blobStore.removeBlob(containerName, blobName);
|
||||
return HttpServletResponse.SC_OK;
|
||||
} catch (RuntimeException re) {
|
||||
logger.error("Error removing blob {} {}: {}", containerName,
|
||||
blobName, re.getMessage());
|
||||
return HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
private void handleBlobMetadata(HttpServletResponse response,
|
||||
String containerName, String blobName) {
|
||||
BlobMetadata metadata = blobStore.blobMetadata(containerName, blobName);
|
||||
if (metadata == null) {
|
||||
response.setStatus(HttpServletResponse.SC_NOT_FOUND);
|
||||
return;
|
||||
}
|
||||
|
||||
response.setStatus(HttpServletResponse.SC_OK);
|
||||
addMetadataToResponse(response, metadata);
|
||||
}
|
||||
|
||||
private int handleGetBlob(HttpServletRequest request,
|
||||
HttpServletResponse response, String containerName,
|
||||
String blobName) {
|
||||
GetOptions options = new GetOptions();
|
||||
String range = request.getHeader(HttpHeaders.RANGE);
|
||||
if (range != null && range.startsWith("bytes=")
|
||||
// ignore multiple ranges
|
||||
&& range.indexOf(',') == -1) {
|
||||
range = range.substring("bytes=".length());
|
||||
String[] ranges = range.split("-", 2);
|
||||
options = options.range(Long.parseLong(ranges[0]),
|
||||
Long.parseLong(ranges[1]));
|
||||
}
|
||||
|
||||
Blob blob = blobStore.getBlob(containerName, blobName, options);
|
||||
if (blob == null) {
|
||||
return HttpServletResponse.SC_NOT_FOUND;
|
||||
}
|
||||
|
||||
response.setStatus(HttpServletResponse.SC_OK);
|
||||
addMetadataToResponse(response, blob.getMetadata());
|
||||
try (InputStream is = blob.getPayload().openStream();
|
||||
OutputStream os = response.getOutputStream()) {
|
||||
ByteStreams.copy(is, os);
|
||||
os.flush();
|
||||
} catch (IOException ioe) {
|
||||
logger.error("Error writing to client: {}", ioe.getMessage());
|
||||
return HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
|
||||
}
|
||||
|
||||
return HttpServletResponse.SC_OK;
|
||||
}
|
||||
|
||||
private int handlePutBlob(HttpServletRequest request,
|
||||
HttpServletResponse response, String containerName,
|
||||
String blobName) throws IOException {
|
||||
ImmutableMap.Builder<String, String> userMetadata =
|
||||
ImmutableMap.builder();
|
||||
Enumeration<String> enumeration = request.getHeaderNames();
|
||||
while (enumeration.hasMoreElements()) {
|
||||
String headerName = enumeration.nextElement();
|
||||
if (headerName.toLowerCase().startsWith(USER_METADATA_PREFIX)) {
|
||||
userMetadata.put(
|
||||
headerName.substring(USER_METADATA_PREFIX.length()),
|
||||
request.getHeader(headerName));
|
||||
}
|
||||
}
|
||||
String contentMD5String = request.getHeader(HttpHeaders.CONTENT_MD5);
|
||||
HashCode contentMD5 = contentMD5String == null ? null
|
||||
: HashCode.fromBytes(BaseEncoding.base64().decode(
|
||||
contentMD5String));
|
||||
try (InputStream is = request.getInputStream()) {
|
||||
BlobBuilder.PayloadBlobBuilder builder = blobStore
|
||||
.blobBuilder(blobName)
|
||||
.userMetadata(userMetadata.build())
|
||||
.payload(is)
|
||||
.contentDisposition(request.getHeader(
|
||||
HttpHeaders.CONTENT_DISPOSITION))
|
||||
.contentEncoding(request.getHeader(
|
||||
HttpHeaders.CONTENT_ENCODING))
|
||||
.contentLanguage(request.getHeader(
|
||||
HttpHeaders.CONTENT_LANGUAGE))
|
||||
.contentLength(request.getContentLength())
|
||||
.contentType(request.getContentType());
|
||||
long expires = request.getDateHeader(HttpHeaders.EXPIRES);
|
||||
if (expires != 0) {
|
||||
builder = builder.expires(new Date(expires));
|
||||
}
|
||||
if (contentMD5 != null) {
|
||||
builder = builder.contentMD5(contentMD5);
|
||||
}
|
||||
try {
|
||||
String eTag = blobStore.putBlob(containerName, builder.build());
|
||||
response.addHeader(HttpHeaders.ETAG, eTag);
|
||||
} catch (HttpResponseException hre) {
|
||||
// TODO: emit hre.getContent() ?
|
||||
return hre.getResponse().getStatusCode();
|
||||
}
|
||||
return HttpServletResponse.SC_OK;
|
||||
} catch (IOException ioe) {
|
||||
logger.error("Error reading from client: {}", ioe.getMessage());
|
||||
return HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
private static void addMetadataToResponse(HttpServletResponse response,
|
||||
BlobMetadata metadata) {
|
||||
ContentMetadata contentMetadata =
|
||||
metadata.getContentMetadata();
|
||||
response.addHeader(HttpHeaders.CONTENT_DISPOSITION,
|
||||
contentMetadata.getContentDisposition());
|
||||
response.addHeader(HttpHeaders.CONTENT_ENCODING,
|
||||
contentMetadata.getContentEncoding());
|
||||
response.addHeader(HttpHeaders.CONTENT_LANGUAGE,
|
||||
contentMetadata.getContentLanguage());
|
||||
response.addHeader(HttpHeaders.CONTENT_LENGTH,
|
||||
contentMetadata.getContentLength().toString());
|
||||
response.setContentType(contentMetadata.getContentType());
|
||||
response.addHeader(HttpHeaders.CONTENT_MD5,
|
||||
BaseEncoding.base64().encode(
|
||||
contentMetadata.getContentMD5AsHashCode().asBytes()));
|
||||
Date expires = contentMetadata.getExpires();
|
||||
if (expires != null) {
|
||||
response.addDateHeader(HttpHeaders.EXPIRES, expires.getTime());
|
||||
}
|
||||
response.addDateHeader(HttpHeaders.LAST_MODIFIED,
|
||||
metadata.getLastModified().getTime());
|
||||
for (Map.Entry<String, String> entry :
|
||||
metadata.getUserMetadata().entrySet()) {
|
||||
response.addHeader(USER_METADATA_PREFIX + entry.getKey(),
|
||||
entry.getValue());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,111 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE module PUBLIC "-//Puppy Crawl//DTD Check Configuration 1.3//EN" "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
||||
|
||||
<module name="Checker">
|
||||
<property name="severity" value="warning"/>
|
||||
<module name="FileLength">
|
||||
<property name="max" value="4000"/>
|
||||
</module>
|
||||
<module name="FileTabCharacter"/>
|
||||
<module name="JavadocPackage">
|
||||
<property name="severity" value="ignore"/>
|
||||
<metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/>
|
||||
</module>
|
||||
<module name="NewlineAtEndOfFile"/>
|
||||
<module name="Translation"/>
|
||||
<module name="TreeWalker">
|
||||
<property name="cacheFile" value="target/cachefile"/>
|
||||
<module name="ArrayTypeStyle"/>
|
||||
<module name="AvoidNestedBlocks"/>
|
||||
<module name="AvoidStarImport"/>
|
||||
<module name="DesignForExtension"/>
|
||||
<module name="EmptyBlock">
|
||||
<property name="option" value="text"/>
|
||||
</module>
|
||||
<module name="EmptyForIteratorPad"/>
|
||||
<module name="EmptyStatement"/>
|
||||
<module name="EqualsHashCode"/>
|
||||
<module name="FinalClass"/>
|
||||
<module name="HiddenField">
|
||||
<property name="ignoreConstructorParameter" value="true"/>
|
||||
<property name="ignoreSetter" value="true"/>
|
||||
</module>
|
||||
<module name="HideUtilityClassConstructor"/>
|
||||
<module name="IllegalImport"/>
|
||||
<module name="IllegalInstantiation"/>
|
||||
<module name="ImportOrder">
|
||||
<property name="groups" value="java,javax,com,org"/>
|
||||
<property name="separated" value="true"/>
|
||||
<property name="option" value="top"/>
|
||||
</module>
|
||||
<module name="InnerAssignment"/>
|
||||
<module name="InterfaceIsType"/>
|
||||
<module name="JavadocStyle"/>
|
||||
<module name="LeftCurly"/>
|
||||
<module name="LineLength">
|
||||
<!-- ignore long string literals -->
|
||||
<property name="ignorePattern" value="^ *".*" \+$"/>
|
||||
</module>
|
||||
<module name="LocalFinalVariableName"/>
|
||||
<module name="LocalVariableName"/>
|
||||
<module name="MagicNumber">
|
||||
<property name="severity" value="ignore"/>
|
||||
<metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/>
|
||||
</module>
|
||||
<module name="MemberName">
|
||||
<property name="applyToPublic" value="false"/>
|
||||
</module>
|
||||
<module name="MethodLength">
|
||||
<property name="max" value="500"/>
|
||||
<property name="countEmpty" value="false"/>
|
||||
</module>
|
||||
<module name="MethodName"/>
|
||||
<module name="MethodParamPad"/>
|
||||
<module name="MissingSwitchDefault"/>
|
||||
<module name="ModifierOrder"/>
|
||||
<module name="MultipleVariableDeclarations"/>
|
||||
<module name="MutableException"/>
|
||||
<module name="NeedBraces"/>
|
||||
<module name="NoWhitespaceAfter"/>
|
||||
<module name="NoWhitespaceBefore"/>
|
||||
<module name="OneStatementPerLine"/>
|
||||
<module name="PackageName"/>
|
||||
<module name="ParameterName"/>
|
||||
<module name="ParameterNumber">
|
||||
<property name="max" value="12"/>
|
||||
</module>
|
||||
<module name="ParenPad"/>
|
||||
<module name="RedundantImport"/>
|
||||
<module name="RedundantModifier"/>
|
||||
<module name="RedundantThrows">
|
||||
<property name="suppressLoadErrors" value="true"/>
|
||||
</module>
|
||||
<module name="RightCurly"/>
|
||||
<module name="SimplifyBooleanExpression"/>
|
||||
<module name="SimplifyBooleanReturn"/>
|
||||
<module name="StaticVariableName"/>
|
||||
<module name="TodoComment">
|
||||
<property name="severity" value="ignore"/>
|
||||
<metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/>
|
||||
</module>
|
||||
<module name="TypecastParenPad"/>
|
||||
<module name="TypeName"/>
|
||||
<module name="UnusedImports"/>
|
||||
<module name="UpperEll"/>
|
||||
<module name="VisibilityModifier">
|
||||
<property name="protectedAllowed" value="true"/>
|
||||
<property name="publicMemberPattern"
|
||||
value="(^thrown$|^[A-Z][A-Z0-9]*(_[A-Z0-9]+)*$)"/>
|
||||
</module>
|
||||
<module name="WhitespaceAfter"/>
|
||||
<module name="WhitespaceAround"/>
|
||||
</module>
|
||||
<module name="Header">
|
||||
<property name="fileExtensions" value="java"/>
|
||||
<property name="headerFile" value="${checkstyle.header.file}"/>
|
||||
</module>
|
||||
<module name="RegexpSingleline">
|
||||
<property name="format" value="\s+$"/>
|
||||
<property name="message" value="Line has trailing spaces."/>
|
||||
</module>
|
||||
</module>
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
/*
|
||||
* Copyright 2014 Andrew Gaul <andrew@gaul.org>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
<configuration>
|
||||
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
|
||||
<encoder>
|
||||
<pattern>%.-1p %d{MM-dd HH:mm:ss.SSS} %t %c{30}:%L %X{clientId}|%X{sessionId}:%X{messageId}:%X{fileId}] %m%n</pattern>
|
||||
</encoder>
|
||||
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
|
||||
<level>${LOG_LEVEL:-info}</level>
|
||||
</filter>
|
||||
</appender>
|
||||
|
||||
<logger name="org.eclipse.jetty" level="${JETTY_LOG_LEVEL:-info}" />
|
||||
|
||||
<root level="${LOG_LEVEL:-info}">
|
||||
<appender-ref ref="STDOUT" />
|
||||
</root>
|
||||
</configuration>
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
jclouds.provider=transient
|
||||
jclouds.identity=identity
|
||||
jclouds.credential=credential
|
||||
#jclouds.endpoint=http://127.0.0.1:8081
|
||||
jclouds.filesystem.basedir=/tmp/blobstore
|
||||
s3proxy.endpoint=http://127.0.0.1:8080
|
||||
#s3proxy.loglevel=INFO # TODO: not yet supported
|
||||
|
|
@ -0,0 +1,246 @@
|
|||
/*
|
||||
* Copyright 2014 Andrew Gaul <andrew@gaul.org>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.gaul.s3proxy;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.net.URI;
|
||||
import java.util.Properties;
|
||||
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.io.ByteSource;
|
||||
|
||||
import org.jclouds.ContextBuilder;
|
||||
import org.jclouds.blobstore.BlobStore;
|
||||
import org.jclouds.blobstore.BlobStoreContext;
|
||||
import org.jclouds.blobstore.domain.Blob;
|
||||
import org.jclouds.blobstore.domain.BlobMetadata;
|
||||
import org.jclouds.blobstore.domain.StorageMetadata;
|
||||
import org.jclouds.blobstore.options.ListContainerOptions;
|
||||
import org.jclouds.io.Payload;
|
||||
import org.jclouds.io.payloads.ByteSourcePayload;
|
||||
import org.jclouds.rest.HttpClient;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
|
||||
public final class S3ProxyTest {
|
||||
// TODO: configurable
|
||||
private static final URI s3Endpoint = URI.create("http://127.0.0.1:8080");
|
||||
private S3Proxy s3Proxy;
|
||||
private BlobStoreContext context;
|
||||
private BlobStoreContext s3Context;
|
||||
private BlobStore s3BlobStore;
|
||||
private static final String containerName = "container";
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
Properties properties = new Properties();
|
||||
context = ContextBuilder
|
||||
.newBuilder("transient")
|
||||
.credentials("identity", "credential")
|
||||
.overrides(properties)
|
||||
.build(BlobStoreContext.class);
|
||||
BlobStore blobStore = context.getBlobStore();
|
||||
blobStore.createContainerInLocation(null, containerName);
|
||||
s3Context = ContextBuilder
|
||||
.newBuilder("s3")
|
||||
.credentials("identity", "credential")
|
||||
.endpoint(s3Endpoint.toString())
|
||||
.build(BlobStoreContext.class);
|
||||
s3BlobStore = s3Context.getBlobStore();
|
||||
s3Proxy = new S3Proxy(blobStore, s3Endpoint);
|
||||
s3Proxy.start();
|
||||
}
|
||||
|
||||
@After
|
||||
public void tearDown() throws Exception {
|
||||
if (s3Proxy != null) {
|
||||
s3Proxy.stop();
|
||||
}
|
||||
if (s3Context != null) {
|
||||
s3Context.close();
|
||||
}
|
||||
if (context != null) {
|
||||
context.close();
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: why does this hang for 30 seconds?
|
||||
@Ignore
|
||||
@Test
|
||||
public void testHttpClient() throws Exception {
|
||||
HttpClient httpClient = context.utils().http();
|
||||
// TODO: how to interpret this?
|
||||
URI uri = URI.create(s3Endpoint + "/container/blob");
|
||||
ByteSource byteSource = ByteSource.wrap(new byte[1]);
|
||||
Payload payload = new ByteSourcePayload(byteSource);
|
||||
payload.getContentMetadata().setContentLength(byteSource.size());
|
||||
httpClient.put(uri, payload);
|
||||
try (InputStream actual = httpClient.get(uri);
|
||||
InputStream expected = byteSource.openStream()) {
|
||||
assertThat(actual).hasContentEqualTo(expected);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testJcloudsClient() throws Exception {
|
||||
ImmutableSet.Builder<String> builder = ImmutableSet.builder();
|
||||
for (StorageMetadata metadata : s3BlobStore.list()) {
|
||||
builder.add(metadata.getName());
|
||||
}
|
||||
assertThat(builder.build()).containsOnly(containerName);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testContainerExists() throws Exception {
|
||||
assertThat(s3BlobStore.containerExists("fakecontainer")).isFalse();
|
||||
assertThat(s3BlobStore.containerExists(containerName)).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testContainerCreate() throws Exception {
|
||||
assertThat(s3BlobStore.createContainerInLocation(null,
|
||||
"newcontainer")).isTrue();
|
||||
assertThat(s3BlobStore.createContainerInLocation(null,
|
||||
"newcontainer")).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testContainerDelete() throws Exception {
|
||||
assertThat(s3BlobStore.containerExists(containerName)).isTrue();
|
||||
s3BlobStore.deleteContainerIfEmpty(containerName);
|
||||
assertThat(s3BlobStore.containerExists(containerName)).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBlobPutGet() throws Exception {
|
||||
String blobName = "blob";
|
||||
ByteSource byteSource = ByteSource.wrap(new byte[42]);
|
||||
Blob blob = s3BlobStore.blobBuilder(blobName)
|
||||
.payload(byteSource)
|
||||
.contentLength(byteSource.size())
|
||||
.build();
|
||||
s3BlobStore.putBlob(containerName, blob);
|
||||
|
||||
Blob blob2 = s3BlobStore.getBlob(containerName, blobName);
|
||||
try (InputStream actual = blob2.getPayload().openStream();
|
||||
InputStream expected = byteSource.openStream()) {
|
||||
assertThat(actual).hasContentEqualTo(expected);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBlobList() throws Exception {
|
||||
assertThat(s3BlobStore.list(containerName)).isEmpty();
|
||||
|
||||
// TODO: hang with zero length blobs?
|
||||
ByteSource byteSource = ByteSource.wrap(new byte[1]);
|
||||
ImmutableSet.Builder<String> builder = ImmutableSet.builder();
|
||||
Blob blob1 = s3BlobStore.blobBuilder("blob1")
|
||||
.payload(byteSource)
|
||||
.contentLength(byteSource.size())
|
||||
.build();
|
||||
s3BlobStore.putBlob(containerName, blob1);
|
||||
for (StorageMetadata metadata : s3BlobStore.list(containerName)) {
|
||||
builder.add(metadata.getName());
|
||||
}
|
||||
assertThat(builder.build()).containsOnly("blob1");
|
||||
|
||||
builder = ImmutableSet.builder();
|
||||
Blob blob2 = s3BlobStore.blobBuilder("blob2")
|
||||
.payload(byteSource)
|
||||
.contentLength(byteSource.size())
|
||||
.build();
|
||||
s3BlobStore.putBlob(containerName, blob2);
|
||||
for (StorageMetadata metadata : s3BlobStore.list(containerName)) {
|
||||
builder.add(metadata.getName());
|
||||
}
|
||||
assertThat(builder.build()).containsOnly("blob1", "blob2");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBlobListRecursive() throws Exception {
|
||||
assertThat(s3BlobStore.list(containerName)).isEmpty();
|
||||
|
||||
ByteSource byteSource = ByteSource.wrap(new byte[1]);
|
||||
Blob blob1 = s3BlobStore.blobBuilder("prefix/blob1")
|
||||
.payload(byteSource)
|
||||
.contentLength(byteSource.size())
|
||||
.build();
|
||||
s3BlobStore.putBlob(containerName, blob1);
|
||||
|
||||
Blob blob2 = s3BlobStore.blobBuilder("prefix/blob2")
|
||||
.payload(byteSource)
|
||||
.contentLength(byteSource.size())
|
||||
.build();
|
||||
s3BlobStore.putBlob(containerName, blob2);
|
||||
|
||||
ImmutableSet.Builder<String> builder = ImmutableSet.builder();
|
||||
for (StorageMetadata metadata : s3BlobStore.list(containerName)) {
|
||||
builder.add(metadata.getName());
|
||||
}
|
||||
assertThat(builder.build()).containsOnly("prefix");
|
||||
|
||||
builder = ImmutableSet.builder();
|
||||
for (StorageMetadata metadata : s3BlobStore.list(containerName,
|
||||
new ListContainerOptions().recursive())) {
|
||||
builder.add(metadata.getName());
|
||||
}
|
||||
assertThat(builder.build()).containsOnly("prefix/blob1",
|
||||
"prefix/blob2");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBlobMetadata() throws Exception {
|
||||
String blobName = "blob";
|
||||
ByteSource byteSource = ByteSource.wrap(new byte[1]);
|
||||
Blob blob1 = s3BlobStore.blobBuilder(blobName)
|
||||
.payload(byteSource)
|
||||
.contentLength(byteSource.size())
|
||||
.build();
|
||||
s3BlobStore.putBlob(containerName, blob1);
|
||||
|
||||
BlobMetadata metadata = s3BlobStore.blobMetadata(containerName,
|
||||
blobName);
|
||||
assertThat(metadata.getName()).isEqualTo(blobName);
|
||||
assertThat(metadata.getContentMetadata().getContentLength())
|
||||
.isEqualTo(byteSource.size());
|
||||
|
||||
assertThat(s3BlobStore.blobMetadata(containerName,
|
||||
"fake-blob")).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBlobRemove() throws Exception {
|
||||
String blobName = "blob";
|
||||
ByteSource byteSource = ByteSource.wrap(new byte[1]);
|
||||
Blob blob = s3BlobStore.blobBuilder(blobName)
|
||||
.payload(byteSource)
|
||||
.contentLength(byteSource.size())
|
||||
.build();
|
||||
s3BlobStore.putBlob(containerName, blob);
|
||||
assertThat(s3BlobStore.blobExists(containerName, blobName)).isTrue();
|
||||
|
||||
s3BlobStore.removeBlob(containerName, blobName);
|
||||
assertThat(s3BlobStore.blobExists(containerName, blobName)).isFalse();
|
||||
|
||||
s3BlobStore.removeBlob(containerName, blobName);
|
||||
}
|
||||
}
|
||||
Ładowanie…
Reference in New Issue