kopia lustrzana https://github.com/gaul/s3proxy
Address Checkstyle LineLength violations
rodzic
a732dca4c2
commit
4aeff5fb28
|
@ -248,8 +248,8 @@ public final class Main {
|
||||||
blobStore = AliasBlobStore.newAliasBlobStore(blobStore, aliases);
|
blobStore = AliasBlobStore.newAliasBlobStore(blobStore, aliases);
|
||||||
}
|
}
|
||||||
|
|
||||||
ImmutableList<Map.Entry<Pattern, String>> regexs = RegexBlobStore.parseRegexs(
|
ImmutableList<Map.Entry<Pattern, String>> regexs =
|
||||||
properties);
|
RegexBlobStore.parseRegexs(properties);
|
||||||
if (!regexs.isEmpty()) {
|
if (!regexs.isEmpty()) {
|
||||||
System.err.println("Using regex backend");
|
System.err.println("Using regex backend");
|
||||||
blobStore = RegexBlobStore.newRegexBlobStore(blobStore, regexs);
|
blobStore = RegexBlobStore.newRegexBlobStore(blobStore, regexs);
|
||||||
|
|
|
@ -56,27 +56,31 @@ import org.slf4j.LoggerFactory;
|
||||||
* stopping as soon as the first regex matches.
|
* stopping as soon as the first regex matches.
|
||||||
*/
|
*/
|
||||||
public final class RegexBlobStore extends ForwardingBlobStore {
|
public final class RegexBlobStore extends ForwardingBlobStore {
|
||||||
private static final Logger logger = LoggerFactory.getLogger(RegexBlobStore.class);
|
private static final Logger logger = LoggerFactory.getLogger(
|
||||||
|
RegexBlobStore.class);
|
||||||
|
|
||||||
private final ImmutableList<Entry<Pattern, String>> regexs;
|
private final ImmutableList<Entry<Pattern, String>> regexs;
|
||||||
|
|
||||||
private RegexBlobStore(BlobStore blobStore, ImmutableList<Entry<Pattern, String>> regexs) {
|
private RegexBlobStore(BlobStore blobStore,
|
||||||
|
ImmutableList<Entry<Pattern, String>> regexs) {
|
||||||
super(blobStore);
|
super(blobStore);
|
||||||
this.regexs = requireNonNull(regexs);
|
this.regexs = requireNonNull(regexs);
|
||||||
}
|
}
|
||||||
|
|
||||||
static BlobStore newRegexBlobStore(BlobStore delegate, ImmutableList<Entry<Pattern, String>> regexs) {
|
static BlobStore newRegexBlobStore(BlobStore delegate,
|
||||||
|
ImmutableList<Entry<Pattern, String>> regexs) {
|
||||||
return new RegexBlobStore(delegate, regexs);
|
return new RegexBlobStore(delegate, regexs);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static ImmutableList<Map.Entry<Pattern, String>> parseRegexs(Properties properties) {
|
public static ImmutableList<Map.Entry<Pattern, String>> parseRegexs(
|
||||||
|
Properties properties) {
|
||||||
List<Entry<String, String>> configRegex = new ArrayList<>();
|
List<Entry<String, String>> configRegex = new ArrayList<>();
|
||||||
List<Entry<Pattern, String>> regexs = new ArrayList<>();
|
List<Entry<Pattern, String>> regexs = new ArrayList<>();
|
||||||
|
|
||||||
for (String key : properties.stringPropertyNames()) {
|
for (String key : properties.stringPropertyNames()) {
|
||||||
if (key.startsWith(S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE)) {
|
if (key.startsWith(S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE)) {
|
||||||
String propKey = key.substring(S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE.length() + 1);
|
String propKey = key.substring(
|
||||||
|
S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE.length() + 1);
|
||||||
String value = properties.getProperty(key);
|
String value = properties.getProperty(key);
|
||||||
|
|
||||||
configRegex.add(new SimpleEntry<>(propKey, value));
|
configRegex.add(new SimpleEntry<>(propKey, value));
|
||||||
|
@ -85,24 +89,26 @@ public final class RegexBlobStore extends ForwardingBlobStore {
|
||||||
|
|
||||||
for (Entry<String, String> entry : configRegex) {
|
for (Entry<String, String> entry : configRegex) {
|
||||||
String key = entry.getKey();
|
String key = entry.getKey();
|
||||||
if (key.startsWith(S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE_MATCH)) {
|
if (key.startsWith(
|
||||||
String regexName = key.substring(S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE_MATCH.length() + 1);
|
S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE_MATCH)) {
|
||||||
|
String regexName = key.substring(S3ProxyConstants
|
||||||
|
.PROPERTY_REGEX_BLOBSTORE_MATCH.length() + 1);
|
||||||
String regex = entry.getValue();
|
String regex = entry.getValue();
|
||||||
Pattern pattern = Pattern.compile(regex);
|
Pattern pattern = Pattern.compile(regex);
|
||||||
|
|
||||||
String replace = properties.getProperty(
|
String replace = properties.getProperty(String.join(
|
||||||
String.join(
|
".", S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE,
|
||||||
".",
|
S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE_REPLACE,
|
||||||
S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE,
|
regexName));
|
||||||
S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE_REPLACE,
|
|
||||||
regexName));
|
|
||||||
|
|
||||||
checkArgument(
|
checkArgument(
|
||||||
replace != null,
|
replace != null,
|
||||||
"Regex %s has no replace property associated",
|
"Regex %s has no replace property associated",
|
||||||
regexName);
|
regexName);
|
||||||
|
|
||||||
logger.info("Adding new regex with name {} replaces with {} to {}", regexName, regex, replace);
|
logger.info(
|
||||||
|
"Adding new regex with name {} replaces with {} to {}",
|
||||||
|
regexName, regex, replace);
|
||||||
|
|
||||||
regexs.add(new SimpleEntry<>(pattern, replace));
|
regexs.add(new SimpleEntry<>(pattern, replace));
|
||||||
}
|
}
|
||||||
|
@ -143,7 +149,8 @@ public final class RegexBlobStore extends ForwardingBlobStore {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String putBlob(String containerName, Blob blob, PutOptions putOptions) {
|
public String putBlob(String containerName, Blob blob,
|
||||||
|
PutOptions putOptions) {
|
||||||
String name = blob.getMetadata().getName();
|
String name = blob.getMetadata().getName();
|
||||||
String newName = replaceBlobName(name);
|
String newName = replaceBlobName(name);
|
||||||
blob.getMetadata().setName(newName);
|
blob.getMetadata().setName(newName);
|
||||||
|
@ -154,9 +161,10 @@ public final class RegexBlobStore extends ForwardingBlobStore {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String copyBlob(String fromContainer, String fromName, String toContainer, String toName,
|
public String copyBlob(String fromContainer, String fromName,
|
||||||
CopyOptions options) {
|
String toContainer, String toName, CopyOptions options) {
|
||||||
return super.copyBlob(fromContainer, replaceBlobName(fromName), toContainer, replaceBlobName(toName), options);
|
return super.copyBlob(fromContainer, replaceBlobName(fromName),
|
||||||
|
toContainer, replaceBlobName(toName), options);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -189,7 +197,8 @@ public final class RegexBlobStore extends ForwardingBlobStore {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void setBlobAccess(String container, String name, BlobAccess access) {
|
public void setBlobAccess(String container, String name,
|
||||||
|
BlobAccess access) {
|
||||||
super.setBlobAccess(container, replaceBlobName(name), access);
|
super.setBlobAccess(container, replaceBlobName(name), access);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -199,8 +208,10 @@ public final class RegexBlobStore extends ForwardingBlobStore {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void downloadBlob(String container, String name, File destination, ExecutorService executor) {
|
public void downloadBlob(String container, String name, File destination,
|
||||||
super.downloadBlob(container, replaceBlobName(name), destination, executor);
|
ExecutorService executor) {
|
||||||
|
super.downloadBlob(container, replaceBlobName(name), destination,
|
||||||
|
executor);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -209,7 +220,8 @@ public final class RegexBlobStore extends ForwardingBlobStore {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public InputStream streamBlob(String container, String name, ExecutorService executor) {
|
public InputStream streamBlob(String container, String name,
|
||||||
|
ExecutorService executor) {
|
||||||
return super.streamBlob(container, replaceBlobName(name), executor);
|
return super.streamBlob(container, replaceBlobName(name), executor);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -667,7 +667,8 @@ public class S3ProxyHandler {
|
||||||
path[1], path[2], uploadId);
|
path[1], path[2], uploadId);
|
||||||
return;
|
return;
|
||||||
} else {
|
} else {
|
||||||
handleBlobRemove(request, response, blobStore, path[1], path[2]);
|
handleBlobRemove(request, response, blobStore, path[1],
|
||||||
|
path[2]);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
case "GET":
|
case "GET":
|
||||||
|
@ -676,7 +677,8 @@ public class S3ProxyHandler {
|
||||||
return;
|
return;
|
||||||
} else if (path.length <= 2 || path[2].isEmpty()) {
|
} else if (path.length <= 2 || path[2].isEmpty()) {
|
||||||
if (request.getParameter("acl") != null) {
|
if (request.getParameter("acl") != null) {
|
||||||
handleGetContainerAcl(request, response, blobStore, path[1]);
|
handleGetContainerAcl(request, response, blobStore,
|
||||||
|
path[1]);
|
||||||
return;
|
return;
|
||||||
} else if (request.getParameter("location") != null) {
|
} else if (request.getParameter("location") != null) {
|
||||||
handleContainerLocation(request, response);
|
handleContainerLocation(request, response);
|
||||||
|
@ -716,7 +718,8 @@ public class S3ProxyHandler {
|
||||||
}
|
}
|
||||||
case "POST":
|
case "POST":
|
||||||
if (request.getParameter("delete") != null) {
|
if (request.getParameter("delete") != null) {
|
||||||
handleMultiBlobRemove(request, response, is, blobStore, path[1]);
|
handleMultiBlobRemove(request, response, is, blobStore,
|
||||||
|
path[1]);
|
||||||
return;
|
return;
|
||||||
} else if (request.getParameter("uploads") != null) {
|
} else if (request.getParameter("uploads") != null) {
|
||||||
handleInitiateMultipartUpload(request, response, blobStore,
|
handleInitiateMultipartUpload(request, response, blobStore,
|
||||||
|
@ -1101,7 +1104,8 @@ public class S3ProxyHandler {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void handleContainerList(HttpServletRequest request,
|
private void handleContainerList(HttpServletRequest request,
|
||||||
HttpServletResponse response, BlobStore blobStore) throws IOException {
|
HttpServletResponse response, BlobStore blobStore)
|
||||||
|
throws IOException {
|
||||||
PageSet<? extends StorageMetadata> buckets = blobStore.list();
|
PageSet<? extends StorageMetadata> buckets = blobStore.list();
|
||||||
|
|
||||||
response.setCharacterEncoding(UTF_8);
|
response.setCharacterEncoding(UTF_8);
|
||||||
|
@ -1490,10 +1494,13 @@ public class S3ProxyHandler {
|
||||||
isListV2 ? "NextContinuationToken" : "NextMarker",
|
isListV2 ? "NextContinuationToken" : "NextMarker",
|
||||||
encodeBlob(encodingType, nextMarker));
|
encodeBlob(encodingType, nextMarker));
|
||||||
if (Quirks.OPAQUE_MARKERS.contains(blobStoreType)) {
|
if (Quirks.OPAQUE_MARKERS.contains(blobStoreType)) {
|
||||||
StorageMetadata sm = Streams.findLast(set.stream()).orElse(null);
|
StorageMetadata sm = Streams.findLast(
|
||||||
|
set.stream()).orElse(null);
|
||||||
if (sm != null) {
|
if (sm != null) {
|
||||||
lastKeyToMarker.put(Maps.immutableEntry(containerName,
|
lastKeyToMarker.put(Maps.immutableEntry(
|
||||||
encodeBlob(encodingType, nextMarker)), nextMarker);
|
containerName,
|
||||||
|
encodeBlob(encodingType, nextMarker)),
|
||||||
|
nextMarker);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
@ -1538,7 +1545,8 @@ public class S3ProxyHandler {
|
||||||
|
|
||||||
Tier tier = metadata.getTier();
|
Tier tier = metadata.getTier();
|
||||||
if (tier != null) {
|
if (tier != null) {
|
||||||
writeSimpleElement(xml, "StorageClass", StorageClass.fromTier(tier).toString());
|
writeSimpleElement(xml, "StorageClass",
|
||||||
|
StorageClass.fromTier(tier).toString());
|
||||||
}
|
}
|
||||||
|
|
||||||
if (fetchOwner) {
|
if (fetchOwner) {
|
||||||
|
@ -3007,7 +3015,8 @@ public class S3ProxyHandler {
|
||||||
response.addHeader(HttpHeaders.ACCESS_CONTROL_ALLOW_METHODS,
|
response.addHeader(HttpHeaders.ACCESS_CONTROL_ALLOW_METHODS,
|
||||||
corsRules.getAllowedMethods());
|
corsRules.getAllowedMethods());
|
||||||
if (corsRules.isAllowCredentials()) {
|
if (corsRules.isAllowCredentials()) {
|
||||||
response.addHeader(HttpHeaders.ACCESS_CONTROL_ALLOW_CREDENTIALS, "true");
|
response.addHeader(HttpHeaders.ACCESS_CONTROL_ALLOW_CREDENTIALS,
|
||||||
|
"true");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -116,9 +116,11 @@ public class S3ProxyJunitCore {
|
||||||
builder.blobStoreProvider)
|
builder.blobStoreProvider)
|
||||||
.overrides(properties);
|
.overrides(properties);
|
||||||
if (!AuthenticationType.NONE.equals(builder.authType)) {
|
if (!AuthenticationType.NONE.equals(builder.authType)) {
|
||||||
blobStoreContextBuilder = blobStoreContextBuilder.credentials(accessKey, secretKey);
|
blobStoreContextBuilder = blobStoreContextBuilder.credentials(
|
||||||
|
accessKey, secretKey);
|
||||||
}
|
}
|
||||||
blobStoreContext = blobStoreContextBuilder.build(BlobStoreContext.class);
|
blobStoreContext = blobStoreContextBuilder.build(
|
||||||
|
BlobStoreContext.class);
|
||||||
|
|
||||||
S3Proxy.Builder s3ProxyBuilder = S3Proxy.builder()
|
S3Proxy.Builder s3ProxyBuilder = S3Proxy.builder()
|
||||||
.blobStore(blobStoreContext.getBlobStore())
|
.blobStore(blobStoreContext.getBlobStore())
|
||||||
|
|
|
@ -71,16 +71,20 @@ public final class RegexBlobStoreTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testRemoveSomeCharsFromName() throws IOException {
|
public void testRemoveSomeCharsFromName() throws IOException {
|
||||||
ImmutableList.Builder<Map.Entry<Pattern, String>> regexBuilder = new ImmutableList.Builder<>();
|
ImmutableList.Builder<Map.Entry<Pattern, String>> regexBuilder =
|
||||||
regexBuilder.add(new SimpleEntry<Pattern, String>(Pattern.compile("[^a-zA-Z0-9/_.]"), "_"));
|
new ImmutableList.Builder<>();
|
||||||
BlobStore regexBlobStore = RegexBlobStore.newRegexBlobStore(delegate, regexBuilder.build());
|
regexBuilder.add(new SimpleEntry<Pattern, String>(Pattern.compile(
|
||||||
|
"[^a-zA-Z0-9/_.]"), "_"));
|
||||||
|
BlobStore regexBlobStore = RegexBlobStore.newRegexBlobStore(delegate,
|
||||||
|
regexBuilder.build());
|
||||||
|
|
||||||
String initialBlobName = "test/remove:badchars-folder/blob.txt";
|
String initialBlobName = "test/remove:badchars-folder/blob.txt";
|
||||||
String targetBlobName = "test/remove_badchars_folder/blob.txt";
|
String targetBlobName = "test/remove_badchars_folder/blob.txt";
|
||||||
ByteSource content = TestUtils.randomByteSource().slice(0, 1024);
|
ByteSource content = TestUtils.randomByteSource().slice(0, 1024);
|
||||||
@SuppressWarnings("deprecation")
|
@SuppressWarnings("deprecation")
|
||||||
String contentHash = Hashing.md5().hashBytes(content.read()).toString();
|
String contentHash = Hashing.md5().hashBytes(content.read()).toString();
|
||||||
Blob blob = regexBlobStore.blobBuilder(initialBlobName).payload(content).build();
|
Blob blob = regexBlobStore.blobBuilder(initialBlobName).payload(
|
||||||
|
content).build();
|
||||||
|
|
||||||
String eTag = regexBlobStore.putBlob(containerName, blob);
|
String eTag = regexBlobStore.putBlob(containerName, blob);
|
||||||
assertThat(eTag).isEqualTo(contentHash);
|
assertThat(eTag).isEqualTo(contentHash);
|
||||||
|
@ -106,15 +110,18 @@ public final class RegexBlobStoreTest {
|
||||||
public void testParseMatchWithoutReplace() {
|
public void testParseMatchWithoutReplace() {
|
||||||
Properties properties = new Properties();
|
Properties properties = new Properties();
|
||||||
properties.put(
|
properties.put(
|
||||||
String.format("%s.%s.sample1", S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE,
|
String.format("%s.%s.sample1",
|
||||||
|
S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE,
|
||||||
S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE_MATCH),
|
S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE_MATCH),
|
||||||
"test");
|
"test");
|
||||||
properties.put(
|
properties.put(
|
||||||
String.format("%s.%s.sample2", S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE,
|
String.format("%s.%s.sample2",
|
||||||
|
S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE,
|
||||||
S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE_MATCH),
|
S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE_MATCH),
|
||||||
"test");
|
"test");
|
||||||
properties.put(
|
properties.put(
|
||||||
String.format("%s.%s.sample1", S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE,
|
String.format("%s.%s.sample1",
|
||||||
|
S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE,
|
||||||
S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE_REPLACE),
|
S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE_REPLACE),
|
||||||
"test");
|
"test");
|
||||||
|
|
||||||
|
|
Ładowanie…
Reference in New Issue