argument parsing improvements

pull/1/head
Mike Barry 2021-08-18 20:28:17 -04:00
rodzic 233bbe3e34
commit 3feff93980
11 zmienionych plików z 378 dodań i 53 usunięć

Wyświetl plik

@ -54,10 +54,6 @@ public class FlatMapRunner {
tmpDir = arguments.file("tmpdir", "temp directory", Path.of("data", "tmp"));
}
public static FlatMapRunner create() {
return createWithArguments(Arguments.fromJvmProperties());
}
public static FlatMapRunner createWithArguments(Arguments arguments) {
return new FlatMapRunner(arguments);
}

Wyświetl plik

@ -1,13 +1,19 @@
package com.onthegomap.flatmap.config;
import com.onthegomap.flatmap.geo.GeoUtils;
import com.onthegomap.flatmap.reader.osm.OsmInputFile;
import com.onthegomap.flatmap.stats.Stats;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.time.format.DateTimeParseException;
import java.time.temporal.ChronoUnit;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.TreeMap;
import java.util.function.Function;
import java.util.stream.Stream;
@ -15,28 +21,118 @@ import org.locationtech.jts.geom.Envelope;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Lightweight abstraction over ways to provide key/value pair arguments to a program like jvm properties, environmental
* variables, or a config file.
*/
public class Arguments {
private static final Logger LOGGER = LoggerFactory.getLogger(Arguments.class);
private final List<Function<String, String>> providers;
private final Function<String, String> provider;
private Arguments(List<Function<String, String>> providers) {
this.providers = providers;
private Arguments(Function<String, String> provider) {
this.provider = provider;
}
/**
* Parses arguments from JVM system properties prefixed with {@code flatmap.}
* <p>
* For example to set {@code key=value}: {@code java -Dflatmap.key=value -jar ...}
*
* @return arguments parsed from JVM system properties
*/
public static Arguments fromJvmProperties() {
return new Arguments(List.of(System::getProperty));
return new Arguments(key -> System.getProperty("flatmap." + key));
}
public static Arguments empty() {
return new Arguments(List.of());
/**
* Parses arguments from environmental variables prefixed with {@code FLATMAP_}
* <p>
* For example to set {@code key=value}: {@code FLATMAP_KEY=value java -jar ...}
*
* @return arguments parsed from environmental variables
*/
public static Arguments fromEnvironment() {
return new Arguments(key -> System.getenv("FLATMAP_" + key.toUpperCase(Locale.ROOT)));
}
/**
* Parses command-line arguments.
* <p>
* For example to set {@code key=value}: {@code java -jar ... key=value}
*
* @param args arguments provided to main method
* @return arguments parsed from command-line arguments
*/
public static Arguments fromArgs(String... args) {
Map<String, String> parsed = new HashMap<>();
for (String arg : args) {
String[] kv = arg.split("=", 2);
if (kv.length == 2) {
String key = kv[0].replaceAll("^[\\s-]+", "");
String value = kv[1];
parsed.put(key, value);
}
}
return of(parsed);
}
/**
* Parses arguments from a properties file.
*
* @param path path to the properties file
* @return arguments parsed from a properties file
* @see <a href="https://en.wikipedia.org/wiki/.properties">.properties format explanation</a>
*/
public static Arguments fromConfigFile(Path path) {
Properties properties = new Properties();
try (var reader = Files.newBufferedReader(path)) {
properties.load(reader);
return new Arguments(properties::getProperty);
} catch (IOException e) {
throw new IllegalArgumentException("Unable to load config file: " + path, e);
}
}
/**
* Look for arguments in the following priority order:
* <ol>
* <li>command-line arguments: {@code java ... key=value}</li>
* <li>jvm properties: {@code java -Dflatmap.key=value ...}</li>
* <li>environmental variables: {@code FLATMAP_KEY=value java ...}</li>
* <li>in a config file from "config" argument from any of the above</li>
* </ol>
*
* @param args command-line args provide to main entrypoint method
* @return arguments parsed from those sources
*/
public static Arguments fromArgsOrConfigFile(String... args) {
Arguments fromArgsOrEnv = fromArgs(args)
.orElse(fromJvmProperties())
.orElse(fromEnvironment());
Path configFile = fromArgsOrEnv.file("config", "path to config file", null);
if (configFile != null) {
return fromArgsOrEnv.orElse(fromConfigFile(configFile));
} else {
return fromArgsOrEnv;
}
}
/**
* @param map map that provides the key/value pairs
* @return arguments provided by map
*/
public static Arguments of(Map<String, String> map) {
return new Arguments(List.of(map::get));
return new Arguments(map::get);
}
/**
* Shorthand for {@link #of(Map)} which constructs the map from a list of key/value pairs
*
* @param args list of key/value pairs
* @return arguments provided by that list of key/value pairs
*/
public static Arguments of(Object... args) {
Map<String, String> map = new TreeMap<>();
for (int i = 0; i < args.length; i += 2) {
@ -45,21 +141,42 @@ public class Arguments {
return of(map);
}
/**
* Chain two argument providers so that {@code other} is used as a fallback to {@code this}.
*
* @param other another arguments provider
* @return arguments instance that checks {@code this} first and if a match is not found then {@code other}
*/
public Arguments orElse(Arguments other) {
return new Arguments(key -> {
String ourResult = provider.apply(key);
return ourResult != null ? ourResult : other.provider.apply(key);
});
}
private String getArg(String key, String defaultValue) {
String value = getArg(key);
return value == null ? defaultValue : value;
}
private String getArg(String key) {
String value = null;
for (int i = 0; i < providers.size() && value == null; i++) {
value = providers.get(i).apply(key);
}
String value = provider.apply(key);
return value == null ? null : value.trim();
}
public Envelope bounds(String arg, String description, BoundsProvider defaultBounds) {
String input = getArg(arg);
/**
* Parse an argument as {@link Envelope}, or use bounds from a {@link BoundsProvider} instead.
* <p>
* Format: {@code westLng,southLat,eastLng,northLat}
*
* @param key argument name
* @param description argument description
* @param defaultBounds fallback provider if argument missing (ie. an {@link OsmInputFile} that contains bounds in
* it's metadata)
* @return An envelope parsed from {@code key} or provided by {@code defaultBounds}
*/
public Envelope bounds(String key, String description, BoundsProvider defaultBounds) {
String input = getArg(key);
Envelope result;
if (input == null && defaultBounds != null) {
// get from input file
@ -73,49 +190,99 @@ public class Arguments {
}
result = new Envelope(bounds[0], bounds[2], bounds[1], bounds[3]);
}
logArgValue(arg, description, result);
logArgValue(key, description, result);
return result;
}
private void logArgValue(String arg, String description, Object result) {
LOGGER.debug(" -D" + arg + "=" + result + " (" + description + ")");
private void logArgValue(String key, String description, Object result) {
LOGGER.debug("argument: " + key + "=" + result + " (" + description + ")");
}
public String get(String arg, String description, String defaultValue) {
String value = getArg(arg, defaultValue);
logArgValue(arg, description, value);
/**
* Return an argument as a string.
*
* @param key argument name
* @param description argument description
* @param defaultValue fallback value if missing
* @return the value for {@code key} otherwise {@code defaultValue}
*/
public String get(String key, String description, String defaultValue) {
String value = getArg(key, defaultValue);
logArgValue(key, description, value);
return value;
}
public Path file(String arg, String description, Path defaultValue) {
String value = getArg(arg);
/**
* Parse an argument as a {@link Path} to a file.
*
* @param key argument name
* @param description argument description
* @param defaultValue fallback path if missing
* @return a path parsed from {@code key} otherwise {@code defaultValue}
*/
public Path file(String key, String description, Path defaultValue) {
String value = getArg(key);
Path file = value == null ? defaultValue : Path.of(value);
logArgValue(arg, description, file);
logArgValue(key, description, file);
return file;
}
public Path inputFile(String arg, String description, Path defaultValue) {
Path path = file(arg, description, defaultValue);
/**
* Parse an argument as a {@link Path} to a file that must exist for the program to work.
*
* @param key argument name
* @param description argument description
* @param defaultValue fallback path if missing
* @return a path parsed from {@code key} otherwise {@code defaultValue}
* @throws IllegalArgumentException if the file does not exist
*/
public Path inputFile(String key, String description, Path defaultValue) {
Path path = file(key, description, defaultValue);
if (!Files.exists(path)) {
throw new IllegalArgumentException(path + " does not exist");
}
return path;
}
public boolean get(String arg, String description, boolean defaultValue) {
boolean value = "true".equalsIgnoreCase(getArg(arg, Boolean.toString(defaultValue)));
logArgValue(arg, description, value);
/**
* Parse an argument as a boolean.
* <p>
* {@code true} is considered true, and anything else will be handled as false.
*
* @param key argument name
* @param description argument description
* @param defaultValue fallback value if missing
* @return a boolean parsed from {@code key} otherwise {@code defaultValue}
*/
public boolean get(String key, String description, boolean defaultValue) {
boolean value = "true".equalsIgnoreCase(getArg(key, Boolean.toString(defaultValue)));
logArgValue(key, description, value);
return value;
}
public List<String> get(String arg, String description, String[] defaultValue) {
String value = getArg(arg, String.join(",", defaultValue));
/**
* Parse an argument as a list of strings.
*
* @param key argument name
* @param description argument description
* @param defaultValue fallback list if missing
* @return a list of strings parsed from {@code key} otherwise {@code defaultValue}
*/
public List<String> get(String key, String description, List<String> defaultValue) {
String value = getArg(key, String.join(",", defaultValue));
List<String> results = Stream.of(value.split("[\\s,]+"))
.filter(c -> !c.isBlank()).toList();
logArgValue(arg, description, value);
logArgValue(key, description, value);
return results;
}
/**
* Get the number of threads from {@link Runtime#availableProcessors()} but allow the user to override it by setting
* the {@code threads} argument.
*
* @return number of threads the program should use
* @throws NumberFormatException if {@code threads} can't be parsed as an integer
*/
public int threads() {
String value = getArg("threads", Integer.toString(Runtime.getRuntime().availableProcessors()));
int threads = Math.max(2, Integer.parseInt(value));
@ -123,6 +290,15 @@ public class Arguments {
return threads;
}
/**
* Return a {@link Stats} implementation based on the arguments provided.
* <p>
* If {@code pushgateway} is set then it uses a stats implementation that pushes to prometheus through a <a
* href="https://github.com/prometheus/pushgateway">push gateway</a> every {@code pushgateway.interval} seconds.
* Otherwise uses an in-memory stats implementation.
*
* @return the stats implementation to use
*/
public Stats getStats() {
String prometheus = getArg("pushgateway");
if (prometheus != null && !prometheus.isBlank()) {
@ -136,6 +312,15 @@ public class Arguments {
}
}
/**
* Parse an argument as integer
*
* @param key argument name
* @param description argument description
* @param defaultValue fallback value if missing
* @return an integer parsed from {@code key} otherwise {@code defaultValue}
* @throws NumberFormatException if the argument cannot be parsed as an integer
*/
public int integer(String key, String description, int defaultValue) {
String value = getArg(key, Integer.toString(defaultValue));
int parsed = Integer.parseInt(value);
@ -143,6 +328,15 @@ public class Arguments {
return parsed;
}
/**
* Parse an argument as {@link Duration} (ie. "10s", "90m", "1h30m")
*
* @param key argument name
* @param description argument description
* @param defaultValue fallback value if missing
* @return the parsed duration value
* @throws DateTimeParseException if the argument cannot be parsed as a duration
*/
public Duration duration(String key, String description, String defaultValue) {
String value = getArg(key, defaultValue);
Duration parsed = Duration.parse("PT" + value);

Wyświetl plik

@ -78,7 +78,7 @@ public record CommonParams(
}
public static CommonParams defaults() {
return from(Arguments.empty());
return from(Arguments.of());
}
public static CommonParams from(Arguments arguments) {

Wyświetl plik

@ -0,0 +1,134 @@
package com.onthegomap.flatmap.config;
import static org.junit.jupiter.api.Assertions.*;
import com.onthegomap.flatmap.TestUtils;
import com.onthegomap.flatmap.reader.osm.OsmInputFile;
import java.nio.file.Path;
import java.time.Duration;
import java.util.List;
import org.junit.jupiter.api.Test;
import org.locationtech.jts.geom.Envelope;
public class ArgumentsTest {
@Test
public void testEmpty() {
assertEquals("fallback", Arguments.of().get("key", "key", "fallback"));
}
@Test
public void testMapBased() {
assertEquals("value", Arguments.of(
"key", "value"
).get("key", "key", "fallback"));
}
@Test
public void testOrElse() {
Arguments args = Arguments.of("key1", "value1a", "key2", "value2a")
.orElse(Arguments.of("key2", "value2b", "key3", "value3b"));
assertEquals("value1a", args.get("key1", "key", "fallback"));
assertEquals("value2a", args.get("key2", "key", "fallback"));
assertEquals("value3b", args.get("key3", "key", "fallback"));
assertEquals("fallback", args.get("key4", "key", "fallback"));
}
@Test
public void testConfigFileParsing() {
Arguments args = Arguments.fromConfigFile(TestUtils.pathToResource("test.properties"));
assertEquals("value1fromfile", args.get("key1", "key", "fallback"));
assertEquals("fallback", args.get("key3", "key", "fallback"));
}
@Test
public void testGetConfigFileFromArgs() {
Arguments args = Arguments.fromArgsOrConfigFile(
"config=" + TestUtils.pathToResource("test.properties"),
"key2=value2fromargs"
);
assertEquals("value1fromfile", args.get("key1", "key", "fallback"));
assertEquals("value2fromargs", args.get("key2", "key", "fallback"));
assertEquals("fallback", args.get("key3", "key", "fallback"));
}
@Test
public void testDefaultsMissingConfigFile() {
Arguments args = Arguments.fromArgsOrConfigFile(
"key=value"
);
assertEquals("value", args.get("key", "key", "fallback"));
assertEquals("fallback", args.get("key2", "key", "fallback"));
}
@Test
public void testDuration() {
Arguments args = Arguments.of(
"duration", "1h30m"
);
assertEquals(Duration.ofMinutes(90), args.duration("duration", "key", "10m"));
assertEquals(Duration.ofSeconds(10), args.duration("duration2", "key", "10s"));
}
@Test
public void testInteger() {
Arguments args = Arguments.of(
"integer", "30"
);
assertEquals(30, args.integer("integer", "key", 10));
assertEquals(10, args.integer("integer2", "key", 10));
}
@Test
public void testThreads() {
assertEquals(2, Arguments.of("threads", "2").threads());
assertTrue(Arguments.of().threads() > 0);
}
@Test
public void testList() {
assertEquals(List.of("1", "2", "3"),
Arguments.of("list", "1,2,3").get("list", "list", List.of("1")));
assertEquals(List.of("1"),
Arguments.of().get("list", "list", List.of("1")));
}
@Test
public void testBoolean() {
assertTrue(Arguments.of("boolean", "true").get("boolean", "list", false));
assertFalse(Arguments.of("boolean", "false").get("boolean", "list", true));
assertFalse(Arguments.of("boolean", "true1").get("boolean", "list", true));
assertFalse(Arguments.of().get("boolean", "list", false));
}
@Test
public void testFile() {
assertNotNull(
Arguments.of("file", TestUtils.pathToResource("test.properties")).inputFile("file", "file", Path.of("")));
assertThrows(IllegalArgumentException.class,
() -> Arguments.of("file", TestUtils.pathToResource("test.Xproperties")).inputFile("file", "file", Path.of("")));
assertNotNull(
Arguments.of("file", TestUtils.pathToResource("test.Xproperties")).file("file", "file", Path.of("")));
}
@Test
public void testBounds() {
assertEquals(new Envelope(1, 3, 2, 4),
Arguments.of("bounds", "1,2,3,4").bounds("bounds", "bounds", BoundsProvider.WORLD));
assertEquals(new Envelope(-180.0, 180.0, -85.0511287798066, 85.0511287798066),
Arguments.of("bounds", "world").bounds("bounds", "bounds", BoundsProvider.WORLD));
assertEquals(new Envelope(7.409205, 7.448637, 43.72335, 43.75169),
Arguments.of().bounds("bounds", "bounds", new OsmInputFile(TestUtils.pathToResource("monaco-latest.osm.pbf"))));
}
@Test
public void testStats() {
assertNotNull(Arguments.of().getStats());
}
}

Wyświetl plik

@ -0,0 +1,3 @@
# used for ArgumentsTest
key1=value1fromfile
key2=value2fromfile

Wyświetl plik

@ -26,7 +26,7 @@ import java.util.List;
* <ol>
* <li>Download a .osm.pbf extract (see <a href="https://download.geofabrik.de/">Geofabrik download site</a></li>
* <li>then build the examples: {@code mvn -DskipTests=true --projects flatmap-examples -am clean package}</li>
* <li>then run this example: {@code java -Dosm="path/to/data.osm.pbf" -Dmbtiles="data/output.mbtiles" -cp flatmap-examples/target/flatmap-examples-*-fatjar.jar com.onthegomap.flatmap.examples.BikeRouteOverlay}</li>
* <li>then run this example: {@code java -cp flatmap-examples/target/flatmap-examples-*-fatjar.jar com.onthegomap.flatmap.examples.BikeRouteOverlay osm="path/to/data.osm.pbf" mbtiles="data/output.mbtiles"}</li>
* <li>then run the demo tileserver: {@code ./scripts/serve-tiles-docker.sh}</li>
* <li>and view the output at <a href="http://localhost:8080">localhost:8080</a></li>
* </ol>
@ -162,7 +162,7 @@ public class BikeRouteOverlay implements Profile {
* Main entrypoint for this example program
*/
public static void main(String[] args) throws Exception {
run(Arguments.fromJvmProperties());
run(Arguments.fromArgsOrConfigFile(args));
}
static void run(Arguments args) throws Exception {
@ -170,9 +170,9 @@ public class BikeRouteOverlay implements Profile {
// See ToiletsOverlayLowLevelApi for an example using the lower-level API
FlatMapRunner.createWithArguments(args)
.setProfile(new BikeRouteOverlay())
// override this default with -Dosm="path/to/data.osm.pbf"
// override this default with osm="path/to/data.osm.pbf"
.addOsmSource("osm", Path.of("data", "sources", "north-america_us_massachusetts.pbf"))
// override this default with -Dmbtiles="path/to/output.mbtiles"
// override this default with mbtiles="path/to/output.mbtiles"
.overwriteOutput("mbtiles", Path.of("data", "bikeroutes.mbtiles"))
.run();
}

Wyświetl plik

@ -17,7 +17,7 @@ import java.util.concurrent.atomic.AtomicInteger;
* <ol>
* <li>Download a .osm.pbf extract (see <a href="https://download.geofabrik.de/">Geofabrik download site</a></li>
* <li>then build the examples: {@code mvn -DskipTests=true --projects flatmap-examples -am clean package}</li>
* <li>then run this example: {@code java -Dosm="path/to/data.osm.pbf" -Dmbtiles="data/output.mbtiles" -cp flatmap-examples/target/flatmap-examples-*-fatjar.jar com.onthegomap.flatmap.examples.ToiletsOverlay}</li>
* <li>then run this example: {@code java -cp flatmap-examples/target/flatmap-examples-*-fatjar.jar com.onthegomap.flatmap.examples.ToiletsOverlay osm="path/to/data.osm.pbf" mbtiles="data/output.mbtiles"}</li>
* <li>then run the demo tileserver: {@code ./scripts/serve-tiles-docker.sh}</li>
* <li>and view the output at <a href="http://localhost:8080">localhost:8080</a></li>
* </ol>
@ -90,7 +90,7 @@ public class ToiletsOverlay implements Profile {
* Main entrypoint for the example program
*/
public static void main(String[] args) throws Exception {
run(Arguments.fromJvmProperties());
run(Arguments.fromArgsOrConfigFile(args));
}
static void run(Arguments args) throws Exception {
@ -98,9 +98,9 @@ public class ToiletsOverlay implements Profile {
// See ToiletsOverlayLowLevelApi for an example using this same profile but the lower-level API
FlatMapRunner.createWithArguments(args)
.setProfile(new ToiletsOverlay())
// override this default with -Dosm="path/to/data.osm.pbf"
// override this default with osm="path/to/data.osm.pbf"
.addOsmSource("osm", Path.of("data", "sources", "north-america_us_massachusetts.pbf"))
// override this default with -Dmbtiles="path/to/output.mbtiles"
// override this default with mbtiles="path/to/output.mbtiles"
.overwriteOutput("mbtiles", Path.of("data", "toilets.mbtiles"))
.run();
}

Wyświetl plik

@ -125,7 +125,7 @@ public class Generate {
}
public static void main(String[] args) throws IOException {
Arguments arguments = Arguments.fromJvmProperties();
Arguments arguments = Arguments.fromArgsOrConfigFile(args);
String tag = arguments.get("tag", "openmaptiles tag to use", "v3.12.2");
String base = "https://raw.githubusercontent.com/openmaptiles/openmaptiles/" + tag + "/";
var rootUrl = new URL(base + "openmaptiles.yaml");

Wyświetl plik

@ -20,7 +20,7 @@ public class OpenMapTilesMain {
private static final Path sourcesDir = Path.of("data", "sources");
public static void main(String[] args) throws Exception {
run(Arguments.fromJvmProperties());
run(Arguments.fromArgsOrConfigFile(args));
}
static void run(Arguments arguments) throws Exception {
@ -48,7 +48,7 @@ public class OpenMapTilesMain {
Path.of("data", "sources", "wikidata_names.json"));
// most common languages: "en,ru,ar,zh,ja,ko,fr,de,fi,pl,es,be,br,he"
List<String> languages = arguments
.get("name_languages", "languages to use", OpenMapTilesSchema.LANGUAGES.toArray(String[]::new));
.get("name_languages", "languages to use", OpenMapTilesSchema.LANGUAGES);
var translations = Translations.defaultProvider(languages).setShouldTransliterate(transliterate);
var profile = new OpenMapTilesProfile(translations, arguments, runner.stats());

Wyświetl plik

@ -69,8 +69,8 @@ public class OpenMapTilesProfile implements Profile {
}
public OpenMapTilesProfile(Translations translations, Arguments arguments, Stats stats) {
List<String> onlyLayers = arguments.get("only_layers", "Include only certain layers", new String[]{});
List<String> excludeLayers = arguments.get("exclude_layers", "Exclude certain layers", new String[]{});
List<String> onlyLayers = arguments.get("only_layers", "Include only certain layers", List.of());
List<String> excludeLayers = arguments.get("exclude_layers", "Exclude certain layers", List.of());
this.layers = OpenMapTilesSchema.createInstances(translations, arguments, stats)
.stream()
.filter(l -> (onlyLayers.isEmpty() || onlyLayers.contains(l.name())) && !excludeLayers.contains(l.name()))

Wyświetl plik

@ -13,11 +13,9 @@ AREA="${1:-north-america_us_massachusetts}"
if [ ! -f "$JAR" ]; then
echo "Building..."
mvn -DskipTests=true --projects openmaptiles -am clean package
mvn -DskipTests=true --projects flatmap-openmaptiles -am clean package
fi
echo "Running..."
java -Dinput="./data/sources/${AREA}.pbf" \
-Dforce=true \
-cp "$JAR" \
com.onthegomap.flatmap.openmaptiles.OpenMaptilesMain
java -cp "$JAR" com.onthegomap.flatmap.openmaptiles.OpenMapTilesMain \
-force=true -input="./data/sources/${AREA}.pbf"