mirror
/
planetiler
kopia lustrzana https://github.com/onthegomap/planetiler
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

logging tweaks

pull/1/head
Mike Barry 2021-07-18 08:17:58 -04:00
rodzic d5e990c3ae
commit 5e0d4b20ba
15 zmienionych plików z 107 dodań i 33 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

18
core/src/main/java/com/onthegomap/flatmap/FeatureCollector.java
Wyświetl plik

@ -47,8 +47,7 @@ public class FeatureCollector implements Iterable<FeatureCollector.Feature> {
}
return geometry(layer, source.worldGeometry());
} catch (GeometryException e) {
stats.dataError("feature_point_" + e.stat());
LOGGER.warn("Error getting point geometry for " + source + ": " + e.getMessage());
e.log(stats, "feature_point", "Error getting point geometry for " + source.id());
return new Feature(layer, EMPTY_GEOM, source.id());
}
}
@ -57,8 +56,7 @@ public class FeatureCollector implements Iterable<FeatureCollector.Feature> {
try {
return geometry(layer, source.centroid());
} catch (GeometryException e) {
stats.dataError("feature_centroid_" + e.stat());
LOGGER.warn("Error getting centroid for " + source + ": " + e.getMessage());
e.log(stats, "feature_centroid", "Error getting centroid for " + source.id());
return new Feature(layer, EMPTY_GEOM, source.id());
}
}
@ -67,8 +65,7 @@ public class FeatureCollector implements Iterable<FeatureCollector.Feature> {
try {
return geometry(layer, source.line());
} catch (GeometryException e) {
stats.dataError("feature_line_" + e.stat());
LOGGER.warn("Error constructing line for " + source + ": " + e.getMessage());
e.log(stats, "feature_line", "Error constructing line for " + source.id());
return new Feature(layer, EMPTY_GEOM, source.id());
}
}
@ -77,8 +74,7 @@ public class FeatureCollector implements Iterable<FeatureCollector.Feature> {
try {
return geometry(layer, source.polygon());
} catch (GeometryException e) {
stats.dataError("feature_polygon_" + e.stat());
LOGGER.warn("Error constructing polygon for " + source + ": " + e.getMessage());
e.log(stats, "feature_polygon", "Error constructing polygon for " + source.id());
return new Feature(layer, EMPTY_GEOM, source.id());
}
}
@ -87,8 +83,7 @@ public class FeatureCollector implements Iterable<FeatureCollector.Feature> {
try {
return geometry(layer, source.centroidIfConvex());
} catch (GeometryException e) {
stats.dataError("feature_centroid_if_convex_" + e.stat());
LOGGER.warn("Error constructing centroid if convex for " + source + ": " + e.getMessage());
e.log(stats, "feature_centroid_if_convex", "Error constructing centroid if convex for " + source.id());
return new Feature(layer, EMPTY_GEOM, source.id());
}
}
@ -97,8 +92,7 @@ public class FeatureCollector implements Iterable<FeatureCollector.Feature> {
try {
return geometry(layer, source.pointOnSurface());
} catch (GeometryException e) {
stats.dataError("feature_point_on_surface_" + e.stat());
LOGGER.warn("Error constructing point on surface for " + source + ": " + e.getMessage());
e.log(stats, "feature_point_on_surface", "Error constructing point on surface for " + source.id());
return new Feature(layer, EMPTY_GEOM, source.id());
}
}

2
core/src/main/java/com/onthegomap/flatmap/VectorTileEncoder.java
Wyświetl plik

@ -275,7 +275,7 @@ public class VectorTileEncoder {
attrs
));
} catch (GeometryException e) {
LOGGER.warn("Error decoding " + tileID + ": " + e);
e.log("Error decoding " + tileID);
}
}
}

3
core/src/main/java/com/onthegomap/flatmap/collections/FeatureGroup.java
Wyświetl plik

@ -388,8 +388,7 @@ public final class FeatureGroup implements Consumer<FeatureSort.Entry>, Iterable
try {
items = profile.postProcessLayerFeatures(currentLayer, tile.z(), items);
} catch (GeometryException e) {
stats.dataError("postprocess_layer_" + e.stat());
LOGGER.warn("error postprocessing features for " + currentLayer + " layer on " + tile + ": " + e.getMessage());
e.log(stats, "postprocess_layer", "Error postprocessing features for " + currentLayer + " layer on " + tile);
}
encoder.addLayerFeatures(currentLayer, items);
}

35
core/src/main/java/com/onthegomap/flatmap/geo/GeometryException.java
Wyświetl plik

@ -1,7 +1,13 @@
package com.onthegomap.flatmap.geo;
import com.onthegomap.flatmap.monitoring.Stats;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class GeometryException extends Exception {
private static final Logger LOGGER = LoggerFactory.getLogger(GeometryException.class);
private final String stat;
public GeometryException(String stat, String message, Throwable cause) {
@ -17,4 +23,33 @@ public class GeometryException extends Exception {
public String stat() {
return stat;
}
public void log(Stats stats, String statContext, String logContext) {
stats.dataError(statContext + "_" + stat());
log(logContext);
}
public void log(String logContext) {
logMessage(logContext + ": " + getMessage());
}
void logMessage(String log) {
LOGGER.warn(log);
}
public static class Verbose extends GeometryException {
public Verbose(String stat, String message, Throwable cause) {
super(stat, message, cause);
}
public Verbose(String stat, String message) {
super(stat, message);
}
@Override
void logMessage(String log) {
LOGGER.trace(log);
}
}
}

2
core/src/main/java/com/onthegomap/flatmap/monitoring/ProgressLoggers.java
Wyświetl plik

@ -135,7 +135,7 @@ public class ProgressLoggers {
}
public ProgressLoggers addProcessStats() {
add("\n" + " ".repeat(4));
add("\n" + " ".repeat(3));
addOptionalDeltaLogger("cpus", ProcessInfo::getProcessCpuTime, Format::formatDecimal);
addDeltaLogger("gc", ProcessInfo::getGcTime, Format::formatPercent);
loggers.add(new ProgressLogger("mem",

4
core/src/main/java/com/onthegomap/flatmap/read/OsmMultipolygon.java
Wyświetl plik

@ -106,7 +106,7 @@ public class OsmMultipolygon {
) throws GeometryException {
try {
if (rings.size() == 0) {
throw new GeometryException("osm_invalid_multipolygon_empty",
throw new GeometryException.Verbose("osm_invalid_multipolygon_empty",
"error building multipolygon " + osmId + ": no rings to process");
}
List<LongArrayList> idSegments = connectPolygonSegments(rings);
@ -123,7 +123,7 @@ public class OsmMultipolygon {
polygons.sort(BY_AREA_DESCENDING);
Set<Ring> shells = groupParentChildShells(polygons);
if (shells.size() == 0) {
throw new GeometryException("osm_invalid_multipolygon_not_closed",
throw new GeometryException.Verbose("osm_invalid_multipolygon_not_closed",
"error building multipolygon " + osmId + ": multipolygon not closed");
} else if (shells.size() == 1) {
return shells.iterator().next().toPolygon();

3
core/src/main/java/com/onthegomap/flatmap/render/FeatureRenderer.java
Wyświetl plik

@ -200,8 +200,7 @@ public class FeatureRenderer implements Consumer<FeatureCollector.Feature> {
emitted++;
}
} catch (GeometryException e) {
stats.dataError("write_tile_features_" + e.stat());
LOGGER.warn(e.getMessage() + ": " + tile + " " + feature);
e.log(stats, "write_tile_features", "Error writing tile " + tile + " feature " + feature);
}
}

4
core/src/test/java/com/onthegomap/flatmap/monitoring/ProgressLoggersTest.java
Wyświetl plik

@ -29,11 +29,11 @@ public class ProgressLoggersTest {
// spin waiting for threads to start
}
assertEquals("[prefix] | reader( 0%) -> (0/10) -> worker( 0% 0%) -> (0/10) -> writer( 0% 0%)",
assertEquals("[prefix]\n reader( 0%) -> (0/10) -> worker( 0% 0%) -> (0/10) -> writer( 0% 0%)",
log.replaceAll("[ 0-9][0-9]%", " 0%"));
latch.countDown();
topology.awaitAndLog(loggers, Duration.ofSeconds(10));
assertEquals("[prefix] | reader( -%) -> (0/10) -> worker( -% -%) -> (0/10) -> writer( -% -%)",
assertEquals("[prefix]\n reader( -%) -> (0/10) -> worker( -% -%) -> (0/10) -> writer( -% -%)",
loggers.getLog());
}
}

4
openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/Park.java
Wyświetl plik

@ -31,8 +31,10 @@ public class Park implements
private static final Logger LOGGER = LoggerFactory.getLogger(Park.class);
private final Translations translations;
private final Stats stats;
public Park(Translations translations, Arguments args, Stats stats) {
this.stats = stats;
this.translations = translations;
}
@ -79,7 +81,7 @@ public class Park implements
areaBoost
).setZoomRange(minzoom, 14);
} catch (GeometryException e) {
LOGGER.warn("Unable to get park area for " + element.source().id() + ": " + e.getMessage());
e.log(stats, "omt_park_area", "Unable to get park area for " + element.source().id());
}
}
}

18
openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/Place.java
Wyświetl plik

@ -54,6 +54,7 @@ public class Place implements
private static final Logger LOGGER = LoggerFactory.getLogger(Place.class);
private final Translations translations;
private final Stats stats;
private static record NaturalEarthRegion(String name, int rank) {
@ -77,6 +78,7 @@ public class Place implements
public Place(Translations translations, Arguments args, Stats stats) {
this.translations = translations;
this.stats = stats;
}
@Override
@ -111,8 +113,8 @@ public class Place implements
));
}
} catch (GeometryException e) {
LOGGER
.warn("Error getting geometry for natural earth feature " + table + " " + feature.getTag("ogc_fid") + " " + e);
e.log(stats, "omt_place_ne",
"Error getting geometry for natural earth feature " + table + " " + feature.getTag("ogc_fid"));
}
}
@ -163,7 +165,8 @@ public class Place implements
.setZoomRange(rank - 1, 14)
.setZorder(-rank);
} catch (GeometryException e) {
LOGGER.warn("Unable to get point for OSM country " + element.source().id());
e.log(stats, "omt_place_country",
"Unable to get point for OSM country " + element.source().id());
}
}
@ -187,7 +190,8 @@ public class Place implements
.setZorder(-rank);
}
} catch (GeometryException e) {
LOGGER.warn("Unable to get point for OSM state " + element.source().id());
e.log(stats, "omt_place_state",
"Unable to get point for OSM state " + element.source().id());
}
}
@ -226,7 +230,8 @@ public class Place implements
.setZoomRange(minzoom, 14)
.setZorder(zOrder);
} catch (GeometryException e) {
LOGGER.warn("Unable to get area for OSM island polygon " + element.source().id() + ": " + e);
e.log(stats, "omt_place_island_poly",
"Unable to get point for OSM island polygon " + element.source().id());
}
}
@ -320,7 +325,8 @@ public class Place implements
}
}
} catch (GeometryException e) {
LOGGER.warn("Unable to get area for OSM city " + element.source().id() + ": " + e);
e.log(stats, "omt_place_city",
"Unable to get point for OSM city " + element.source().id());
}
}

5
openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/Transportation.java
Wyświetl plik

@ -72,8 +72,10 @@ public class Transportation implements
.put(4, 1_000);
private static final double PIXEL = 256d / 4096d;
private final boolean z13Paths;
private final Stats stats;
public Transportation(Translations translations, Arguments args, Stats stats) {
this.stats = stats;
this.z13Paths = args.get(
"transportation_z13_paths",
"transportation(_name) layer: show paths on z13",
@ -149,7 +151,8 @@ public class Transportation implements
return;
}
} catch (GeometryException e) {
LOGGER.warn("Unable to decode pier geometry for " + element.source().id());
e.log(stats, "omt_transportation_pier",
"Unable to decode pier geometry for " + element.source().id());
return;
}
minzoom = 13;

5
openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/TransportationName.java
Wyświetl plik

@ -67,10 +67,12 @@ public class TransportationName implements
private final boolean sizeForShield;
private final boolean limitMerge;
private final boolean z13Paths;
private final Stats stats;
private PreparedGeometry greatBritain = null;
private AtomicBoolean loggedNoGb = new AtomicBoolean(false);
public TransportationName(Translations translations, Arguments args, Stats stats) {
this.stats = stats;
this.brunnel = args.get(
"transportation_name_brunnel",
"transportation_name layer: set to false to omit brunnel and help merge long highways",
@ -230,7 +232,8 @@ public class TransportationName implements
relation = new RouteRelation(refMatcher.group(), networkType, 0);
}
} catch (GeometryException e) {
LOGGER.warn("Unable to test highway against GB route network: " + element.source().id());
e.log(stats, "omt_transportation_name_gb_test",
"Unable to test highway against GB route network: " + element.source().id());
}
}
}

6
openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/layers/WaterName.java
Wyświetl plik

@ -33,6 +33,7 @@ public class WaterName implements OpenMapTilesSchema.WaterName,
private final Translations translations;
private final LongObjectMap<Geometry> lakeCenterlines = new GHLongObjectHashMap<>();
private final TreeMap<String, Integer> importantMarinePoints = new TreeMap<>();
private final Stats stats;
@Override
public void release() {
@ -42,6 +43,7 @@ public class WaterName implements OpenMapTilesSchema.WaterName,
public WaterName(Translations translations, Arguments args, Stats stats) {
this.translations = translations;
this.stats = stats;
}
@Override
@ -69,7 +71,7 @@ public class WaterName implements OpenMapTilesSchema.WaterName,
try {
lakeCenterlines.put(osmId, feature.worldGeometry());
} catch (GeometryException e) {
LOGGER.warn("Bad lake centerline geometry: " + feature, e);
e.log(stats, "omt_water_name_lakeline", "Bad lake centerline: " + feature);
}
}
}
@ -134,7 +136,7 @@ public class WaterName implements OpenMapTilesSchema.WaterName,
.setAttr(Fields.INTERMITTENT, element.isIntermittent() ? 1 : 0)
.setZoomRange(minzoom, 14);
} catch (GeometryException e) {
LOGGER.warn("Unable to get geometry for water polygon " + element.source().id() + ": " + e);
e.log(stats, "omt_water_polygon", "Unable to get geometry for water polygon " + element.source().id());
}
}
}

7
scripts/build.sh 100755
Wyświetl plik

@ -0,0 +1,7 @@
#!/usr/bin/env bash
set -o errexit
set -o pipefail
set -o nounset
mvn -DskipTests=true --projects openmaptiles -am clean package

24
scripts/ubuntu_vm_setup.sh 100644
Wyświetl plik

@ -0,0 +1,24 @@
#!/usr/bin/env bash
set -o errexit
set -o pipefail
set -o nounset
set -x
if test "$#" -ne 1; then
echo "Usage: ubuntu_vm_setup.sh user@ip"
exit 1
fi
"$(dirname "$0")"/build.sh
scp openmaptiles/target/flatmap-openmaptiles-0.1-SNAPSHOT-jar-with-dependencies.jar "${1}":flatmap.jar
scp scripts/download-other-sources.sh "${1}":download-other-sources.sh
scp scripts/download-osm.sh "${1}":download-osm.sh
ssh "${1}" "
wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | sudo apt-key add - && \
add-apt-repository --yes https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/ && \
apt-get update -y && \
apt-get install adoptopenjdk-16-hotspot-jre -y && \
./download-other-sources.sh
"