diff --git a/core/src/main/java/com/onthegomap/flatmap/CommonParams.java b/core/src/main/java/com/onthegomap/flatmap/CommonParams.java index cab5cfa8..a781a8d4 100644 --- a/core/src/main/java/com/onthegomap/flatmap/CommonParams.java +++ b/core/src/main/java/com/onthegomap/flatmap/CommonParams.java @@ -13,6 +13,7 @@ public record CommonParams( int maxzoom, boolean deferIndexCreation, boolean optimizeDb, + boolean emitTilesInOrder, boolean forceOverwrite, boolean gzipTempStorage, String longLongMap, @@ -30,6 +31,7 @@ public record CommonParams( int maxzoom, boolean deferIndexCreation, boolean optimizeDb, + boolean emitTilesInOrder, boolean forceOverwrite, boolean gzipTempStorage, String longLongMap @@ -42,6 +44,7 @@ public record CommonParams( maxzoom, deferIndexCreation, optimizeDb, + emitTilesInOrder, forceOverwrite, gzipTempStorage, longLongMap, @@ -84,6 +87,7 @@ public record CommonParams( arguments.integer("maxzoom", "maximum zoom level (limit 14)", MAX_MAXZOOM), arguments.get("defer_mbtiles_index_creation", "add index to mbtiles file after finished writing", false), arguments.get("optimize_db", "optimize mbtiles after writing", false), + arguments.get("emit_tiles_in_order", "emit tiles in index order", false), arguments.get("force", "force overwriting output file", false), arguments.get("gzip_temp", "gzip temporary feature storage (uses more CPU, but less disk space)", false), arguments.get("llmap", "type of long long map", "mapdb") diff --git a/core/src/main/java/com/onthegomap/flatmap/geo/PointIndex.java b/core/src/main/java/com/onthegomap/flatmap/geo/PointIndex.java index 8db9f4b1..b0aa5292 100644 --- a/core/src/main/java/com/onthegomap/flatmap/geo/PointIndex.java +++ b/core/src/main/java/com/onthegomap/flatmap/geo/PointIndex.java @@ -2,6 +2,7 @@ package com.onthegomap.flatmap.geo; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; @@ -22,7 +23,21 @@ public class PointIndex { return new PointIndex<>(); } + private volatile boolean built = false; + + private void build() { + if (!built) { + synchronized (this) { + if (!built) { + index.build(); + built = true; + } + } + } + } + public List getWithin(Point point, double threshold) { + build(); Coordinate coord = point.getCoordinate(); Envelope envelope = point.getEnvelopeInternal(); envelope.expandBy(threshold); @@ -41,6 +56,7 @@ public class PointIndex { } public T getNearest(Point point, double threshold) { + build(); Coordinate coord = point.getCoordinate(); Envelope envelope = point.getEnvelopeInternal(); envelope.expandBy(threshold); @@ -62,7 +78,8 @@ public class PointIndex { public void put(Geometry geom, T item) { if (geom instanceof Point point && !point.isEmpty()) { - index.insert(point.getEnvelopeInternal(), new GeomWithData<>(point.getCoordinate(), item)); + Envelope envelope = Objects.requireNonNull(point.getEnvelopeInternal()); + index.insert(envelope, new GeomWithData<>(point.getCoordinate(), item)); } else if (geom instanceof GeometryCollection geoms) { for (int i = 0; i < geoms.getNumGeometries(); i++) { put(geoms.getGeometryN(i), item); diff --git a/core/src/main/java/com/onthegomap/flatmap/write/MbtilesWriter.java b/core/src/main/java/com/onthegomap/flatmap/write/MbtilesWriter.java index 78582239..38173f5f 100644 --- a/core/src/main/java/com/onthegomap/flatmap/write/MbtilesWriter.java +++ b/core/src/main/java/com/onthegomap/flatmap/write/MbtilesWriter.java @@ -99,20 +99,31 @@ public class MbtilesWriter { var topology = Topology.start("mbtiles", stats); - int queueSize = 30_000 / BATCH_SIZE; + int queueSize = 10_000; WorkQueue writerQueue = new WorkQueue<>("mbtiles_writer_queue", queueSize, 1, stats); - var encodeBranch = topology - .fromGenerator("reader", next -> writer.readFeatures(batch -> { - next.accept(batch); - writerQueue.accept(batch); // also send immediately to writer - }), 1) - .addBuffer("reader_queue", queueSize) - .sinkTo("encoder", config.threads(), writer::tileEncoder); + Topology encodeBranch, writeBranch; + if (true || config.emitTilesInOrder()) { + encodeBranch = topology + .fromGenerator("reader", next -> writer.readFeatures(batch -> { + next.accept(batch); + writerQueue.accept(batch); // also send immediately to writer + }), 1) + .addBuffer("reader_queue", queueSize) + .sinkTo("encoder", config.threads(), writer::tileEncoder); - // the tile writer will wait on the result of each batch to ensure tiles are written in order - var writeBranch = topology.readFromQueue(writerQueue) - .sinkTo("writer", 1, writer::tileWriter); + // the tile writer will wait on the result of each batch to ensure tiles are written in order + writeBranch = topology.readFromQueue(writerQueue) + .sinkTo("writer", 1, writer::tileWriter); + } else { + // TODO +// encodeBranch = topology +// .fromGenerator("reader", writer::readFeatures, 1) +// .addBuffer("reader_queue", queueSize) +// .addWorker("encoder", config.threads(), (prev, next) -> { +// TOO +// }) + } var loggers = new ProgressLoggers("mbtiles") .addRatePercentCounter("features", features.numFeatures(), writer.featuresProcessed) @@ -164,16 +175,24 @@ public class MbtilesWriter { void readFeatures(Consumer next) { int currentZoom = Integer.MIN_VALUE; TileBatch batch = new TileBatch(); + long featuresInThisBatch = 0; + long tilesInThisBatch = 0; + // 249 vs. 24,900 + long MAX_FEATURES_PER_BATCH = BATCH_SIZE * 100; for (var feature : features) { int z = feature.coord().z(); if (z > currentZoom) { LOGGER.info("[mbtiles] Starting z" + z); currentZoom = z; } - if (batch.in.size() >= BATCH_SIZE) { + if (tilesInThisBatch > BATCH_SIZE || featuresInThisBatch > MAX_FEATURES_PER_BATCH) { next.accept(batch); batch = new TileBatch(); + featuresInThisBatch = 0; + tilesInThisBatch = 0; } + featuresInThisBatch++; + tilesInThisBatch += feature.getNumFeatures(); batch.in.offer(feature); } if (!batch.in.isEmpty()) { diff --git a/scripts/build.sh b/scripts/build.sh index ad0b2500..b51de91e 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -4,4 +4,4 @@ set -o errexit set -o pipefail set -o nounset -mvn -DskipTests=true --projects openmaptiles -am package +mvn -DskipTests=true --projects openmaptiles -am clean package diff --git a/scripts/ubuntu_vm_setup.sh b/scripts/ubuntu_vm_setup.sh index bca653aa..9fecbce6 100755 --- a/scripts/ubuntu_vm_setup.sh +++ b/scripts/ubuntu_vm_setup.sh @@ -15,7 +15,7 @@ fi rsync -avzP openmaptiles/target/flatmap-openmaptiles-0.1-SNAPSHOT-fatjar.jar "${1}":flatmap.jar scp scripts/download-other-sources.sh "${1}":download-other-sources.sh scp scripts/download-osm.sh "${1}":download-osm.sh -ssh "${1}" "bash -s" <