From 74db638dbc69f10ab6de2f7a6447327d062fc86f Mon Sep 17 00:00:00 2001 From: Michael Barry Date: Sat, 18 Mar 2023 14:38:04 -0400 Subject: [PATCH] Expose pmtiles writer (#520) --- .github/workflows/maven.yml | 2 +- .github/workflows/performance.yml | 130 +++---- PLANET.md | 6 +- README.md | 14 +- .../benchmarks/BenchmarkMbtilesWriter.java | 4 +- .../com/onthegomap/planetiler/Planetiler.java | 125 ++++--- .../archive/ReadableTileArchive.java | 5 + .../planetiler/archive/TileArchiveConfig.java | 196 ++++++++++ .../archive/TileArchiveMetadata.java | 143 +++++-- .../planetiler/archive/TileArchiveWriter.java | 24 +- .../planetiler/archive/TileArchives.java | 75 ++++ .../archive/WriteableTileArchive.java | 55 +-- .../planetiler/config/Arguments.java | 106 ++++-- .../planetiler/config/PlanetilerConfig.java | 8 - .../planetiler/mbtiles/Mbtiles.java | 348 ++++++++++-------- .../onthegomap/planetiler/mbtiles/Verify.java | 2 +- .../planetiler/pmtiles/Pmtiles.java | 20 +- .../planetiler/pmtiles/ReadablePmtiles.java | 38 ++ .../planetiler/pmtiles/WriteablePmtiles.java | 66 ++-- .../onthegomap/planetiler/stats/Stats.java | 24 +- .../onthegomap/planetiler/util/FileUtils.java | 20 +- .../onthegomap/planetiler/util/Format.java | 76 ++-- .../planetiler/util/ResourceUsage.java | 2 +- .../planetiler/PlanetilerTests.java | 62 ++-- .../com/onthegomap/planetiler/TestUtils.java | 23 +- .../archive/TileArchiveConfigTest.java | 36 ++ .../archive/TileArchiveMetadataTest.java | 67 ++++ .../planetiler/config/ArgumentsTest.java | 56 +++ .../planetiler/mbtiles/MbtilesTest.java | 124 +++---- .../planetiler/mbtiles/VerifyTest.java | 4 +- .../planetiler/pmtiles/PmtilesTest.java | 90 +++-- planetiler-custommap/README.md | 3 - planetiler-custommap/planetiler.schema.json | 33 -- .../custommap/ConfiguredMapMain.java | 4 +- .../planetiler/custommap/Contexts.java | 3 - .../custommap/ConfiguredMapTest.java | 4 +- planetiler-examples/README.md | 4 +- .../planetiler/examples/BikeRouteOverlay.java | 4 +- .../planetiler/examples/OsmQaTiles.java | 2 +- .../planetiler/examples/ToiletsOverlay.java | 4 +- .../examples/ToiletsOverlayLowLevelApi.java | 9 +- .../examples/BikeRouteOverlayTest.java | 4 +- .../planetiler/examples/OsmQaTilesTest.java | 4 +- .../ToiletsOverlayLowLevelApiTest.java | 2 +- .../examples/ToiletsProfileTest.java | 4 +- planetiler-openmaptiles | 2 +- quickstart.sh | 5 +- scripts/test-release.sh | 8 +- 48 files changed, 1386 insertions(+), 664 deletions(-) create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchives.java create mode 100644 planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileArchiveConfigTest.java create mode 100644 planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileArchiveMetadataTest.java diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 1dc859cd..e26ca7e7 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -85,7 +85,7 @@ jobs: run: mv target/*with-deps.jar ./run.jar working-directory: planetiler-examples - name: Run - run: java -jar run.jar --osm-path=../planetiler-core/src/test/resources/monaco-latest.osm.pbf --mbtiles=data/out.mbtiles + run: java -jar run.jar --osm-path=../planetiler-core/src/test/resources/monaco-latest.osm.pbf --output=data/out.mbtiles working-directory: planetiler-examples - name: Verify run: java -cp run.jar com.onthegomap.planetiler.mbtiles.Verify data/out.mbtiles diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 5e706b84..639c37ff 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -22,73 +22,73 @@ jobs: timeout-minutes: 20 continue-on-error: true steps: - - name: 'Cancel previous runs' - uses: styfle/cancel-workflow-action@0.11.0 - with: - access_token: ${{ github.token }} - - name: 'Checkout branch' - uses: actions/checkout@v3 - with: - path: branch - submodules: true - - name: 'Checkout base' - uses: actions/checkout@v3 - with: - path: base - ref: ${{ github.event.pull_request.base.sha }} - submodules: true - - name: Cache data/sources - uses: ./branch/.github/cache-sources-action - with: - basedir: branch - - name: Set up JDK - uses: actions/setup-java@v3 - with: - java-version: 17 - distribution: 'temurin' - cache: 'maven' - - uses: actions/setup-node@v3 - with: - node-version: '14' - - run: npm install -g strip-ansi-cli@3.0.2 + - name: 'Cancel previous runs' + uses: styfle/cancel-workflow-action@0.11.0 + with: + access_token: ${{ github.token }} + - name: 'Checkout branch' + uses: actions/checkout@v3 + with: + path: branch + submodules: true + - name: 'Checkout base' + uses: actions/checkout@v3 + with: + path: base + ref: ${{ github.event.pull_request.base.sha }} + submodules: true + - name: Cache data/sources + uses: ./branch/.github/cache-sources-action + with: + basedir: branch + - name: Set up JDK + uses: actions/setup-java@v3 + with: + java-version: 17 + distribution: 'temurin' + cache: 'maven' + - uses: actions/setup-node@v3 + with: + node-version: '14' + - run: npm install -g strip-ansi-cli@3.0.2 - - name: 'Build branch' - run: ./scripts/build.sh - working-directory: branch - - name: 'Build base' - run: ./scripts/build.sh - working-directory: base + - name: 'Build branch' + run: ./scripts/build.sh + working-directory: branch + - name: 'Build base' + run: ./scripts/build.sh + working-directory: base - - name: 'Download data' - run: | - set -eo pipefail - cp base/planetiler-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}" - cp branch/planetiler-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}" + - name: 'Download data' + run: | + set -eo pipefail + cp base/planetiler-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}" + cp branch/planetiler-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}" - - name: 'Store build info' - run: | - mkdir build-info - echo "${{ github.event.pull_request.base.sha }}" > build-info/base_sha - echo "${{ github.sha }}" > build-info/branch_sha - echo "${{ github.event.number }}" > build-info/pull_request_number + - name: 'Store build info' + run: | + mkdir build-info + echo "${{ github.event.pull_request.base.sha }}" > build-info/base_sha + echo "${{ github.sha }}" > build-info/branch_sha + echo "${{ github.event.number }}" > build-info/pull_request_number - - name: 'Run branch' - run: | - rm -rf data/out.mbtiles data/tmp - cp branch/planetiler-dist/target/*with-deps.jar run.jar - java -Xms${{ env.RAM }} -Xmx${{ env.RAM }} -jar run.jar --area="${{ env.AREA }}" "${{ env.BOUNDS_ARG }}" --mbtiles=data/out.mbtiles 2>&1 | tee log - ls -alh run.jar | tee -a log - cat log | strip-ansi > build-info/branchlogs.txt - - name: 'Run base' - run: | - rm -rf data/out.mbtiles data/tmp - cp base/planetiler-dist/target/*with-deps.jar run.jar - java -Xms${{ env.RAM }} -Xmx${{ env.RAM }} -jar run.jar --area="${{ env.AREA }}" "${{ env.BOUNDS_ARG }}" --mbtiles=data/out.mbtiles 2>&1 | tee log - ls -alh run.jar | tee -a log - cat log | strip-ansi > build-info/baselogs.txt + - name: 'Run branch' + run: | + rm -rf data/out.mbtiles data/tmp + cp branch/planetiler-dist/target/*with-deps.jar run.jar + java -Xms${{ env.RAM }} -Xmx${{ env.RAM }} -jar run.jar --area="${{ env.AREA }}" "${{ env.BOUNDS_ARG }}" --output=data/out.mbtiles 2>&1 | tee log + ls -alh run.jar | tee -a log + cat log | strip-ansi > build-info/branchlogs.txt + - name: 'Run base' + run: | + rm -rf data/out.mbtiles data/tmp + cp base/planetiler-dist/target/*with-deps.jar run.jar + java -Xms${{ env.RAM }} -Xmx${{ env.RAM }} -jar run.jar --area="${{ env.AREA }}" "${{ env.BOUNDS_ARG }}" --output=data/out.mbtiles 2>&1 | tee log + ls -alh run.jar | tee -a log + cat log | strip-ansi > build-info/baselogs.txt - - name: 'Upload build-info' - uses: actions/upload-artifact@v3 - with: - name: build-info - path: ./build-info + - name: 'Upload build-info' + uses: actions/upload-artifact@v3 + with: + name: build-info + path: ./build-info diff --git a/PLANET.md b/PLANET.md index 1fd9d4cc..6c8bd268 100644 --- a/PLANET.md +++ b/PLANET.md @@ -50,7 +50,7 @@ java -Xmx110g \ --download-threads=10 --download-chunk-size-mb=1000 \ `# Also download name translations from wikidata` \ --fetch-wikidata \ - --mbtiles=output.mbtiles \ + --output=output.mbtiles \ `# Store temporary node locations in memory` \ --nodemap-type=array --storage=ram ``` @@ -67,7 +67,7 @@ java -Xmx20g \ --download-threads=10 --download-chunk-size-mb=1000 \ `# Also download name translations from wikidata` \ --fetch-wikidata \ - --mbtiles=output.mbtiles \ + --output=output.mbtiles \ `# Store temporary node locations at fixed positions in a memory-mapped file` \ --nodemap-type=array --storage=mmap ``` @@ -103,7 +103,7 @@ java -Xmx100g \ --download-threads=10 --download-chunk-size-mb=1000 \ `# Also download name translations from wikidata` \ --fetch-wikidata \ - --mbtiles=output.mbtiles \ + --output=output.mbtiles \ --nodemap-type=sparsearray --nodemap-storage=ram 2>&1 | tee logs.txt ``` diff --git a/README.md b/README.md index 2d6303c0..5aecca43 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,10 @@ or database. Vector tiles contain raw point, line, and polygon geometries that clients like [MapLibre](https://github.com/maplibre) can use to render custom maps in the browser, native apps, or on a server. Planetiler packages tiles into -an [MBTiles](https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md) (sqlite) file that can be served using -tools like [TileServer GL](https://github.com/maptiler/tileserver-gl) or even -[queried directly from the browser](https://github.com/phiresky/sql.js-httpvfs). +an [MBTiles](https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md) (sqlite) +or [PMTiles](https://github.com/protomaps/PMTiles) file that can be served using tools +like [TileServer GL](https://github.com/maptiler/tileserver-gl) or [Martin](https://github.com/maplibre/martin) or +even [queried directly from the browser](https://github.com/protomaps/PMTiles/tree/main/js). See [awesome-vector-tiles](https://github.com/mapbox/awesome-vector-tiles) for more projects that work with data in this format. @@ -87,7 +88,7 @@ Using [Node.js](https://nodejs.org/en/download/): ```bash npm install -g tileserver-gl-light -tileserver-gl-light --mbtiles data/output.mbtiles +tileserver-gl-light data/output.mbtiles ``` Or using [Docker](https://docs.docker.com/get-docker/): @@ -100,6 +101,8 @@ Then open http://localhost:8080 to view tiles. Some common arguments: +- `--output` tells planetiler where to write output to, and what format to write it in. For + example `--output=australia.pmtiles` creates a pmtiles archive named `australia.pmtiles`. - `--download` downloads input sources automatically and `--only-download` exits after downloading - `--area=monaco` downloads a `.osm.pbf` extract from [Geofabrik](https://download.geofabrik.de/) - `--osm-path=path/to/file.osm.pbf` points Planetiler at an existing OSM extract on disk @@ -209,6 +212,8 @@ download regularly-updated tilesets. OpenStreetMap [.osm.pbf](https://wiki.openstreetmap.org/wiki/PBF_Format), [`geopackage`](https://www.geopackage.org/), and [Esri Shapefiles](https://en.wikipedia.org/wiki/Shapefile) data sources +- Writes to [MBTiles](https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md) or + or [PMTiles](https://github.com/protomaps/PMTiles) output. - Java-based [Profile API](planetiler-core/src/main/java/com/onthegomap/planetiler/Profile.java) to customize how source elements map to vector tile features, and post-process generated tiles using [JTS geometry utilities](https://github.com/locationtech/jts) @@ -328,6 +333,7 @@ Planetiler is made possible by these awesome open source projects: - [cel-java](https://github.com/projectnessie/cel-java) for the Java implementation of Google's [Common Expression Language](https://github.com/google/cel-spec) that powers dynamic expressions embedded in schema config files. +- [PMTiles](https://github.com/protomaps/PMTiles) optimized tile storage format See [NOTICE.md](NOTICE.md) for a full list and license details. diff --git a/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesWriter.java b/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesWriter.java index 47379b9f..41545a01 100644 --- a/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesWriter.java +++ b/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesWriter.java @@ -66,9 +66,9 @@ public class BenchmarkMbtilesWriter { for (int repetition = 0; repetition < repetitions; repetition++) { Path outputPath = getTempOutputPath(); - try (var mbtiles = Mbtiles.newWriteToFileDatabase(outputPath, config.compactDb())) { + try (var mbtiles = Mbtiles.newWriteToFileDatabase(outputPath, config.arguments())) { - if (config.skipIndexCreation()) { + if (mbtiles.skipIndexCreation()) { mbtiles.createTablesWithoutIndexes(); } else { mbtiles.createTablesWithIndexes(); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java index 72e3cf4a..596942be 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java @@ -1,14 +1,15 @@ package com.onthegomap.planetiler; +import com.onthegomap.planetiler.archive.TileArchiveConfig; import com.onthegomap.planetiler.archive.TileArchiveMetadata; import com.onthegomap.planetiler.archive.TileArchiveWriter; +import com.onthegomap.planetiler.archive.TileArchives; import com.onthegomap.planetiler.archive.WriteableTileArchive; import com.onthegomap.planetiler.collection.FeatureGroup; import com.onthegomap.planetiler.collection.LongLongMap; import com.onthegomap.planetiler.collection.LongLongMultimap; import com.onthegomap.planetiler.config.Arguments; import com.onthegomap.planetiler.config.PlanetilerConfig; -import com.onthegomap.planetiler.mbtiles.Mbtiles; import com.onthegomap.planetiler.reader.GeoPackageReader; import com.onthegomap.planetiler.reader.NaturalEarthReader; import com.onthegomap.planetiler.reader.ShapefileReader; @@ -85,7 +86,7 @@ public class Planetiler { private final PlanetilerConfig config; private FeatureGroup featureGroup; private OsmInputFile osmInputFile; - private Path output; + private TileArchiveConfig output; private boolean overwrite = false; private boolean ran = false; // most common OSM languages @@ -151,8 +152,8 @@ public class Planetiler { * * @param name string to use in stats and logs to identify this stage * @param defaultPath path to the input file to use if {@code name_path} argument is not set - * @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and {@code - * name_url} argument is not set. As a shortcut, can use "geofabrik:monaco" or + * @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and + * {@code name_url} argument is not set. As a shortcut, can use "geofabrik:monaco" or * "geofabrik:australia" shorthand to find an extract by name from * Geofabrik download site or "aws:latest" to download * the latest {@code planet.osm.pbf} file from AWS @@ -183,10 +184,11 @@ public class Planetiler { ), ifSourceUsed(name, () -> { var header = osmInputFile.getHeader(); - tileArchiveMetadata.set("planetiler:" + name + ":osmosisreplicationtime", header.instant()); - tileArchiveMetadata.set("planetiler:" + name + ":osmosisreplicationseq", + tileArchiveMetadata.setExtraMetadata("planetiler:" + name + ":osmosisreplicationtime", header.instant()); + tileArchiveMetadata.setExtraMetadata("planetiler:" + name + ":osmosisreplicationseq", header.osmosisReplicationSequenceNumber()); - tileArchiveMetadata.set("planetiler:" + name + ":osmosisreplicationurl", header.osmosisReplicationBaseUrl()); + tileArchiveMetadata.setExtraMetadata("planetiler:" + name + ":osmosisreplicationurl", + header.osmosisReplicationBaseUrl()); try ( var nodeLocations = LongLongMap.from(config.nodeMapType(), config.nodeMapStorage(), nodeDbPath, config.nodeMapMadvise()); @@ -253,8 +255,8 @@ public class Planetiler { * @param defaultPath path to the input file to use if {@code name_path} key is not set through arguments. Can be a * {@code .shp} file with other shapefile components in the same directory, or a {@code .zip} file * containing the shapefile components. - * @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and {@code - * name_url} argument is not set + * @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and + * {@code name_url} argument is not set * @return this runner instance for chaining * @see ShapefileReader * @see Downloader @@ -327,8 +329,8 @@ public class Planetiler { * @param defaultPath path to the input file to use if {@code name_path} key is not set through arguments. Can be a * {@code .shp} file with other shapefile components in the same directory, or a {@code .zip} file * containing the shapefile components. - * @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and {@code - * name_url} argument is not set + * @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and + * {@code name_url} argument is not set * @return this runner instance for chaining * @see ShapefileReader * @see Downloader @@ -362,8 +364,8 @@ public class Planetiler { * {@link org.geotools.referencing.CRS#decode(String)} * @param name string to use in stats and logs to identify this stage * @param defaultPath path to the input file to use if {@code name_path} key is not set through arguments - * @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and {@code - * name_url} argument is not set + * @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and + * {@code name_url} argument is not set * @return this runner instance for chaining * @see GeoPackageReader * @see Downloader @@ -399,8 +401,8 @@ public class Planetiler { * * @param name string to use in stats and logs to identify this stage * @param defaultPath path to the input file to use if {@code name_path} key is not set through arguments - * @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and {@code - * name_url} argument is not set + * @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and + * {@code name_url} argument is not set * @return this runner instance for chaining * @see GeoPackageReader * @see Downloader @@ -415,12 +417,12 @@ public class Planetiler { * To override the location of the {@code sqlite} file, set {@code name_path=newpath.zip} in the arguments and to * override the download URL set {@code name_url=http://url/of/natural_earth.zip}. * - * @deprecated can be replaced by {@link #addGeoPackageSource(String, Path, String)}. * @param name string to use in stats and logs to identify this stage * @param defaultPath path to the input file to use if {@code name} key is not set through arguments. Can be the * {@code .sqlite} file or a {@code .zip} file containing the sqlite file. * @return this runner instance for chaining * @see NaturalEarthReader + * @deprecated can be replaced by {@link #addGeoPackageSource(String, Path, String)}. */ @Deprecated(forRemoval = true) public Planetiler addNaturalEarthSource(String name, Path defaultPath) { @@ -436,16 +438,15 @@ public class Planetiler { * To override the location of the {@code sqlite} file, set {@code name_path=newpath.zip} in the arguments and to * override the download URL set {@code name_url=http://url/of/natural_earth.zip}. * - * @deprecated can be replaced by {@link #addGeoPackageSource(String, Path, String)}. - * * @param name string to use in stats and logs to identify this stage * @param defaultPath path to the input file to use if {@code name} key is not set through arguments. Can be the * {@code .sqlite} file or a {@code .zip} file containing the sqlite file. - * @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and {@code - * name_url} argument is not set + * @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and + * {@code name_url} argument is not set * @return this runner instance for chaining * @see NaturalEarthReader * @see Downloader + * @deprecated can be replaced by {@link #addGeoPackageSource(String, Path, String)}. */ @Deprecated(forRemoval = true) public Planetiler addNaturalEarthSource(String name, Path defaultPath, String defaultUrl) { @@ -540,35 +541,69 @@ public class Planetiler { } /** - * Sets the location of the output archive to write rendered tiles to. Fails if the archive already exists. - *

- * To override the location of the file, set {@code argument=newpath} in the arguments. + * Sets the location of the output archive to write rendered tiles to. * - * @param argument the argument key to check for an override to {@code fallback} - * @param fallback the fallback value if {@code argument} is not set in arguments - * @return this runner instance for chaining - * @see TileArchiveWriter + * @deprecated Use {@link #setOutput(String)} instead */ + @Deprecated(forRemoval = true) public Planetiler setOutput(String argument, Path fallback) { - this.output = arguments.file(argument, "output tile archive", fallback); + this.output = + TileArchiveConfig + .from(arguments.getString("output|" + argument, "output tile archive path", fallback.toString())); return this; } /** - * Sets the location of the output archive to write rendered tiles to. Overwrites file if it already exists. + * Sets the location of the output archive to write rendered tiles to. Fails if the archive already exists. *

- * To override the location of the file, set {@code argument=newpath} in the arguments. + * To override the location of the file, set {@code argument=newpath} in the arguments. To set options for the output + * drive add {@code output.mbtiles?arg=value} or add command-line argument {@code mbtiles_arg=value}. * - * @param argument the argument key to check for an override to {@code fallback} - * @param fallback the fallback value if {@code argument} is not set in arguments + * @param defaultOutputUri The default output URI string to write to. * @return this runner instance for chaining - * @see TileArchiveWriter + * @see TileArchiveConfig For details on URI string formats and options. */ + public Planetiler setOutput(String defaultOutputUri) { + this.output = TileArchiveConfig.from(arguments.getString("output", "output tile archive URI", defaultOutputUri)); + return this; + } + + /** Alias for {@link #setOutput(String)} which infers the output type based on extension. */ + public Planetiler setOutput(Path path) { + return setOutput(path.toString()); + } + + /** + * Sets the location of the output archive to write rendered tiles to. + * + * @deprecated Use {@link #overwriteOutput(String)} instead + */ + @Deprecated(forRemoval = true) public Planetiler overwriteOutput(String argument, Path fallback) { this.overwrite = true; return setOutput(argument, fallback); } + /** + * Sets the location of the output archive to write rendered tiles to. Overwrites if the archive already exists. + *

+ * To override the location of the file, set {@code argument=newpath} in the arguments. To set options for the output + * drive add {@code output.mbtiles?arg=value} or add command-line argument {@code mbtiles_arg=value}. + * + * @param defaultOutputUri The default output URI string to write to. + * @return this runner instance for chaining + * @see TileArchiveConfig For details on URI string formats and options. + */ + public Planetiler overwriteOutput(String defaultOutputUri) { + this.overwrite = true; + return setOutput(defaultOutputUri); + } + + /** Alias for {@link #overwriteOutput(String)} which infers the output type based on extension. */ + public Planetiler overwriteOutput(Path defaultOutput) { + return overwriteOutput(defaultOutput.toString()); + } + /** * Reads all elements from all sourced that have been added, generates map features according to the profile, and * writes the rendered tiles to the output archive. @@ -600,19 +635,18 @@ public class Planetiler { throw new IllegalArgumentException("Can only run once"); } ran = true; - tileArchiveMetadata = new TileArchiveMetadata(profile, config.arguments()); if (arguments.getBoolean("help", "show arguments then exit", false)) { System.exit(0); } else if (onlyDownloadSources) { // don't check files if not generating map } else if (overwrite || config.force()) { - FileUtils.deleteFile(output); - } else if (Files.exists(output)) { - throw new IllegalArgumentException(output + " already exists, use the --force argument to overwrite."); + output.delete(); + } else if (output.exists()) { + throw new IllegalArgumentException(output.uri() + " already exists, use the --force argument to overwrite."); } - LOGGER.info("Building {} profile into {} in these phases:", profile.getClass().getSimpleName(), output); + LOGGER.info("Building {} profile into {} in these phases:", profile.getClass().getSimpleName(), output.uri()); if (!toDownload.isEmpty()) { LOGGER.info(" download: Download sources {}", toDownload.stream().map(d -> d.id).toList()); @@ -635,7 +669,7 @@ public class Planetiler { // in case any temp files are left from a previous run... FileUtils.delete(tmpDir, nodeDbPath, featureDbPath, multipolygonPath); Files.createDirectories(tmpDir); - FileUtils.createParentDirectories(nodeDbPath, featureDbPath, multipolygonPath, output); + FileUtils.createParentDirectories(nodeDbPath, featureDbPath, multipolygonPath, output.getLocalPath()); if (!toDownload.isEmpty()) { download(); @@ -661,14 +695,16 @@ public class Planetiler { } bounds.addFallbackProvider(new OsmNodeBoundsProvider(osmInputFile, config, stats)); } + // must construct this after bounds providers are added in order to infer bounds from the input source if not provided + tileArchiveMetadata = new TileArchiveMetadata(profile, config); - try (WriteableTileArchive archive = Mbtiles.newWriteToFileDatabase(output, config.compactDb())) { + try (WriteableTileArchive archive = TileArchives.newWriter(output, config)) { featureGroup = FeatureGroup.newDiskBackedFeatureGroup(archive.tileOrder(), featureDbPath, profile, config, stats); stats.monitorFile("nodes", nodeDbPath); stats.monitorFile("features", featureDbPath); stats.monitorFile("multipolygons", multipolygonPath); - stats.monitorFile("archive", output); + stats.monitorFile("archive", output.getLocalPath()); for (Stage stage : stages) { stage.task.run(); @@ -685,9 +721,8 @@ public class Planetiler { featureGroup.prepare(); - TileArchiveWriter.writeOutput(featureGroup, archive, () -> FileUtils.fileSize(output), tileArchiveMetadata, - config, - stats); + TileArchiveWriter.writeOutput(featureGroup, archive, output::size, tileArchiveMetadata, + config, stats); } catch (IOException e) { throw new IllegalStateException("Unable to write to " + output, e); } @@ -716,7 +751,7 @@ public class Planetiler { readPhase.addDisk(featureDbPath, featureSize, "temporary feature storage"); writePhase.addDisk(featureDbPath, featureSize, "temporary feature storage"); // output only needed during write phase - writePhase.addDisk(output, outputSize, "archive output"); + writePhase.addDisk(output.getLocalPath(), outputSize, "archive output"); // if the user opts to remove an input source after reading to free up additional space for the output... for (var input : inputPaths) { if (input.freeAfterReading()) { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/ReadableTileArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/ReadableTileArchive.java index 6702de02..51b0b0e0 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/ReadableTileArchive.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/ReadableTileArchive.java @@ -40,5 +40,10 @@ public interface ReadableTileArchive extends Closeable { */ CloseableIterator getAllTileCoords(); + /** + * Returns the metadata stored in this archive. + */ + TileArchiveMetadata metadata(); + // TODO access archive metadata } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java new file mode 100644 index 00000000..021ccc50 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java @@ -0,0 +1,196 @@ +package com.onthegomap.planetiler.archive; + +import static com.onthegomap.planetiler.util.LanguageUtils.nullIfEmpty; + +import com.onthegomap.planetiler.config.Arguments; +import com.onthegomap.planetiler.util.FileUtils; +import java.net.URI; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; + +/** + * Definition for a tileset, parsed from a URI-like string. + *

+ * {@link #from(String)} can accept: + *

+ *

+ * Both of these can also have archive-specific options added to the end, for example + * {@code "output.mbtiles?compact=false&page_size=16384"}. + * + * @param format The {@link Format format} of the archive, either inferred from the filename extension or the + * {@code ?format=} query parameter + * @param scheme Scheme for accessing the archive + * @param uri Full URI including scheme, location, and options + * @param options Parsed query parameters from the definition string + */ +public record TileArchiveConfig( + Format format, + Scheme scheme, + URI uri, + Map options +) { + + private static TileArchiveConfig.Scheme getScheme(URI uri) { + String scheme = uri.getScheme(); + if (scheme == null) { + return Scheme.FILE; + } + for (var value : TileArchiveConfig.Scheme.values()) { + if (value.id().equals(scheme)) { + return value; + } + } + throw new IllegalArgumentException("Unsupported scheme " + scheme + " from " + uri); + } + + private static String getExtension(URI uri) { + String path = uri.getPath(); + if (path != null && (path.contains("."))) { + return nullIfEmpty(path.substring(path.lastIndexOf(".") + 1)); + } + return null; + } + + private static Map parseQuery(URI uri) { + String query = uri.getRawQuery(); + Map result = new HashMap<>(); + if (query != null) { + for (var part : query.split("&")) { + var split = part.split("=", 2); + result.put( + URLDecoder.decode(split[0], StandardCharsets.UTF_8), + split.length == 1 ? "true" : URLDecoder.decode(split[1], StandardCharsets.UTF_8) + ); + } + } + return result; + } + + private static TileArchiveConfig.Format getFormat(URI uri) { + String format = parseQuery(uri).get("format"); + if (format == null) { + format = getExtension(uri); + } + if (format == null) { + return TileArchiveConfig.Format.MBTILES; + } + for (var value : TileArchiveConfig.Format.values()) { + if (value.id().equals(format)) { + return value; + } + } + throw new IllegalArgumentException("Unsupported format " + format + " from " + uri); + } + + /** + * Parses a string definition of a tileset from a URI-like string. + */ + public static TileArchiveConfig from(String string) { + // unix paths parse fine as URIs, but need to explicitly parse windows paths with backslashes + if (string.contains("\\")) { + String[] parts = string.split("\\?", 2); + string = Path.of(parts[0]).toUri().toString(); + if (parts.length > 1) { + string += "?" + parts[1]; + } + } + return from(URI.create(string)); + } + + /** + * Parses a string definition of a tileset from a URI. + */ + public static TileArchiveConfig from(URI uri) { + if (uri.getScheme() == null) { + String base = Path.of(uri.getPath()).toAbsolutePath().toUri().normalize().toString(); + if (uri.getRawQuery() != null) { + base += "?" + uri.getRawQuery(); + } + uri = URI.create(base); + } + return new TileArchiveConfig( + getFormat(uri), + getScheme(uri), + uri, + parseQuery(uri) + ); + } + + /** + * Returns the local path on disk that this archive reads/writes to, or {@code null} if it is not on disk (ie. an HTTP + * repository). + */ + public Path getLocalPath() { + return scheme == Scheme.FILE ? Path.of(URI.create(uri.toString().replaceAll("\\?.*$", ""))) : null; + } + + + /** + * Deletes the archive if possible. + */ + public void delete() { + if (scheme == Scheme.FILE) { + FileUtils.delete(getLocalPath()); + } + } + + /** + * Returns {@code true} if the archive already exists, {@code false} otherwise. + */ + public boolean exists() { + return getLocalPath() != null && Files.exists(getLocalPath()); + } + + /** + * Returns the current size of this archive. + */ + public long size() { + return getLocalPath() == null ? 0 : FileUtils.size(getLocalPath()); + } + + /** + * Returns an {@link Arguments} instance that returns the value for options directly from the query parameters in the + * URI, or from {@code arguments} prefixed by {@code "format_"}. + */ + public Arguments applyFallbacks(Arguments arguments) { + return Arguments.of(options).orElse(arguments.withPrefix(format.id)); + } + + public enum Format { + MBTILES("mbtiles"), + PMTILES("pmtiles"); + + private final String id; + + Format(String id) { + this.id = id; + } + + public String id() { + return id; + } + } + + public enum Scheme { + FILE("file"); + + private final String id; + + Scheme(String id) { + this.id = id; + } + + public String id() { + return id; + } + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveMetadata.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveMetadata.java index 842e2ae1..5184d7cd 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveMetadata.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveMetadata.java @@ -1,42 +1,90 @@ package com.onthegomap.planetiler.archive; -import com.onthegomap.planetiler.Profile; -import com.onthegomap.planetiler.config.Arguments; -import com.onthegomap.planetiler.util.BuildInfo; -import java.util.LinkedHashMap; -import java.util.Map; +import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_ABSENT; +import static com.onthegomap.planetiler.util.Format.joinCoordinates; -/** Controls information that {@link TileArchiveWriter} will write to the archive metadata. */ +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import com.onthegomap.planetiler.Profile; +import com.onthegomap.planetiler.config.PlanetilerConfig; +import com.onthegomap.planetiler.geo.GeoUtils; +import com.onthegomap.planetiler.util.BuildInfo; +import com.onthegomap.planetiler.util.LayerStats; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.locationtech.jts.geom.CoordinateXY; +import org.locationtech.jts.geom.Envelope; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Metadata associated with a tile archive. */ public record TileArchiveMetadata( - String name, - String description, - String attribution, - String version, - String type, - Map planetilerSpecific + @JsonProperty(NAME_KEY) String name, + @JsonProperty(DESCRIPTION_KEY) String description, + @JsonProperty(ATTRIBUTION_KEY) String attribution, + @JsonProperty(VERSION_KEY) String version, + @JsonProperty(TYPE_KEY) String type, + @JsonProperty(FORMAT_KEY) String format, + @JsonIgnore Envelope bounds, + @JsonIgnore CoordinateXY center, + @JsonProperty(ZOOM_KEY) Double zoom, + @JsonProperty(MINZOOM_KEY) Integer minzoom, + @JsonProperty(MAXZOOM_KEY) Integer maxzoom, + @JsonIgnore List vectorLayers, + @JsonAnyGetter Map others ) { - public TileArchiveMetadata(Profile profile) { + public static final String NAME_KEY = "name"; + public static final String DESCRIPTION_KEY = "description"; + public static final String ATTRIBUTION_KEY = "attribution"; + public static final String VERSION_KEY = "version"; + public static final String TYPE_KEY = "type"; + public static final String FORMAT_KEY = "format"; + public static final String BOUNDS_KEY = "bounds"; + public static final String CENTER_KEY = "center"; + public static final String ZOOM_KEY = "zoom"; + public static final String MINZOOM_KEY = "minzoom"; + public static final String MAXZOOM_KEY = "maxzoom"; + public static final String VECTOR_LAYERS_KEY = "vector_layers"; + + public static final String MVT_FORMAT = "pbf"; + + private static final Logger LOGGER = LoggerFactory.getLogger(TileArchiveMetadata.class); + private static final ObjectMapper mapper = new ObjectMapper() + .registerModules(new Jdk8Module()) + .setSerializationInclusion(NON_ABSENT); + + public TileArchiveMetadata(Profile profile, PlanetilerConfig config) { + this(profile, config, null); + } + + public TileArchiveMetadata(Profile profile, PlanetilerConfig config, List vectorLayers) { this( - profile.name(), - profile.description(), - profile.attribution(), - profile.version(), - profile.isOverlay() ? "overlay" : "baselayer", + getString(config, NAME_KEY, profile.name()), + getString(config, DESCRIPTION_KEY, profile.description()), + getString(config, ATTRIBUTION_KEY, profile.attribution()), + getString(config, VERSION_KEY, profile.version()), + getString(config, TYPE_KEY, profile.isOverlay() ? "overlay" : "baselayer"), + getString(config, FORMAT_KEY, MVT_FORMAT), + config.bounds().latLon(), + new CoordinateXY(config.bounds().latLon().centre()), + GeoUtils.getZoomFromLonLatBounds(config.bounds().latLon()), + config.minzoom(), + config.maxzoom(), + vectorLayers, mapWithBuildInfo() ); } - public TileArchiveMetadata(Profile profile, Arguments args) { - this( - args.getString("mbtiles_name", "'name' attribute for tileset metadata", profile.name()), - args.getString("mbtiles_description", "'description' attribute for tileset metadata", profile.description()), - args.getString("mbtiles_attribution", "'attribution' attribute for tileset metadata", profile.attribution()), - args.getString("mbtiles_version", "'version' attribute for tileset metadata", profile.version()), - args.getString("mbtiles_type", "'type' attribute for tileset metadata", - profile.isOverlay() ? "overlay" : "baselayer"), - mapWithBuildInfo() - ); + private static String getString(PlanetilerConfig config, String key, String fallback) { + return config.arguments() + .getString("archive_" + key + "|mbtiles_" + key, "'" + key + "' attribute for tileset metadata", fallback); } private static Map mapWithBuildInfo() { @@ -56,20 +104,39 @@ public record TileArchiveMetadata( return result; } - public TileArchiveMetadata set(String key, Object value) { + /** Sets an extra metadata entry in {@link #others}. */ + public TileArchiveMetadata setExtraMetadata(String key, Object value) { if (key != null && value != null) { - planetilerSpecific.put(key, value.toString()); + others.put(key, value.toString()); } return this; } - public Map getAll() { - var allKvs = new LinkedHashMap(planetilerSpecific); - allKvs.put("name", this.name); - allKvs.put("description", this.description); - allKvs.put("attribution", this.attribution); - allKvs.put("version", this.version); - allKvs.put("type", this.type); - return allKvs; + /** + * Returns a map with all key-value pairs from this metadata entry, including {@link #others} hoisted to top-level + * keys. + */ + public Map toMap() { + Map result = new LinkedHashMap<>(mapper.convertValue(this, new TypeReference<>() {})); + if (bounds != null) { + result.put(BOUNDS_KEY, joinCoordinates(bounds.getMinX(), bounds.getMinY(), bounds.getMaxX(), bounds.getMaxY())); + } + if (center != null) { + result.put(CENTER_KEY, joinCoordinates(center.getX(), center.getY())); + } + if (vectorLayers != null) { + try { + result.put(VECTOR_LAYERS_KEY, mapper.writeValueAsString(vectorLayers)); + } catch (JsonProcessingException e) { + LOGGER.warn("Error encoding vector_layers as json", e); + } + } + return result; + } + + /** Returns a copy of this instance with {@link #vectorLayers} set to {@code layerStats}. */ + public TileArchiveMetadata withLayerStats(List layerStats) { + return new TileArchiveMetadata(name, description, attribution, version, type, format, bounds, center, zoom, minzoom, + maxzoom, layerStats, others); } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java index ad95e1c1..27a34144 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java @@ -15,7 +15,6 @@ import com.onthegomap.planetiler.stats.Timer; import com.onthegomap.planetiler.util.DiskBacked; import com.onthegomap.planetiler.util.Format; import com.onthegomap.planetiler.util.Hashing; -import com.onthegomap.planetiler.util.LayerStats; import com.onthegomap.planetiler.worker.WorkQueue; import com.onthegomap.planetiler.worker.Worker; import com.onthegomap.planetiler.worker.WorkerPipeline; @@ -52,7 +51,6 @@ public class TileArchiveWriter { private final WriteableTileArchive archive; private final PlanetilerConfig config; private final Stats stats; - private final LayerStats layerStats; private final Counter.Readable[] tilesByZoom; private final Counter.Readable[] totalTileSizesByZoom; private final LongAccumulator[] maxTileSizesByZoom; @@ -61,14 +59,12 @@ public class TileArchiveWriter { private final TileArchiveMetadata tileArchiveMetadata; private TileArchiveWriter(Iterable inputTiles, WriteableTileArchive archive, - PlanetilerConfig config, - TileArchiveMetadata tileArchiveMetadata, Stats stats, LayerStats layerStats) { + PlanetilerConfig config, TileArchiveMetadata tileArchiveMetadata, Stats stats) { this.inputTiles = inputTiles; this.archive = archive; this.config = config; this.tileArchiveMetadata = tileArchiveMetadata; this.stats = stats; - this.layerStats = layerStats; tilesByZoom = IntStream.rangeClosed(0, config.maxzoom()) .mapToObj(i -> Counter.newSingleThreadCounter()) .toArray(Counter.Readable[]::new); @@ -111,8 +107,9 @@ public class TileArchiveWriter { readWorker = reader.readWorker(); } - TileArchiveWriter writer = new TileArchiveWriter(inputTiles, output, config, tileArchiveMetadata, stats, - features.layerStats()); + TileArchiveWriter writer = + new TileArchiveWriter(inputTiles, output, config, tileArchiveMetadata.withLayerStats(features.layerStats() + .getTileStats()), stats); var pipeline = WorkerPipeline.start("archive", stats); @@ -231,7 +228,6 @@ public class TileArchiveWriter { byte[] lastBytes = null, lastEncoded = null; Long lastTileDataHash = null; boolean lastIsFill = false; - boolean compactDb = config.compactDb(); boolean skipFilled = config.skipFilledTiles(); for (TileBatch batch : prev) { @@ -265,7 +261,7 @@ public class TileArchiveWriter { lastEncoded = encoded; lastBytes = bytes; last = tileFeatures; - if (compactDb && en.likelyToBeDuplicated() && bytes != null) { + if (archive.deduplicates() && en.likelyToBeDuplicated() && bytes != null) { tileDataHash = generateContentHash(bytes); } else { tileDataHash = null; @@ -292,7 +288,8 @@ public class TileArchiveWriter { private void tileWriter(Iterable tileBatches) throws ExecutionException, InterruptedException { - archive.initialize(config, tileArchiveMetadata, layerStats); + archive.initialize(tileArchiveMetadata); + var order = archive.tileOrder(); TileCoord lastTile = null; Timer time = null; @@ -303,8 +300,9 @@ public class TileArchiveWriter { TileEncodingResult encodedTile; while ((encodedTile = encodedTiles.poll()) != null) { TileCoord tileCoord = encodedTile.coord(); - assert lastTile == null || lastTile.compareTo(tileCoord) < 0 : "Tiles out of order %s before %s" - .formatted(lastTile, tileCoord); + assert lastTile == null || + order.encode(tileCoord) > order.encode(lastTile) : "Tiles out of order %s before %s" + .formatted(lastTile, tileCoord); lastTile = encodedTile.coord(); int z = tileCoord.z(); if (z != currentZ) { @@ -331,7 +329,7 @@ public class TileArchiveWriter { } - archive.finish(config); + archive.finish(tileArchiveMetadata); } private void printTileStats() { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchives.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchives.java new file mode 100644 index 00000000..fdd4617b --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchives.java @@ -0,0 +1,75 @@ +package com.onthegomap.planetiler.archive; + +import com.onthegomap.planetiler.config.PlanetilerConfig; +import com.onthegomap.planetiler.mbtiles.Mbtiles; +import com.onthegomap.planetiler.pmtiles.ReadablePmtiles; +import com.onthegomap.planetiler.pmtiles.WriteablePmtiles; +import java.io.IOException; +import java.nio.file.Path; + +/** Utilities for creating {@link ReadableTileArchive} and {@link WriteableTileArchive} instances. */ +public class TileArchives { + private TileArchives() {} + + /** + * Returns a new {@link WriteableTileArchive} from the string definition in {@code archive} that will be parsed with + * {@link TileArchiveConfig}. + * + * @throws IOException if an error occurs creating the resource. + */ + public static WriteableTileArchive newWriter(String archive, PlanetilerConfig config) throws IOException { + return newWriter(TileArchiveConfig.from(archive), config); + } + + /** + * Returns a new {@link ReadableTileArchive} from the string definition in {@code archive} that will be parsed with + * {@link TileArchiveConfig}. + * + * @throws IOException if an error occurs opening the resource. + */ + public static ReadableTileArchive newReader(String archive, PlanetilerConfig config) throws IOException { + return newReader(TileArchiveConfig.from(archive), config); + } + + /** + * Returns a new {@link WriteableTileArchive} from the string definition in {@code archive}. + * + * @throws IOException if an error occurs creating the resource. + */ + public static WriteableTileArchive newWriter(TileArchiveConfig archive, PlanetilerConfig config) + throws IOException { + var options = archive.applyFallbacks(config.arguments()); + return switch (archive.format()) { + case MBTILES -> + // pass-through legacy arguments for fallback + Mbtiles.newWriteToFileDatabase(archive.getLocalPath(), options.orElse(config.arguments() + .subset(Mbtiles.LEGACY_VACUUM_ANALYZE, Mbtiles.LEGACY_COMPACT_DB, Mbtiles.LEGACY_SKIP_INDEX_CREATION))); + case PMTILES -> WriteablePmtiles.newWriteToFile(archive.getLocalPath()); + }; + } + + /** + * Returns a new {@link ReadableTileArchive} from the string definition in {@code archive}. + * + * @throws IOException if an error occurs opening the resource. + */ + public static ReadableTileArchive newReader(TileArchiveConfig archive, PlanetilerConfig config) + throws IOException { + var options = archive.applyFallbacks(config.arguments()); + return switch (archive.format()) { + case MBTILES -> Mbtiles.newReadOnlyDatabase(archive.getLocalPath(), options); + case PMTILES -> ReadablePmtiles.newReadFromFile(archive.getLocalPath()); + }; + } + + /** Alias for {@link #newReader(String, PlanetilerConfig)}. */ + public static ReadableTileArchive newReader(Path path, PlanetilerConfig config) throws IOException { + return newReader(path.toString(), config); + } + + /** Alias for {@link #newWriter(String, PlanetilerConfig)}. */ + public static WriteableTileArchive newWriter(Path path, PlanetilerConfig config) throws IOException { + return newWriter(path.toString(), config); + } + +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java index 7449c0b1..3c3b6df7 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java @@ -2,7 +2,6 @@ package com.onthegomap.planetiler.archive; import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.geo.TileOrder; -import com.onthegomap.planetiler.util.LayerStats; import java.io.Closeable; import javax.annotation.concurrent.NotThreadSafe; @@ -15,6 +14,36 @@ import javax.annotation.concurrent.NotThreadSafe; @NotThreadSafe public interface WriteableTileArchive extends Closeable { + /** + * Returns true if this tile archive deduplicates tiles with the same content. + *

+ * If false, then {@link TileWriter} will skip computing tile hashes. + */ + boolean deduplicates(); + + /** + * Specify the preferred insertion order for this archive, e.g. {@link TileOrder#TMS} or {@link TileOrder#HILBERT}. + */ + TileOrder tileOrder(); + + /** + * Called before any tiles are written into {@link TileWriter}. Implementations of TileArchive should set up any + * required state here. + */ + default void initialize(TileArchiveMetadata metadata) {} + + /** + * Implementations should return a object that implements {@link TileWriter} The specific TileWriter returned might + * depend on {@link PlanetilerConfig}. + */ + TileWriter newTileWriter(); + + /** + * Called after all tiles are written into {@link TileWriter}. After this is called, the archive should be complete on + * disk. + */ + default void finish(TileArchiveMetadata tileArchiveMetadata) {} + interface TileWriter extends Closeable { void write(TileEncodingResult encodingResult); @@ -30,29 +59,5 @@ public interface WriteableTileArchive extends Closeable { default void printStats() {} } - - /** - * Specify the preferred insertion order for this archive, e.g. {@link TileOrder#TMS} or {@link TileOrder#HILBERT}. - */ - TileOrder tileOrder(); - - /** - * Called before any tiles are written into {@link TileWriter}. Implementations of TileArchive should set up any - * required state here. - */ - void initialize(PlanetilerConfig config, TileArchiveMetadata metadata, LayerStats layerStats); - - /** - * Implementations should return a object that implements {@link TileWriter} The specific TileWriter returned might - * depend on {@link PlanetilerConfig}. - */ - TileWriter newTileWriter(); - - /** - * Called after all tiles are written into {@link TileWriter}. After this is called, the archive should be complete on - * disk. - */ - void finish(PlanetilerConfig config); - // TODO update archive metadata } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java index 02ad44ea..573ad2f5 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java @@ -12,6 +12,8 @@ import java.time.format.DateTimeParseException; import java.time.temporal.ChronoUnit; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -20,6 +22,7 @@ import java.util.Set; import java.util.TreeMap; import java.util.function.Supplier; import java.util.function.UnaryOperator; +import java.util.regex.Pattern; import java.util.stream.Stream; import org.locationtech.jts.geom.Envelope; import org.slf4j.Logger; @@ -28,6 +31,12 @@ import org.slf4j.LoggerFactory; /** * Lightweight abstraction over ways to provide key/value pair arguments to a program like jvm properties, environmental * variables, or a config file. + *

+ * When looking up a key, tries to find a case-and-separator-insensitive match, for example {@code "CONFIG_OPTION"} will + * match {@code "config-option"} and {@code "config_option"}. + *

+ * If you replace an option with a new value, you can read a value from the new option and fall back to old one by using + * {@code "new_flag|old_flag"} as the key. */ public class Arguments { @@ -42,15 +51,6 @@ public class Arguments { this.keys = keys; } - private static Arguments from(UnaryOperator provider, Supplier> rawKeys, - UnaryOperator forward, UnaryOperator reverse) { - Supplier> keys = () -> rawKeys.get().stream().flatMap(key -> { - String reversed = reverse.apply(key); - return key.equalsIgnoreCase(reversed) ? Stream.empty() : Stream.of(reversed); - }).toList(); - return new Arguments(key -> provider.apply(forward.apply(key)), keys); - } - /** * Returns arguments from JVM system properties prefixed with {@code planetiler.} *

@@ -64,10 +64,7 @@ public class Arguments { } static Arguments fromJvmProperties(UnaryOperator getter, Supplier> keys) { - return from(getter, keys, - key -> "planetiler." + key.toLowerCase(Locale.ROOT), - key -> key.replaceFirst("^planetiler\\.", "").toLowerCase(Locale.ROOT) - ); + return fromPrefixed(getter, keys, "planetiler", ".", false); } /** @@ -83,10 +80,7 @@ public class Arguments { } static Arguments fromEnvironment(UnaryOperator getter, Supplier> keys) { - return from(getter, keys, - key -> "PLANETILER_" + key.toUpperCase(Locale.ROOT), - key -> key.replaceFirst("^PLANETILER_", "").toLowerCase(Locale.ROOT) - ); + return fromPrefixed(getter, keys, "PLANETILER", "_", true); } /** @@ -191,8 +185,21 @@ public class Arguments { .orElse(fromEnvironment()); } + private static String normalize(String key, String separator, boolean upperCase) { + String result = key.replaceAll("[._-]", separator); + return upperCase ? result.toUpperCase(Locale.ROOT) : result.toLowerCase(Locale.ROOT); + } + + private static String normalize(String key) { + return normalize(key, "_", false); + } + public static Arguments of(Map map) { - return new Arguments(map::get, map::keySet); + Map updated = new LinkedHashMap<>(); + for (var entry : map.entrySet()) { + updated.put(normalize(entry.getKey()), entry.getValue()); + } + return new Arguments(updated::get, updated::keySet); } /** Shorthand for {@link #of(Map)} which constructs the map from a list of key/value pairs. */ @@ -204,12 +211,36 @@ public class Arguments { return of(map); } + private static Arguments from(UnaryOperator provider, Supplier> rawKeys, + UnaryOperator forward, UnaryOperator reverse) { + Supplier> keys = () -> rawKeys.get().stream().flatMap(key -> { + String reversed = reverse.apply(key); + return normalize(key).equals(normalize(reversed)) ? Stream.empty() : Stream.of(reversed); + }).toList(); + return new Arguments(key -> provider.apply(forward.apply(key)), keys); + } + + private static Arguments fromPrefixed(UnaryOperator provider, Supplier> keys, + String prefix, String separator, boolean uppperCase) { + var prefixRegex = Pattern.compile("^" + Pattern.quote(normalize(prefix + separator, separator, uppperCase)), + Pattern.CASE_INSENSITIVE); + return from(provider, keys, + key -> normalize(prefix + separator + key, separator, uppperCase), + key -> normalize(prefixRegex.matcher(key).replaceFirst("")) + ); + } + private String get(String key) { - String value = provider.apply(key); - if (value == null) { - value = provider.apply(key.replace('-', '_')); - if (value == null) { - value = provider.apply(key.replace('_', '-')); + String[] options = key.split("\\|"); + String value = null; + for (int i = 0; i < options.length; i++) { + String option = options[i].strip(); + value = provider.apply(normalize(option)); + if (value != null) { + if (i != 0) { + LOGGER.warn("Argument '{}' is deprecated", option); + } + break; } } return value; @@ -274,8 +305,8 @@ public class Arguments { } protected void logArgValue(String key, String description, Object result) { - if (!silent) { - LOGGER.debug("argument: {}={} ({})", key, result, description); + if (!silent && LOGGER.isDebugEnabled()) { + LOGGER.debug("argument: {}={} ({})", key.replaceFirst("\\|.*$", ""), result, description); } } @@ -460,7 +491,7 @@ public class Arguments { public Map toMap() { Map result = new HashMap<>(); for (var key : keys.get()) { - result.put(key, get(key)); + result.put(normalize(key), get(key)); } return result; } @@ -484,4 +515,27 @@ public class Arguments { public boolean silenced() { return silent; } + + public Arguments copy() { + return new Arguments(provider, keys); + } + + /** + * Returns a new arguments instance that translates requests for a {@code "key"} to {@code "prefix_key"}. + */ + public Arguments withPrefix(String prefix) { + return fromPrefixed(provider, keys, prefix, "_", false); + } + + /** Returns a view of this instance, that only supports requests for {@code allowedKeys}. */ + public Arguments subset(String... allowedKeys) { + Set allowed = new HashSet<>(); + for (String key : allowedKeys) { + allowed.add(normalize(key)); + } + return new Arguments( + key -> allowed.contains(normalize(key)) ? provider.apply(key) : null, + () -> keys.get().stream().filter(key -> allowed.contains(normalize(key))).toList() + ); + } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java index ec4f0562..ba50f9db 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java @@ -24,8 +24,6 @@ public record PlanetilerConfig( int minzoom, int maxzoom, int maxzoomForRendering, - boolean skipIndexCreation, - boolean optimizeDb, boolean force, boolean gzipTempStorage, boolean mmapTempStorage, @@ -47,7 +45,6 @@ public record PlanetilerConfig( double simplifyToleranceAtMaxZoom, double simplifyToleranceBelowMaxZoom, boolean osmLazyReads, - boolean compactDb, boolean skipFilledTiles, int tileWarningSizeBytes, Boolean color @@ -125,8 +122,6 @@ public record PlanetilerConfig( minzoom, maxzoom, renderMaxzoom, - arguments.getBoolean("skip_mbtiles_index_creation", "skip adding index to mbtiles file", false), - arguments.getBoolean("optimize_db", "Vacuum analyze mbtiles after writing", false), arguments.getBoolean("force", "overwriting output file and ignore disk/RAM warnings", false), arguments.getBoolean("gzip_temp", "gzip temporary feature storage (uses more CPU, but less disk space)", false), arguments.getBoolean("mmap_temp", "use memory-mapped IO for temp feature files", true), @@ -168,9 +163,6 @@ public record PlanetilerConfig( arguments.getBoolean("osm_lazy_reads", "Read OSM blocks from disk in worker threads", true), - arguments.getBoolean("compact_db", - "Reduce the DB size by separating and deduping the tile data", - true), arguments.getBoolean("skip_filled_tiles", "Skip writing tiles containing only polygon fills to the output", false), diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java index 66a5f16d..723f6900 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java @@ -1,6 +1,7 @@ package com.onthegomap.planetiler.mbtiles; import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_ABSENT; +import static com.onthegomap.planetiler.util.Format.joinCoordinates; import com.carrotsearch.hppc.LongIntHashMap; import com.fasterxml.jackson.annotation.JsonProperty; @@ -11,14 +12,14 @@ import com.onthegomap.planetiler.archive.ReadableTileArchive; import com.onthegomap.planetiler.archive.TileArchiveMetadata; import com.onthegomap.planetiler.archive.TileEncodingResult; import com.onthegomap.planetiler.archive.WriteableTileArchive; -import com.onthegomap.planetiler.config.PlanetilerConfig; -import com.onthegomap.planetiler.geo.GeoUtils; +import com.onthegomap.planetiler.config.Arguments; import com.onthegomap.planetiler.geo.TileCoord; import com.onthegomap.planetiler.geo.TileOrder; import com.onthegomap.planetiler.reader.FileFormatException; import com.onthegomap.planetiler.util.CloseableIterator; import com.onthegomap.planetiler.util.Format; import com.onthegomap.planetiler.util.LayerStats; +import com.onthegomap.planetiler.util.Parse; import java.io.IOException; import java.nio.file.Path; import java.sql.Connection; @@ -27,21 +28,20 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.text.NumberFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.NoSuchElementException; import java.util.Objects; import java.util.OptionalLong; import java.util.TreeMap; import java.util.stream.Collectors; -import java.util.stream.DoubleStream; import java.util.stream.IntStream; -import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.CoordinateXY; import org.locationtech.jts.geom.Envelope; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,6 +54,17 @@ import org.sqlite.SQLiteConfig; */ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive { + // Options that can be set through "file.mbtiles?compact=true" query parameters + // or "file.mbtiles" with "--mbtiles-compact=true" command-line flag + public static final String COMPACT_DB = "compact"; + public static final String SKIP_INDEX_CREATION = "no_index"; + public static final String VACUUM_ANALYZE = "vacuum_analyze"; + + public static final String LEGACY_COMPACT_DB = "compact_db"; + public static final String LEGACY_SKIP_INDEX_CREATION = "skip_mbtiles_index_creation"; + public static final String LEGACY_VACUUM_ANALYZE = "optimize_db"; + + // https://www.sqlite.org/src/artifact?ci=trunk&filename=magic.txt private static final int MBTILES_APPLICATION_ID = 0x4d504258; @@ -93,72 +104,115 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive private final Connection connection; private final boolean compactDb; + private final boolean skipIndexCreation; + private final boolean vacuumAnalyze; private PreparedStatement getTileStatement = null; - private Mbtiles(Connection connection, boolean compactDb) { + private Mbtiles(Connection connection, Arguments arguments) { this.connection = connection; - this.compactDb = compactDb; + this.compactDb = arguments.getBoolean( + COMPACT_DB + "|" + LEGACY_COMPACT_DB, + "mbtiles: reduce the DB size by separating and deduping the tile data", + true + ); + this.skipIndexCreation = arguments.getBoolean( + SKIP_INDEX_CREATION + "|" + LEGACY_SKIP_INDEX_CREATION, + "mbtiles: skip adding index to sqlite DB", + false + ); + this.vacuumAnalyze = arguments.getBoolean( + VACUUM_ANALYZE + "|" + LEGACY_VACUUM_ANALYZE, + "mbtiles: vacuum analyze sqlite DB after writing", + false + ); } /** Returns a new mbtiles file that won't get written to disk. Useful for toy use-cases like unit tests. */ public static Mbtiles newInMemoryDatabase(boolean compactDb) { - try { - SQLiteConfig config = new SQLiteConfig(); - config.setApplicationId(MBTILES_APPLICATION_ID); - return new Mbtiles(DriverManager.getConnection("jdbc:sqlite::memory:", config.toProperties()), compactDb); - } catch (SQLException throwables) { - throw new IllegalStateException("Unable to create in-memory database", throwables); - } + return newInMemoryDatabase(Arguments.of(COMPACT_DB, compactDb ? "true" : "false")); } - /** @see {@link #newInMemoryDatabase(boolean)} */ + /** Returns an in-memory database with extra mbtiles and pragma options set from {@code options}. */ + public static Mbtiles newInMemoryDatabase(Arguments options) { + SQLiteConfig config = new SQLiteConfig(); + config.setApplicationId(MBTILES_APPLICATION_ID); + return new Mbtiles(newConnection("jdbc:sqlite::memory:", config, options), options); + } + + /** Alias for {@link #newInMemoryDatabase(boolean)} */ public static Mbtiles newInMemoryDatabase() { return newInMemoryDatabase(true); } - /** Returns a new connection to an mbtiles file optimized for fast bulk writes. */ - public static Mbtiles newWriteToFileDatabase(Path path, boolean compactDb) { - try { - SQLiteConfig config = new SQLiteConfig(); - config.setJournalMode(SQLiteConfig.JournalMode.OFF); - config.setSynchronous(SQLiteConfig.SynchronousMode.OFF); - config.setCacheSize(1_000_000); // 1GB - config.setLockingMode(SQLiteConfig.LockingMode.EXCLUSIVE); - config.setTempStore(SQLiteConfig.TempStore.MEMORY); - config.setApplicationId(MBTILES_APPLICATION_ID); - return new Mbtiles(DriverManager.getConnection("jdbc:sqlite:" + path.toAbsolutePath(), config.toProperties()), - compactDb); - } catch (SQLException throwables) { - throw new IllegalArgumentException("Unable to open " + path, throwables); - } + /** + * Returns a new connection to an mbtiles file optimized for fast bulk writes with extra mbtiles and pragma options + * set from {@code options}. + */ + public static Mbtiles newWriteToFileDatabase(Path path, Arguments options) { + Objects.requireNonNull(path); + SQLiteConfig sqliteConfig = new SQLiteConfig(); + sqliteConfig.setJournalMode(SQLiteConfig.JournalMode.OFF); + sqliteConfig.setSynchronous(SQLiteConfig.SynchronousMode.OFF); + sqliteConfig.setCacheSize(1_000_000); // 1GB + sqliteConfig.setLockingMode(SQLiteConfig.LockingMode.EXCLUSIVE); + sqliteConfig.setTempStore(SQLiteConfig.TempStore.MEMORY); + sqliteConfig.setApplicationId(MBTILES_APPLICATION_ID); + var connection = newConnection("jdbc:sqlite:" + path.toAbsolutePath(), sqliteConfig, options); + return new Mbtiles(connection, options); } /** Returns a new connection to an mbtiles file optimized for reads. */ public static Mbtiles newReadOnlyDatabase(Path path) { + return newReadOnlyDatabase(path, Arguments.of()); + } + + /** + * Returns a new connection to an mbtiles file optimized for reads with extra mbtiles and pragma options set from + * {@code options}. + */ + public static Mbtiles newReadOnlyDatabase(Path path, Arguments options) { + Objects.requireNonNull(path); + SQLiteConfig config = new SQLiteConfig(); + config.setReadOnly(true); + config.setCacheSize(100_000); + config.setLockingMode(SQLiteConfig.LockingMode.EXCLUSIVE); + config.setPageSize(32_768); + // helps with 3 or more threads concurrently accessing: + // config.setOpenMode(SQLiteOpenMode.NOMUTEX); + Connection connection = newConnection("jdbc:sqlite:" + path.toAbsolutePath(), config, options); + return new Mbtiles(connection, options); + } + + private static Connection newConnection(String url, SQLiteConfig defaults, Arguments args) { try { - SQLiteConfig config = new SQLiteConfig(); - config.setReadOnly(true); - config.setCacheSize(100_000); - config.setLockingMode(SQLiteConfig.LockingMode.EXCLUSIVE); - config.setPageSize(32_768); - // helps with 3 or more threads concurrently accessing: - // config.setOpenMode(SQLiteOpenMode.NOMUTEX); - Connection connection = DriverManager - .getConnection("jdbc:sqlite:" + path.toAbsolutePath(), config.toProperties()); - return new Mbtiles(connection, false /* in read-only mode, it's irrelevant if compact or not */); + args = args.copy().silence(); + var config = new SQLiteConfig(defaults.toProperties()); + for (var pragma : SQLiteConfig.Pragma.values()) { + var value = args.getString(pragma.getPragmaName(), pragma.getPragmaName(), null); + if (value != null) { + LOGGER.info("Setting custom mbtiles sqlite pragma {}={}", pragma.getPragmaName(), value); + config.setPragma(pragma, value); + } + } + return DriverManager.getConnection(url, config.toProperties()); } catch (SQLException throwables) { - throw new IllegalArgumentException("Unable to open " + path, throwables); + throw new IllegalArgumentException("Unable to open " + url, throwables); } } + @Override + public boolean deduplicates() { + return compactDb; + } + @Override public TileOrder tileOrder() { return TileOrder.TMS; } @Override - public void initialize(PlanetilerConfig config, TileArchiveMetadata tileArchiveMetadata, LayerStats layerStats) { - if (config.skipIndexCreation()) { + public void initialize(TileArchiveMetadata tileArchiveMetadata) { + if (skipIndexCreation) { createTablesWithoutIndexes(); if (LOGGER.isInfoEnabled()) { LOGGER.info("Skipping index creation. Add later by executing: {}", @@ -168,26 +222,12 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive createTablesWithIndexes(); } - var metadata = metadata() - .setName(tileArchiveMetadata.name()) - .setFormat("pbf") - .setDescription(tileArchiveMetadata.description()) - .setAttribution(tileArchiveMetadata.attribution()) - .setVersion(tileArchiveMetadata.version()) - .setType(tileArchiveMetadata.type()) - .setBoundsAndCenter(config.bounds().latLon()) - .setMinzoom(config.minzoom()) - .setMaxzoom(config.maxzoom()) - .setJson(new MetadataJson(layerStats.getTileStats())); - - for (var entry : tileArchiveMetadata.planetilerSpecific().entrySet()) { - metadata.setMetadata(entry.getKey(), entry.getValue()); - } + metadataTable().set(tileArchiveMetadata); } @Override - public void finish(PlanetilerConfig config) { - if (config.optimizeDb()) { + public void finish(TileArchiveMetadata tileArchiveMetadata) { + if (vacuumAnalyze) { vacuumAnalyze(); } } @@ -341,9 +381,9 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive return newTileWriter(); } - /** Returns the contents of the metadata table. */ - public Metadata metadata() { - return new Metadata(); + @Override + public TileArchiveMetadata metadata() { + return new Metadata().get(); } /** Returns the contents of the metadata table. */ @@ -389,12 +429,21 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive return connection; } + public boolean skipIndexCreation() { + return skipIndexCreation; + } + + public boolean compactDb() { + return compactDb; + } + /** * Data contained in the {@code json} row of the metadata table * * @see MBtiles * schema */ + // TODO add tilestats public record MetadataJson( @JsonProperty("vector_layers") List vectorLayers ) { @@ -405,7 +454,7 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive public static MetadataJson fromJson(String json) { try { - return objectMapper.readValue(json, MetadataJson.class); + return json == null ? null : objectMapper.readValue(json, MetadataJson.class); } catch (JsonProcessingException e) { throw new IllegalStateException("Invalid metadata json: " + json, e); } @@ -466,6 +515,7 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive /** Contents of a row of the tiles_data table. */ private record TileDataEntry(int tileDataId, byte[] tileData) { + @Override public String toString() { return "TileDataEntry [tileDataId=" + tileDataId + ", tileData=" + Arrays.toString(tileData) + "]"; @@ -494,6 +544,7 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive /** Iterates through tile coordinates one at a time without materializing the entire list in memory. */ private class TileCoordIterator implements CloseableIterator { + private final Statement statement; private final ResultSet rs; private boolean hasNext = false; @@ -568,7 +619,7 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive insertStmtTableName = tableName; insertStmtInsertIgnore = insertIgnore; insertStmtValuesPlaceHolder = columns.stream().map(c -> "?").collect(Collectors.joining(",", "(", ")")); - insertStmtColumnsCsv = columns.stream().collect(Collectors.joining(",")); + insertStmtColumnsCsv = String.join(",", columns); batchStatement = createBatchInsertPreparedStatement(batchLimit); } @@ -779,16 +830,7 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive /** Data contained in the metadata table. */ public class Metadata { - private static final NumberFormat nf = NumberFormat.getNumberInstance(Locale.US); - - static { - nf.setMaximumFractionDigits(5); - } - - private static String join(double... items) { - return DoubleStream.of(items).mapToObj(nf::format).collect(Collectors.joining(",")); - } - + /** Inserts a row into the metadata table that sets {@code name=value}. */ public Metadata setMetadata(String name, Object value) { if (value != null) { String stringValue = value.toString(); @@ -810,79 +852,15 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive return this; } - public Metadata setName(String value) { - return setMetadata("name", value); - } - - /** Format of the tile data, should always be pbf {@code pbf}. */ - public Metadata setFormat(String format) { - return setMetadata("format", format); - } - - public Metadata setBounds(double left, double bottom, double right, double top) { - return setMetadata("bounds", join(left, bottom, right, top)); - } - - public Metadata setBounds(Envelope envelope) { - return setBounds(envelope.getMinX(), envelope.getMinY(), envelope.getMaxX(), envelope.getMaxY()); - } - - public Metadata setCenter(double longitude, double latitude, double zoom) { - return setMetadata("center", join(longitude, latitude, zoom)); - } - - public Metadata setBoundsAndCenter(Envelope envelope) { - return setBounds(envelope).setCenter(envelope); - } - - /** Estimate a reasonable center for the map to fit an envelope. */ - public Metadata setCenter(Envelope envelope) { - Coordinate center = envelope.centre(); - double zoom = Math.ceil(GeoUtils.getZoomFromLonLatBounds(envelope)); - return setCenter(center.x, center.y, zoom); - } - - public Metadata setMinzoom(int value) { - return setMetadata("minzoom", value); - } - - public Metadata setMaxzoom(int maxZoom) { - return setMetadata("maxzoom", maxZoom); - } - - public Metadata setAttribution(String value) { - return setMetadata("attribution", value); - } - - public Metadata setDescription(String value) { - return setMetadata("description", value); - } - - /** {@code overlay} or {@code baselayer}. */ - public Metadata setType(String value) { - return setMetadata("type", value); - } - - public Metadata setTypeIsOverlay() { - return setType("overlay"); - } - - public Metadata setTypeIsBaselayer() { - return setType("baselayer"); - } - - public Metadata setVersion(String value) { - return setMetadata("version", value); - } - - public Metadata setJson(String value) { - return setMetadata("json", value); - } - + /** + * Inserts a row into the metadata table that sets the value for {@code "json"} key to {@code value} serialized as a + * string. + */ public Metadata setJson(MetadataJson value) { - return value == null ? this : setJson(value.toJson()); + return value == null ? this : setMetadata("json", value.toJson()); } + /** Returns all key-value pairs from the metadata table. */ public Map getAll() { TreeMap result = new TreeMap<>(); try (Statement statement = connection.createStatement()) { @@ -895,10 +873,86 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive ); } } catch (SQLException throwables) { - LOGGER.warn("Error retrieving metadata: " + throwables); + LOGGER.warn("Error retrieving metadata: {}", throwables.toString()); LOGGER.trace("Error retrieving metadata details: ", throwables); } return result; } + + /** + * Inserts rows into the metadata table that set all of the well-known metadata keys from + * {@code tileArchiveMetadata} and passes through the raw values of any options not explicitly called out in the + * MBTiles specification. + * + * @see MBTiles 1.3 + * specification + */ + public Metadata set(TileArchiveMetadata tileArchiveMetadata) { + var map = new LinkedHashMap<>(tileArchiveMetadata.toMap()); + + setMetadata(TileArchiveMetadata.FORMAT_KEY, tileArchiveMetadata.format()); + var center = tileArchiveMetadata.center(); + var zoom = tileArchiveMetadata.zoom(); + if (center != null) { + if (zoom != null) { + setMetadata(TileArchiveMetadata.CENTER_KEY, joinCoordinates(center.x, center.y, Math.ceil(zoom))); + } else { + setMetadata(TileArchiveMetadata.CENTER_KEY, joinCoordinates(center.x, center.y)); + } + } + var bounds = tileArchiveMetadata.bounds(); + if (bounds != null) { + setMetadata(TileArchiveMetadata.BOUNDS_KEY, + joinCoordinates(bounds.getMinX(), bounds.getMinY(), bounds.getMaxX(), bounds.getMaxY())); + } + setJson(new MetadataJson(tileArchiveMetadata.vectorLayers())); + + map.remove(TileArchiveMetadata.FORMAT_KEY); + map.remove(TileArchiveMetadata.CENTER_KEY); + map.remove(TileArchiveMetadata.ZOOM_KEY); + map.remove(TileArchiveMetadata.BOUNDS_KEY); + map.remove(TileArchiveMetadata.VECTOR_LAYERS_KEY); + + for (var entry : map.entrySet()) { + setMetadata(entry.getKey(), entry.getValue()); + } + return this; + } + + /** + * Returns a {@link TileArchiveMetadata} instance parsed from all the rows in the metadata table. + */ + public TileArchiveMetadata get() { + Map map = new HashMap<>(getAll()); + String[] bounds = map.containsKey(TileArchiveMetadata.BOUNDS_KEY) ? + map.remove(TileArchiveMetadata.BOUNDS_KEY).split(",") : null; + String[] center = map.containsKey(TileArchiveMetadata.CENTER_KEY) ? + map.remove(TileArchiveMetadata.CENTER_KEY).split(",") : null; + var metadataJson = MetadataJson.fromJson(map.remove("json")); + return new TileArchiveMetadata( + map.remove(TileArchiveMetadata.NAME_KEY), + map.remove(TileArchiveMetadata.DESCRIPTION_KEY), + map.remove(TileArchiveMetadata.ATTRIBUTION_KEY), + map.remove(TileArchiveMetadata.VERSION_KEY), + map.remove(TileArchiveMetadata.TYPE_KEY), + map.remove(TileArchiveMetadata.FORMAT_KEY), + bounds == null || bounds.length < 4 ? null : new Envelope( + Double.parseDouble(bounds[0]), + Double.parseDouble(bounds[2]), + Double.parseDouble(bounds[1]), + Double.parseDouble(bounds[3]) + ), + center == null || center.length < 2 ? null : new CoordinateXY( + Double.parseDouble(center[0]), + Double.parseDouble(center[1]) + ), + center == null || center.length < 3 ? null : Double.parseDouble(center[2]), + Parse.parseIntOrNull(map.remove(TileArchiveMetadata.MINZOOM_KEY)), + Parse.parseIntOrNull(map.remove(TileArchiveMetadata.MAXZOOM_KEY)), + metadataJson == null ? null : metadataJson.vectorLayers, + // any left-overs: + map + ); + } } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java index 8fa73fd6..99a43f15 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java @@ -189,7 +189,7 @@ public class Verify { } private void checkBasicStructure() { - check("contains name attribute", () -> mbtiles.metadata().getAll().containsKey("name")); + check("contains name attribute", () -> mbtiles.metadata().toMap().containsKey("name")); check("contains at least one tile", () -> mbtiles.getAllTileCoords().stream().findAny().isPresent()); checkWithMessage("all tiles are valid", () -> { List invalidTiles = mbtiles.getAllTileCoords().stream() diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/Pmtiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/Pmtiles.java index d05716ec..107ce094 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/Pmtiles.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/Pmtiles.java @@ -23,6 +23,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import org.locationtech.jts.geom.CoordinateXY; +import org.locationtech.jts.geom.Envelope; public class Pmtiles { public enum Compression { @@ -201,6 +203,22 @@ public class Pmtiles { throw new FileFormatException("Failed to read enough bytes for PMTiles header."); } } + + public Envelope bounds() { + return new Envelope( + minLonE7 / 1e7, + maxLonE7 / 1e7, + minLatE7 / 1e7, + maxLatE7 / 1e7 + ); + } + + public CoordinateXY center() { + return new CoordinateXY( + centerLonE7 / 1e7, + centerLatE7 / 1e7 + ); + } } public static final class Entry implements Comparable { @@ -366,7 +384,7 @@ public class Pmtiles { try { return objectMapper.readValue(bytes, JsonMetadata.class); } catch (IOException e) { - throw new IllegalStateException("Invalid metadata json: " + bytes, e); + throw new IllegalStateException("Invalid metadata json: " + new String(bytes, StandardCharsets.UTF_8), e); } } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/ReadablePmtiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/ReadablePmtiles.java index 05d0c265..fbabc851 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/ReadablePmtiles.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/ReadablePmtiles.java @@ -1,13 +1,19 @@ package com.onthegomap.planetiler.pmtiles; import com.onthegomap.planetiler.archive.ReadableTileArchive; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; import com.onthegomap.planetiler.geo.TileCoord; import com.onthegomap.planetiler.util.CloseableIterator; import com.onthegomap.planetiler.util.Gzip; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import java.nio.channels.SeekableByteChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -22,6 +28,10 @@ public class ReadablePmtiles implements ReadableTileArchive { this.header = Pmtiles.Header.fromBytes(getBytes(0, Pmtiles.HEADER_LEN)); } + public static ReadableTileArchive newReadFromFile(Path path) throws IOException { + return new ReadablePmtiles(FileChannel.open(path, StandardOpenOption.READ)); + } + private synchronized byte[] getBytes(long start, int length) throws IOException { channel.position(start); var buf = ByteBuffer.allocate(length); @@ -103,6 +113,34 @@ public class ReadablePmtiles implements ReadableTileArchive { return Pmtiles.JsonMetadata.fromBytes(buf); } + @Override + public TileArchiveMetadata metadata() { + try { + var jsonMetadata = getJsonMetadata(); + var map = new LinkedHashMap<>(jsonMetadata.otherMetadata()); + return new TileArchiveMetadata( + map.remove(TileArchiveMetadata.NAME_KEY), + map.remove(TileArchiveMetadata.DESCRIPTION_KEY), + map.remove(TileArchiveMetadata.ATTRIBUTION_KEY), + map.remove(TileArchiveMetadata.VERSION_KEY), + map.remove(TileArchiveMetadata.TYPE_KEY), + switch (header.tileType()) { + case MVT -> TileArchiveMetadata.MVT_FORMAT; + default -> null; + }, + header.bounds(), + header.center(), + (double) header.centerZoom(), + (int) header.minZoom(), + (int) header.maxZoom(), + jsonMetadata.vectorLayers(), + map + ); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + private static class TileCoordIterator implements CloseableIterator { private final Stream stream; private final Iterator iterator; diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/WriteablePmtiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/WriteablePmtiles.java index ed541ed6..e99b4b6b 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/WriteablePmtiles.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/WriteablePmtiles.java @@ -12,7 +12,6 @@ import com.onthegomap.planetiler.geo.TileCoord; import com.onthegomap.planetiler.geo.TileOrder; import com.onthegomap.planetiler.util.Format; import com.onthegomap.planetiler.util.Gzip; -import com.onthegomap.planetiler.util.LayerStats; import com.onthegomap.planetiler.util.SeekableInMemoryByteChannel; import java.io.IOException; import java.io.UncheckedIOException; @@ -24,10 +23,10 @@ import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; import java.util.Objects; import java.util.OptionalLong; -import org.locationtech.jts.geom.Envelope; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,8 +45,6 @@ public final class WriteablePmtiles implements WriteableTileArchive { private long currentOffset = 0; private long numUnhashedTiles = 0; private long numAddressedTiles = 0; - private LayerStats layerStats; - private TileArchiveMetadata tileArchiveMetadata; private boolean isClustered = true; private WriteablePmtiles(SeekableByteChannel channel) throws IOException { @@ -91,7 +88,7 @@ public final class WriteablePmtiles implements WriteableTileArchive { * @return byte arrays of the root and all leaf directories, and the # of leaves. * @throws IOException if compression fails */ - protected static Directories makeDirectories(List entries) throws IOException { + static Directories makeDirectories(List entries) throws IOException { int maxEntriesRootOnly = 16384; int attemptNum = 1; if (entries.size() < maxEntriesRootOnly) { @@ -124,19 +121,18 @@ public final class WriteablePmtiles implements WriteableTileArchive { return new WriteablePmtiles(bytes); } + @Override + public boolean deduplicates() { + return true; + } + @Override public TileOrder tileOrder() { return TileOrder.HILBERT; } @Override - public void initialize(PlanetilerConfig config, TileArchiveMetadata tileArchiveMetadata, LayerStats layerStats) { - this.layerStats = layerStats; - this.tileArchiveMetadata = tileArchiveMetadata; - } - - @Override - public void finish(PlanetilerConfig config) { + public void finish(TileArchiveMetadata tileArchiveMetadata) { if (!isClustered) { LOGGER.info("Tile data was not written in order, sorting entries..."); Collections.sort(entries); @@ -144,10 +140,32 @@ public final class WriteablePmtiles implements WriteableTileArchive { } try { Directories directories = makeDirectories(entries); - byte[] jsonBytes = new Pmtiles.JsonMetadata(layerStats.getTileStats(), tileArchiveMetadata.getAll()).toBytes(); + var otherMetadata = new LinkedHashMap<>(tileArchiveMetadata.toMap()); + + // exclude keys included in top-level header + otherMetadata.remove(TileArchiveMetadata.CENTER_KEY); + otherMetadata.remove(TileArchiveMetadata.ZOOM_KEY); + otherMetadata.remove(TileArchiveMetadata.BOUNDS_KEY); + otherMetadata.remove(TileArchiveMetadata.FORMAT_KEY); + otherMetadata.remove(TileArchiveMetadata.MINZOOM_KEY); + otherMetadata.remove(TileArchiveMetadata.MAXZOOM_KEY); + otherMetadata.remove(TileArchiveMetadata.VECTOR_LAYERS_KEY); + + byte[] jsonBytes = + new Pmtiles.JsonMetadata(tileArchiveMetadata.vectorLayers(), otherMetadata).toBytes(); jsonBytes = Gzip.gzip(jsonBytes); - Envelope envelope = config.bounds().latLon(); + String formatString = tileArchiveMetadata.format(); + var outputFormat = + TileArchiveMetadata.MVT_FORMAT.equals(formatString) ? Pmtiles.TileType.MVT : Pmtiles.TileType.UNKNOWN; + + var bounds = tileArchiveMetadata.bounds() == null ? GeoUtils.WORLD_LAT_LON_BOUNDS : tileArchiveMetadata.bounds(); + var center = tileArchiveMetadata.center() == null ? bounds.centre() : tileArchiveMetadata.center(); + int zoom = (int) Math.ceil(tileArchiveMetadata.zoom() == null ? GeoUtils.getZoomFromLonLatBounds(bounds) : + tileArchiveMetadata.zoom()); + int minzoom = tileArchiveMetadata.minzoom() == null ? 0 : tileArchiveMetadata.minzoom(); + int maxzoom = + tileArchiveMetadata.maxzoom() == null ? PlanetilerConfig.MAX_MAXZOOM : tileArchiveMetadata.maxzoom(); Pmtiles.Header header = new Pmtiles.Header( (byte) 3, @@ -165,16 +183,16 @@ public final class WriteablePmtiles implements WriteableTileArchive { isClustered, Pmtiles.Compression.GZIP, Pmtiles.Compression.GZIP, - Pmtiles.TileType.MVT, - (byte) config.minzoom(), - (byte) config.maxzoom(), - (int) (envelope.getMinX() * 10_000_000), - (int) (envelope.getMinY() * 10_000_000), - (int) (envelope.getMaxX() * 10_000_000), - (int) (envelope.getMaxY() * 10_000_000), - (byte) Math.ceil(GeoUtils.getZoomFromLonLatBounds(envelope)), - (int) ((envelope.getMinX() + envelope.getMaxX()) / 2 * 10_000_000), - (int) ((envelope.getMinY() + envelope.getMaxY()) / 2 * 10_000_000) + outputFormat, + (byte) minzoom, + (byte) maxzoom, + (int) (bounds.getMinX() * 10_000_000), + (int) (bounds.getMinY() * 10_000_000), + (int) (bounds.getMaxX() * 10_000_000), + (int) (bounds.getMaxY() * 10_000_000), + (byte) zoom, + (int) center.x * 10_000_000, + (int) center.y * 10_000_000 ); LOGGER.info("Writing metadata and leaf directories..."); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java index 75e55405..e0891438 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java @@ -43,15 +43,17 @@ public interface Stats extends AutoCloseable { */ default void printSummary() { Format format = Format.defaultInstance(); - Logger LOGGER = LoggerFactory.getLogger(getClass()); - LOGGER.info(""); - LOGGER.info("-".repeat(40)); - timers().printSummary(); - LOGGER.info("-".repeat(40)); - for (var entry : monitoredFiles().entrySet()) { - long size = FileUtils.size(entry.getValue()); - if (size > 0) { - LOGGER.info("\t" + entry.getKey() + "\t" + format.storage(size, false) + "B"); + Logger logger = LoggerFactory.getLogger(getClass()); + if (logger.isInfoEnabled()) { + logger.info(""); + logger.info("-".repeat(40)); + timers().printSummary(); + logger.info("-".repeat(40)); + for (var entry : monitoredFiles().entrySet()) { + long size = FileUtils.size(entry.getValue()); + if (size > 0) { + logger.info("\t{}\t{}B", entry.getKey(), format.storage(size, false)); + } } } } @@ -110,7 +112,9 @@ public interface Stats extends AutoCloseable { /** Adds a stat that will track the size of a file or directory located at {@code path}. */ default void monitorFile(String name, Path path) { - monitoredFiles().put(name, path); + if (path != null) { + monitoredFiles().put(name, path); + } } /** Adds a stat that will track the estimated in-memory size of {@code object}. */ diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/FileUtils.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/FileUtils.java index ccca9b45..e78c7a64 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/FileUtils.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/FileUtils.java @@ -217,17 +217,19 @@ public class FileUtils { */ public static void createParentDirectories(Path... paths) { for (var path : paths) { - try { - if (Files.isDirectory(path) && !Files.exists(path)) { - Files.createDirectories(path); - } else { - Path parent = path.getParent(); - if (parent != null && !Files.exists(parent)) { - Files.createDirectories(parent); + if (path != null) { + try { + if (Files.isDirectory(path) && !Files.exists(path)) { + Files.createDirectories(path); + } else { + Path parent = path.getParent(); + if (parent != null && !Files.exists(parent)) { + Files.createDirectories(parent); + } } + } catch (IOException e) { + throw new IllegalStateException("Unable to create parent directories " + path, e); } - } catch (IOException e) { - throw new IllegalStateException("Unable to create parent directories " + path, e); } } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Format.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Format.java index cd8a4a2a..6614766f 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Format.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Format.java @@ -8,6 +8,8 @@ import java.util.NavigableMap; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.stream.Collectors; +import java.util.stream.DoubleStream; import org.apache.commons.text.StringEscapeUtils; import org.locationtech.jts.geom.Coordinate; @@ -19,6 +21,26 @@ public class Format { public static final Locale DEFAULT_LOCALE = Locale.getDefault(Locale.Category.FORMAT); private static final ConcurrentMap instances = new ConcurrentHashMap<>(); + @SuppressWarnings("java:S5164") + private static final NumberFormat latLonNF = NumberFormat.getNumberInstance(Locale.US); + private static final NavigableMap STORAGE_SUFFIXES = new TreeMap<>(Map.ofEntries( + Map.entry(1_000L, "k"), + Map.entry(1_000_000L, "M"), + Map.entry(1_000_000_000L, "G"), + Map.entry(1_000_000_000_000L, "T"), + Map.entry(1_000_000_000_000_000L, "P") + )); + private static final NavigableMap NUMERIC_SUFFIXES = new TreeMap<>(Map.ofEntries( + Map.entry(1_000L, "k"), + Map.entry(1_000_000L, "M"), + Map.entry(1_000_000_000L, "B"), + Map.entry(1_000_000_000_000L, "T"), + Map.entry(1_000_000_000_000_000L, "Q") + )); + + static { + latLonNF.setMaximumFractionDigits(5); + } // `NumberFormat` instances are not thread safe, so we need to wrap them inside a `ThreadLocal`. // @@ -49,6 +71,11 @@ public class Format { }); } + /** Returns a string with {@code items} rounded to 5 decimals and joined with a comma. */ + public static synchronized String joinCoordinates(double... items) { + return DoubleStream.of(items).mapToObj(latLonNF::format).collect(Collectors.joining(",")); + } + public static Format forLocale(Locale locale) { return instances.computeIfAbsent(locale, Format::new); } @@ -57,21 +84,6 @@ public class Format { return forLocale(DEFAULT_LOCALE); } - private static final NavigableMap STORAGE_SUFFIXES = new TreeMap<>(Map.ofEntries( - Map.entry(1_000L, "k"), - Map.entry(1_000_000L, "M"), - Map.entry(1_000_000_000L, "G"), - Map.entry(1_000_000_000_000L, "T"), - Map.entry(1_000_000_000_000_000L, "P") - )); - private static final NavigableMap NUMERIC_SUFFIXES = new TreeMap<>(Map.ofEntries( - Map.entry(1_000L, "k"), - Map.entry(1_000_000L, "M"), - Map.entry(1_000_000_000L, "B"), - Map.entry(1_000_000_000_000L, "T"), - Map.entry(1_000_000_000_000_000L, "Q") - )); - public static String padRight(String str, int size) { StringBuilder strBuilder = new StringBuilder(str); while (strBuilder.length() < size) { @@ -88,6 +100,23 @@ public class Format { return strBuilder.toString(); } + /** Returns Java code that can re-create {@code string}: {@code null} if null, or {@code "contents"} if not empty. */ + public static String quote(Object string) { + if (string == null) { + return "null"; + } + return '"' + StringEscapeUtils.escapeJava(string.toString()) + '"'; + } + + /** Returns an openstreetmap.org map link for a lat/lon */ + public static String osmDebugUrl(int zoom, Coordinate coord) { + return "https://www.openstreetmap.org/#map=%d/%.5f/%.5f".formatted( + zoom, + coord.y, + coord.x + ); + } + /** Returns a number of bytes formatted like "123" "1.2k" "240M", etc. */ public String storage(Number num, boolean pad) { return format(num, pad, STORAGE_SUFFIXES); @@ -161,21 +190,4 @@ public class Format { } return simplified.toString().replace("PT", "").toLowerCase(Locale.ROOT); } - - /** Returns Java code that can re-create {@code string}: {@code null} if null, or {@code "contents"} if not empty. */ - public static String quote(Object string) { - if (string == null) { - return "null"; - } - return '"' + StringEscapeUtils.escapeJava(string.toString()) + '"'; - } - - /** Returns an openstreetmap.org map link for a lat/lon */ - public static String osmDebugUrl(int zoom, Coordinate coord) { - return "https://www.openstreetmap.org/#map=%d/%.5f/%.5f".formatted( - zoom, - coord.y, - coord.x - ); - } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/ResourceUsage.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/ResourceUsage.java index b21182ee..1cc3fa03 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/ResourceUsage.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/ResourceUsage.java @@ -70,7 +70,7 @@ public class ResourceUsage { /** Requests {@code amount} bytes on the file system that contains {@code path}. */ public ResourceUsage addDisk(Path path, long amount, String description) { - return add(new DiskUsage(path), amount, description); + return path == null ? this : add(new DiskUsage(path), amount, description); } /** Requests {@code amount} bytes of RAM in the JVM heap. */ diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java index 1769797a..19e9e40f 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java @@ -15,6 +15,7 @@ import com.onthegomap.planetiler.geo.GeometryException; import com.onthegomap.planetiler.geo.TileCoord; import com.onthegomap.planetiler.geo.TileOrder; import com.onthegomap.planetiler.mbtiles.Mbtiles; +import com.onthegomap.planetiler.pmtiles.ReadablePmtiles; import com.onthegomap.planetiler.reader.SimpleFeature; import com.onthegomap.planetiler.reader.SimpleReader; import com.onthegomap.planetiler.reader.SourceFeature; @@ -141,14 +142,14 @@ class PlanetilerTests { FeatureGroup featureGroup = FeatureGroup.newInMemoryFeatureGroup(TileOrder.TMS, profile, stats); runner.run(featureGroup, profile, config); featureGroup.prepare(); - try (Mbtiles db = Mbtiles.newInMemoryDatabase(config.compactDb())) { - TileArchiveWriter.writeOutput(featureGroup, db, () -> 0L, new TileArchiveMetadata(profile, config.arguments()), + try (Mbtiles db = Mbtiles.newInMemoryDatabase(config.arguments())) { + TileArchiveWriter.writeOutput(featureGroup, db, () -> 0L, new TileArchiveMetadata(profile, config), config, stats); var tileMap = TestUtils.getTileMap(db); tileMap.values().forEach(fs -> fs.forEach(f -> f.geometry().validate())); - int tileDataCount = config.compactDb() ? TestUtils.getTilesDataCount(db) : 0; - return new PlanetilerResults(tileMap, db.metadata().getAll(), tileDataCount); + int tileDataCount = db.compactDb() ? TestUtils.getTilesDataCount(db) : 0; + return new PlanetilerResults(tileMap, db.metadata().toMap(), tileDataCount); } } @@ -248,20 +249,15 @@ class PlanetilerTests { "format", "pbf", "minzoom", "0", "maxzoom", "14", - "center", "0,0,0", + "center", "0,0", "bounds", "-180,-85.05113,180,85.05113" ), results.metadata); assertSubmap(Map.of( "planetiler:version", BuildInfo.get().version() ), results.metadata); assertSameJson( - """ - { - "vector_layers": [ - ] - } - """, - results.metadata.get("json") + "[]", + results.metadata.get("vector_layers") ); } @@ -269,11 +265,11 @@ class PlanetilerTests { void testOverrideMetadata() throws Exception { var results = runWithReaderFeatures( Map.of( - "mbtiles_name", "override_name", - "mbtiles_description", "override_description", - "mbtiles_attribution", "override_attribution", - "mbtiles_version", "override_version", - "mbtiles_type", "override_type" + "archive_name", "override_name", + "archive_description", "override_description", + "archive_attribution", "override_attribution", + "archive_version", "override_version", + "archive_type", "override_type" ), List.of(), (sourceFeature, features) -> { @@ -331,13 +327,11 @@ class PlanetilerTests { ), results.tiles); assertSameJson( """ - { - "vector_layers": [ - {"id": "layer", "fields": {"name": "String", "attr": "String"}, "minzoom": 13, "maxzoom": 15} - ] - } + [ + {"id": "layer", "fields": {"name": "String", "attr": "String"}, "minzoom": 13, "maxzoom": 15} + ] """, - results.metadata.get("json") + results.metadata.get("vector_layers") ); } @@ -1686,13 +1680,14 @@ class PlanetilerTests { @ValueSource(strings = { "", "--write-threads=2 --process-threads=2 --feature-read-threads=2 --threads=4", - "--emit-tiles-in-order=false", "--free-osm-after-read", "--osm-parse-node-bounds", + "--output-format=pmtiles" }) void testPlanetilerRunner(String args) throws Exception { + boolean pmtiles = args.contains("pmtiles"); Path originalOsm = TestUtils.pathToResource("monaco-latest.osm.pbf"); - Path mbtiles = tempDir.resolve("output.mbtiles"); + Path output = tempDir.resolve(pmtiles ? "output.pmtiles" : "output.mbtiles"); Path tempOsm = tempDir.resolve("monaco-temp.osm.pbf"); Files.copy(originalOsm, tempOsm); Planetiler.create(Arguments.fromArgs( @@ -1710,7 +1705,7 @@ class PlanetilerTests { .addNaturalEarthSource("ne", TestUtils.pathToResource("natural_earth_vector.sqlite")) .addShapefileSource("shapefile", TestUtils.pathToResource("shapefile.zip")) .addGeoPackageSource("geopackage", TestUtils.pathToResource("geopackage.gpkg.zip"), null) - .setOutput("mbtiles", mbtiles) + .setOutput(output) .run(); // make sure it got deleted after write @@ -1718,7 +1713,9 @@ class PlanetilerTests { assertFalse(Files.exists(tempOsm)); } - try (Mbtiles db = Mbtiles.newReadOnlyDatabase(mbtiles)) { + try ( + var db = pmtiles ? ReadablePmtiles.newReadFromFile(output) : Mbtiles.newReadOnlyDatabase(output) + ) { int features = 0; var tileMap = TestUtils.getTileMap(db); for (var tile : tileMap.values()) { @@ -1735,7 +1732,7 @@ class PlanetilerTests { "planetiler:osm:osmosisreplicationtime", "2021-04-21T20:21:46Z", "planetiler:osm:osmosisreplicationseq", "2947", "planetiler:osm:osmosisreplicationurl", "http://download.geofabrik.de/europe/monaco-updates" - ), db.metadata().getAll()); + ), db.metadata().toMap()); } } @@ -1760,7 +1757,7 @@ class PlanetilerTests { .addShapefileGlobSource("shapefile-glob-zip", resourceDir.resolve("shapefile.zip"), "*.shp") // Match *.shp within shapefile.zip .addShapefileSource("shapefile", resourceDir.resolve("shapefile.zip")) - .setOutput("mbtiles", mbtiles) + .setOutput(mbtiles) .run(); try (Mbtiles db = Mbtiles.newReadOnlyDatabase(mbtiles)) { @@ -1806,7 +1803,7 @@ class PlanetilerTests { } }) .addGeoPackageSource("geopackage", TestUtils.pathToResource(inputFile), null) - .setOutput("mbtiles", mbtiles) + .setOutput(mbtiles) .run(); try (Mbtiles db = Mbtiles.newReadOnlyDatabase(mbtiles)) { @@ -1834,7 +1831,7 @@ class PlanetilerTests { .addNaturalEarthSource("ne", TestUtils.pathToResource("natural_earth_vector.sqlite")) .addShapefileSource("shapefile", TestUtils.pathToResource("shapefile.zip")) .addGeoPackageSource("geopackage", TestUtils.pathToResource("geopackage.gpkg.zip"), null) - .setOutput("mbtiles", tempDir.resolve("output.mbtiles")) + .setOutput(tempDir.resolve("output.mbtiles")) .run(); } @@ -1909,9 +1906,8 @@ class PlanetilerTests { private PlanetilerResults runForCompactTest(boolean compactDbEnabled) throws Exception { - return runWithReaderFeatures( - Map.of("threads", "1", "compact-db", Boolean.toString(compactDbEnabled)), + Map.of("threads", "1", "mbtiles-compact", Boolean.toString(compactDbEnabled)), List.of( newReaderFeature(WORLD_POLYGON, Map.of()) ), diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java index 819eac6b..d7de8d7b 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java @@ -15,6 +15,7 @@ import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import com.onthegomap.planetiler.archive.ReadableTileArchive; import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.geo.GeoUtils; import com.onthegomap.planetiler.geo.GeometryException; @@ -199,7 +200,8 @@ public class TestUtils { return round(input, 1e5); } - public static Map> getTileMap(Mbtiles db) throws SQLException, IOException { + public static Map> getTileMap(ReadableTileArchive db) + throws IOException { Map> tiles = new TreeMap<>(); for (var tile : getAllTiles(db)) { var bytes = gunzip(tile.bytes()); @@ -218,21 +220,10 @@ public class TestUtils { } } - public static Set getAllTiles(Mbtiles db) throws SQLException { - Set result = new HashSet<>(); - try (Statement statement = db.connection().createStatement()) { - ResultSet rs = statement.executeQuery("select zoom_level, tile_column, tile_row, tile_data from tiles"); - while (rs.next()) { - int z = rs.getInt("zoom_level"); - int rawy = rs.getInt("tile_row"); - int x = rs.getInt("tile_column"); - result.add(new Mbtiles.TileEntry( - TileCoord.ofXYZ(x, (1 << z) - 1 - rawy, z), - rs.getBytes("tile_data") - )); - } - } - return result; + public static Set getAllTiles(ReadableTileArchive db) { + return db.getAllTileCoords().stream() + .map(coord -> new Mbtiles.TileEntry(coord, db.getTile(coord))) + .collect(Collectors.toSet()); } public static int getTilesDataCount(Mbtiles db) throws SQLException { diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileArchiveConfigTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileArchiveConfigTest.java new file mode 100644 index 00000000..78ff0f06 --- /dev/null +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileArchiveConfigTest.java @@ -0,0 +1,36 @@ +package com.onthegomap.planetiler.archive; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.nio.file.Path; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class TileArchiveConfigTest { + + @Test + void testMbtiles() { + var config = TileArchiveConfig.from("output.mbtiles"); + assertEquals(TileArchiveConfig.Format.MBTILES, config.format()); + assertEquals(TileArchiveConfig.Scheme.FILE, config.scheme()); + assertEquals(Map.of(), config.options()); + assertEquals(Path.of("output.mbtiles").toAbsolutePath(), config.getLocalPath()); + } + + @Test + void testMbtilesWithOptions() { + var config = TileArchiveConfig.from("output.mbtiles?compact=true"); + assertEquals(TileArchiveConfig.Format.MBTILES, config.format()); + assertEquals(TileArchiveConfig.Scheme.FILE, config.scheme()); + assertEquals(Map.of("compact", "true"), config.options()); + assertEquals(Path.of("output.mbtiles").toAbsolutePath(), config.getLocalPath()); + } + + @Test + void testPmtiles() { + assertEquals(TileArchiveConfig.Format.PMTILES, TileArchiveConfig.from("output.pmtiles").format()); + assertEquals(TileArchiveConfig.Format.PMTILES, TileArchiveConfig.from("output.mbtiles?format=pmtiles").format()); + assertEquals(TileArchiveConfig.Format.PMTILES, + TileArchiveConfig.from("file:///output.mbtiles?format=pmtiles").format()); + } +} diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileArchiveMetadataTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileArchiveMetadataTest.java new file mode 100644 index 00000000..985e5820 --- /dev/null +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileArchiveMetadataTest.java @@ -0,0 +1,67 @@ +package com.onthegomap.planetiler.archive; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import com.onthegomap.planetiler.Profile; +import com.onthegomap.planetiler.config.Arguments; +import com.onthegomap.planetiler.config.PlanetilerConfig; +import com.onthegomap.planetiler.geo.GeoUtils; +import java.util.Map; +import java.util.TreeMap; +import org.junit.jupiter.api.Test; +import org.locationtech.jts.geom.CoordinateXY; +import org.locationtech.jts.geom.Envelope; + +class TileArchiveMetadataTest { + + @Test + void testAddMetadataWorldBounds() { + var bounds = GeoUtils.WORLD_LAT_LON_BOUNDS; + var metadata = new TileArchiveMetadata(new Profile.NullProfile(), PlanetilerConfig.from(Arguments.of(Map.of( + "bounds", bounds.getMinX() + "," + bounds.getMinY() + "," + bounds.getMaxX() + "," + bounds.getMaxY() + )))); + assertEquals(bounds, metadata.bounds()); + assertEquals(new CoordinateXY(0, 0), metadata.center()); + assertEquals(0d, metadata.zoom().doubleValue()); + } + + @Test + void testAddMetadataSmallBounds() { + var bounds = new Envelope(-73.6632, -69.7598, 41.1274, 43.0185); + var metadata = new TileArchiveMetadata(new Profile.NullProfile(), PlanetilerConfig.from(Arguments.of(Map.of( + "bounds", "-73.6632,41.1274,-69.7598,43.0185" + )))); + assertEquals(bounds, metadata.bounds()); + assertEquals(-71.7115, metadata.center().x, 1e-5); + assertEquals(42.07295, metadata.center().y, 1e-5); + assertEquals(7, Math.ceil(metadata.zoom())); + } + + @Test + void testToMap() { + var bounds = "-73.6632,41.1274,-69.7598,43.0185"; + var metadata = new TileArchiveMetadata( + new Profile.NullProfile(), + PlanetilerConfig.from(Arguments.of(Map.of( + "bounds", bounds + )))); + var map = new TreeMap<>(metadata.toMap()); + assertNotNull(map.remove("planetiler:version")); + assertNotNull(map.remove("planetiler:githash")); + assertNotNull(map.remove("planetiler:buildtime")); + assertEquals( + new TreeMap(Map.of( + "name", "Null", + "type", "baselayer", + "format", "pbf", + "zoom", "6.5271217861412305", + "minzoom", "0", + "maxzoom", "14", + "bounds", "-73.6632,41.1274,-69.7598,43.0185", + "center", "-71.7115,42.07295" + )), + map + ); + } +} diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/config/ArgumentsTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/config/ArgumentsTest.java index a4147f93..a51e4f3a 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/config/ArgumentsTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/config/ArgumentsTest.java @@ -9,6 +9,7 @@ import java.nio.file.Path; import java.time.Duration; import java.util.List; import java.util.Map; +import java.util.Set; import org.junit.jupiter.api.Test; import org.locationtech.jts.geom.Envelope; @@ -293,4 +294,59 @@ class ArgumentsTest { assertEquals(false, args.getBooleanObject("BOOL_FALSE", "test")); assertEquals(false, args.getBooleanObject("BOOL_NO", "test")); } + + @Test + void testDeprecatedArgs() { + assertEquals("newvalue", + Arguments.of("oldkey", "oldvalue", "newkey", "newvalue") + .getString("newkey|oldkey", "key", "fallback")); + assertEquals("oldvalue", + Arguments.of("oldkey", "oldvalue") + .getString("newkey|oldkey", "key", "fallback")); + assertEquals("fallback", + Arguments.of() + .getString("newkey|oldkey", "key", "fallback")); + } + + @Test + void testWithPrefix() { + var args = Arguments.of("prefix_a", "a_val", "prefix-b", "b_val", "other", "other_val").withPrefix("prefix"); + assertEquals("a_val", args.getArg("a")); + assertEquals("b_val", args.getArg("b")); + assertNull(args.getArg("other")); + assertNull(args.getArg("prefix_a")); + assertNull(args.getArg("prefix_b")); + assertNull(args.getArg("prefix_other")); + assertEquals(Set.of("a", "b"), args.toMap().keySet()); + } + + @Test + void testPrefixFromEnvironment() { + Map env = Map.of( + "OTHER", "value", + "PLANETILEROTHER", "VALUE", + "PLANETILER_MBTILES_KEY1", "value1", + "PLANETILER_PMTILES_KEY2", "value2" + ); + Arguments args = Arguments.fromEnvironment(env::get, env::keySet).withPrefix("mbtiles"); + assertEquals(Map.of( + "key1", "value1" + ), args.toMap()); + assertEquals("value1", args.getArg("key1")); + } + + @Test + void testSubset() { + var args = Arguments.of(Map.of( + "key_1", "val_1", + "key-2", "val_2", + "key-3", "val_3" + )).subset("key-1", "key-2"); + assertEquals(Map.of( + "key_1", "val_1", + "key_2", "val_2" + ), args.toMap()); + assertEquals("val_1", args.getArg("key-1")); + assertNull(args.getArg("key-3")); + } } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/MbtilesTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/MbtilesTest.java index 696947b3..9ac149e1 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/MbtilesTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/MbtilesTest.java @@ -5,24 +5,27 @@ import static org.junit.jupiter.api.Assertions.*; import com.google.common.math.IntMath; import com.onthegomap.planetiler.TestUtils; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; import com.onthegomap.planetiler.archive.TileEncodingResult; -import com.onthegomap.planetiler.geo.GeoUtils; +import com.onthegomap.planetiler.config.Arguments; import com.onthegomap.planetiler.geo.TileCoord; import com.onthegomap.planetiler.util.LayerStats; import java.io.IOException; import java.math.RoundingMode; +import java.nio.file.Path; import java.sql.SQLException; import java.sql.Statement; import java.util.List; import java.util.Map; import java.util.OptionalLong; import java.util.Set; -import java.util.TreeMap; import java.util.TreeSet; import java.util.stream.Collectors; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import org.locationtech.jts.geom.CoordinateXY; import org.locationtech.jts.geom.Envelope; class MbtilesTest { @@ -33,11 +36,12 @@ class MbtilesTest { private static final int TILES_DATA_BATCH = MAX_PARAMETERS_IN_PREPARED_STATEMENT / 2; - private static final - - void testWriteTiles(int howMany, boolean skipIndexCreation, boolean optimize, boolean compactDb) - throws IOException, SQLException { - try (Mbtiles db = Mbtiles.newInMemoryDatabase(compactDb)) { + private static void testWriteTiles(Path path, int howMany, boolean skipIndexCreation, boolean optimize, + boolean compactDb) throws IOException, SQLException { + var options = Arguments.of("compact", Boolean.toString(compactDb)); + try ( + Mbtiles db = path == null ? Mbtiles.newInMemoryDatabase(options) : Mbtiles.newWriteToFileDatabase(path, options) + ) { if (skipIndexCreation) { db.createTablesWithoutIndexes(); } else { @@ -84,24 +88,42 @@ class MbtilesTest { @ParameterizedTest @ValueSource(ints = {0, 1, TILES_BATCH, TILES_BATCH + 1, 2 * TILES_BATCH, 2 * TILES_BATCH + 1}) void testWriteTilesDifferentSizeInNonCompactMode(int howMany) throws IOException, SQLException { - testWriteTiles(howMany, false, false, false); + testWriteTiles(null, howMany, false, false, false); } @ParameterizedTest @ValueSource(ints = {0, 1, TILES_DATA_BATCH, TILES_DATA_BATCH + 1, 2 * TILES_DATA_BATCH, 2 * TILES_DATA_BATCH + 1, TILES_SHALLOW_BATCH, TILES_SHALLOW_BATCH + 1, 2 * TILES_SHALLOW_BATCH, 2 * TILES_SHALLOW_BATCH + 1}) void testWriteTilesDifferentSizeInCompactMode(int howMany) throws IOException, SQLException { - testWriteTiles(howMany, false, false, true); + testWriteTiles(null, howMany, false, false, true); } @Test void testSkipIndexCreation() throws IOException, SQLException { - testWriteTiles(10, true, false, false); + testWriteTiles(null, 10, true, false, false); } @Test void testVacuumAnalyze() throws IOException, SQLException { - testWriteTiles(10, false, true, false); + testWriteTiles(null, 10, false, true, false); + } + + @Test + void testWriteToFile(@TempDir Path tmpDir) throws IOException, SQLException { + testWriteTiles(tmpDir.resolve("archive.mbtiles"), 10, false, false, true); + } + + @Test + void testCustomPragma() throws IOException, SQLException { + try ( + Mbtiles db = Mbtiles.newInMemoryDatabase(Arguments.of( + "cache-size", "123", + "garbage", "456" + )); + ) { + int result = db.connection().createStatement().executeQuery("pragma cache_size").getInt(1); + assertEquals(123, result); + } } @ParameterizedTest @@ -121,71 +143,47 @@ class MbtilesTest { } @Test - void testAddMetadata() throws IOException { - Map expected = new TreeMap<>(); - try (Mbtiles db = Mbtiles.newInMemoryDatabase()) { - var metadata = db.createTablesWithoutIndexes().metadata(); - metadata.setName("name value"); - expected.put("name", "name value"); - - metadata.setFormat("pbf"); - expected.put("format", "pbf"); - - metadata.setAttribution("attribution value"); - expected.put("attribution", "attribution value"); - - metadata.setBoundsAndCenter(GeoUtils.toLatLonBoundsBounds(new Envelope(0.25, 0.75, 0.25, 0.75))); - expected.put("bounds", "-90,-66.51326,90,66.51326"); - expected.put("center", "0,0,1"); - - metadata.setDescription("description value"); - expected.put("description", "description value"); - - metadata.setMinzoom(1); - expected.put("minzoom", "1"); - - metadata.setMaxzoom(13); - expected.put("maxzoom", "13"); - - metadata.setVersion("1.2.3"); - expected.put("version", "1.2.3"); - - metadata.setTypeIsBaselayer(); - expected.put("type", "baselayer"); - - assertEquals(expected, metadata.getAll()); - } + void testRoundTripMetadata() throws IOException { + roundTripMetadata(new TileArchiveMetadata( + "MyName", + "MyDescription", + "MyAttribution", + "MyVersion", + "baselayer", + TileArchiveMetadata.MVT_FORMAT, + new Envelope(1, 2, 3, 4), + new CoordinateXY(5, 6), + 7d, + 8, + 9, + List.of(new LayerStats.VectorLayer("MyLayer", Map.of())), + Map.of("other key", "other value") + )); } @Test - void testAddMetadataWorldBounds() throws IOException { - Map expected = new TreeMap<>(); + void testRoundTripMinimalMetadata() throws IOException { + var empty = + new TileArchiveMetadata(null, null, null, null, null, null, null, null, null, null, null, null, Map.of()); + roundTripMetadata(empty); try (Mbtiles db = Mbtiles.newInMemoryDatabase()) { - var metadata = db.createTablesWithoutIndexes().metadata(); - metadata.setBoundsAndCenter(GeoUtils.WORLD_LAT_LON_BOUNDS); - expected.put("bounds", "-180,-85.05113,180,85.05113"); - expected.put("center", "0,0,0"); - - assertEquals(expected, metadata.getAll()); + db.createTablesWithoutIndexes(); + assertEquals(empty, db.metadata()); } } - @Test - void testAddMetadataSmallBounds() throws IOException { - Map expected = new TreeMap<>(); + private static void roundTripMetadata(TileArchiveMetadata metadata) throws IOException { try (Mbtiles db = Mbtiles.newInMemoryDatabase()) { - var metadata = db.createTablesWithoutIndexes().metadata(); - metadata.setBoundsAndCenter(new Envelope(-73.6632, -69.7598, 41.1274, 43.0185)); - expected.put("bounds", "-73.6632,41.1274,-69.7598,43.0185"); - expected.put("center", "-71.7115,42.07295,7"); - - assertEquals(expected, metadata.getAll()); + db.createTablesWithoutIndexes(); + var metadataTable = db.metadataTable(); + metadataTable.set(metadata); + assertEquals(metadata, metadataTable.get()); } } private void testMetadataJson(Mbtiles.MetadataJson object, String expected) throws IOException { try (Mbtiles db = Mbtiles.newInMemoryDatabase()) { - var metadata = db.createTablesWithoutIndexes().metadata(); + var metadata = db.createTablesWithoutIndexes().metadataTable(); metadata.setJson(object); var actual = metadata.getAll().get("json"); assertSameJson(expected, actual); diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/VerifyTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/VerifyTest.java index 9324add2..52d6ba56 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/VerifyTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/VerifyTest.java @@ -45,7 +45,7 @@ class VerifyTest { @Test void testValidWithNameAndOneTile() throws IOException { mbtiles.createTablesWithIndexes(); - mbtiles.metadata().setName("name"); + mbtiles.metadataTable().setMetadata("name", "name"); try (var writer = mbtiles.newTileWriter()) { VectorTile tile = new VectorTile(); tile.addLayerFeatures("layer", List.of(new VectorTile.Feature( @@ -62,7 +62,7 @@ class VerifyTest { @Test void testInvalidGeometry() throws IOException { mbtiles.createTablesWithIndexes(); - mbtiles.metadata().setName("name"); + mbtiles.metadataTable().setMetadata("name", "name"); try (var writer = mbtiles.newTileWriter()) { VectorTile tile = new VectorTile(); tile.addLayerFeatures("layer", List.of(new VectorTile.Feature( diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/pmtiles/PmtilesTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/pmtiles/PmtilesTest.java index 122c701e..d5b5f008 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/pmtiles/PmtilesTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/pmtiles/PmtilesTest.java @@ -15,15 +15,16 @@ import com.onthegomap.planetiler.util.SeekableInMemoryByteChannel; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; -import java.nio.file.Path; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.OptionalLong; import java.util.Set; import java.util.stream.Collectors; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; +import org.locationtech.jts.geom.CoordinateXY; +import org.locationtech.jts.geom.Envelope; class PmtilesTest { @@ -181,11 +182,12 @@ class PmtilesTest { var in = WriteablePmtiles.newWriteToMemory(bytes); var config = PlanetilerConfig.defaults(); - in.initialize(config, new TileArchiveMetadata(new Profile.NullProfile()), new LayerStats()); + var metadata = new TileArchiveMetadata(new Profile.NullProfile(), config); + in.initialize(metadata); var writer = in.newTileWriter(); writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 1), new byte[]{0xa, 0x2}, OptionalLong.empty())); - in.finish(config); + in.finish(metadata); var reader = new ReadablePmtiles(bytes); var header = reader.getHeader(); assertEquals(1, header.numAddressedTiles()); @@ -200,28 +202,59 @@ class PmtilesTest { } @Test - void testWritePmtilesToFileWithMetadata(@TempDir Path tempDir) throws IOException { + void testRoundtripMetadata() throws IOException { + roundTripMetadata(new TileArchiveMetadata( + "MyName", + "MyDescription", + "MyAttribution", + "MyVersion", + "baselayer", + TileArchiveMetadata.MVT_FORMAT, + new Envelope(1, 2, 3, 4), + new CoordinateXY(5, 6), + 7d, + 8, + 9, + List.of(new LayerStats.VectorLayer("MyLayer", Map.of())), + Map.of("other key", "other value") + )); + } - try (var in = WriteablePmtiles.newWriteToFile(tempDir.resolve("tmp.pmtiles"))) { - var config = PlanetilerConfig.defaults(); - in.initialize(config, - new TileArchiveMetadata("MyName", "MyDescription", "MyAttribution", "MyVersion", "baselayer", new HashMap<>()), - new LayerStats()); + @Test + void testRoundtripMetadataMinimal() throws IOException { + roundTripMetadata( + new TileArchiveMetadata(null, null, null, null, null, null, null, null, null, null, null, null, Map.of()), + new TileArchiveMetadata(null, null, null, null, null, null, + new Envelope(-180, 180, -85.0511287, 85.0511287), + new CoordinateXY(0, 0), + 0d, + 0, + 15, + null, + Map.of() + ) + ); + } + + private static void roundTripMetadata(TileArchiveMetadata metadata) throws IOException { + roundTripMetadata(metadata, metadata); + } + + private static void roundTripMetadata(TileArchiveMetadata input, TileArchiveMetadata output) throws IOException { + try ( + var channel = new SeekableInMemoryByteChannel(0); + var in = WriteablePmtiles.newWriteToMemory(channel) + ) { + in.initialize(input); var writer = in.newTileWriter(); writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0xa, 0x2}, OptionalLong.empty())); - in.finish(config); + in.finish(input); + var reader = new ReadablePmtiles(channel); + assertArrayEquals(new byte[]{0xa, 0x2}, reader.getTile(0, 0, 0)); + + assertEquals(output, reader.metadata()); } - - var reader = new ReadablePmtiles(FileChannel.open(tempDir.resolve("tmp.pmtiles"))); - assertArrayEquals(new byte[]{0xa, 0x2}, reader.getTile(0, 0, 0)); - - var metadata = reader.getJsonMetadata(); - assertEquals("MyName", metadata.otherMetadata().get("name")); - assertEquals("MyDescription", metadata.otherMetadata().get("description")); - assertEquals("MyAttribution", metadata.otherMetadata().get("attribution")); - assertEquals("MyVersion", metadata.otherMetadata().get("version")); - assertEquals("baselayer", metadata.otherMetadata().get("type")); } @Test @@ -250,13 +283,14 @@ class PmtilesTest { var in = WriteablePmtiles.newWriteToMemory(bytes); var config = PlanetilerConfig.defaults(); - in.initialize(config, new TileArchiveMetadata(new Profile.NullProfile()), new LayerStats()); + var metadata = new TileArchiveMetadata(new Profile.NullProfile(), config); + in.initialize(metadata); var writer = in.newTileWriter(); writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0xa, 0x2}, OptionalLong.of(42))); writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 1), new byte[]{0xa, 0x2}, OptionalLong.of(42))); writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 2), new byte[]{0xa, 0x2}, OptionalLong.of(42))); - in.finish(config); + in.finish(metadata); var reader = new ReadablePmtiles(bytes); var header = reader.getHeader(); assertEquals(3, header.numAddressedTiles()); @@ -276,12 +310,13 @@ class PmtilesTest { var in = WriteablePmtiles.newWriteToMemory(bytes); var config = PlanetilerConfig.defaults(); - in.initialize(config, new TileArchiveMetadata(new Profile.NullProfile()), new LayerStats()); + var metadata = new TileArchiveMetadata(new Profile.NullProfile(), config); + in.initialize(metadata); var writer = in.newTileWriter(); writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 1), new byte[]{0xa, 0x2}, OptionalLong.of(42))); writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0xa, 0x2}, OptionalLong.of(42))); - in.finish(config); + in.finish(metadata); var reader = new ReadablePmtiles(bytes); var header = reader.getHeader(); assertEquals(2, header.numAddressedTiles()); @@ -301,7 +336,8 @@ class PmtilesTest { var in = WriteablePmtiles.newWriteToMemory(bytes); var config = PlanetilerConfig.defaults(); - in.initialize(config, new TileArchiveMetadata(new Profile.NullProfile()), new LayerStats()); + var metadata = new TileArchiveMetadata(new Profile.NullProfile(), config); + in.initialize(metadata); var writer = in.newTileWriter(); int ENTRIES = 20000; @@ -311,7 +347,7 @@ class PmtilesTest { OptionalLong.empty())); } - in.finish(config); + in.finish(metadata); var reader = new ReadablePmtiles(bytes); var header = reader.getHeader(); assertEquals(ENTRIES, header.numAddressedTiles()); diff --git a/planetiler-custommap/README.md b/planetiler-custommap/README.md index 7da4d729..f62d2808 100644 --- a/planetiler-custommap/README.md +++ b/planetiler-custommap/README.md @@ -151,8 +151,6 @@ cat planetiler-custommap/planetiler.schema.json | jq -r '.properties.args.proper - `minzoom` - Minimum tile zoom level to emit - `maxzoom` - Maximum tile zoom level to emit - `render_maxzoom` - Maximum rendering zoom level up to -- `skip_mbtiles_index_creation` - Skip adding index to mbtiles file -- `optimize_db` - Vacuum analyze mbtiles file after writing - `force` - Overwriting output file and ignore warnings - `gzip_temp` - Gzip temporary feature storage (uses more CPU, but less disk space) - `mmap_temp` - Use memory-mapped IO for temp feature files @@ -175,7 +173,6 @@ cat planetiler-custommap/planetiler.schema.json | jq -r '.properties.args.proper maximum zoom level to allow for overzooming - `simplify_tolerance` - Default value for the tile pixel tolerance to use when simplifying features below the maximum zoom level -- `compact_db` - Reduce the DB size by separating and deduping the tile data - `skip_filled_tiles` - Skip writing tiles containing only polygon fills to the output - `tile_warning_size_mb` - Maximum size in megabytes of a tile to emit a warning about diff --git a/planetiler-custommap/planetiler.schema.json b/planetiler-custommap/planetiler.schema.json index 99a8c541..e8b6f2a6 100644 --- a/planetiler-custommap/planetiler.schema.json +++ b/planetiler-custommap/planetiler.schema.json @@ -138,28 +138,6 @@ "render_maxzoom": { "description": "Maximum rendering zoom level up to" }, - "skip_mbtiles_index_creation": { - "description": "Skip adding index to mbtiles file", - "anyOf": [ - { - "type": "string" - }, - { - "type": "boolean" - } - ] - }, - "optimize_db": { - "description": "Vacuum analyze mbtiles file after writing", - "anyOf": [ - { - "type": "string" - }, - { - "type": "boolean" - } - ] - }, "force": { "description": "Overwriting output file and ignore warnings", "anyOf": [ @@ -294,17 +272,6 @@ "simplify_tolerance": { "description": "Default value for the tile pixel tolerance to use when simplifying features below the maximum zoom level" }, - "compact_db": { - "description": "Reduce the DB size by separating and deduping the tile data", - "anyOf": [ - { - "type": "string" - }, - { - "type": "boolean" - } - ] - }, "skip_filled_tiles": { "description": "Skip writing tiles containing only polygon fills to the output", "anyOf": [ diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredMapMain.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredMapMain.java index 0e0677e2..23283e53 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredMapMain.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredMapMain.java @@ -10,7 +10,7 @@ import java.nio.file.Path; /** * Main driver to create maps configured by a YAML file. - * + *

* Parses the config file into a {@link ConfiguredProfile}, loads sources into {@link Planetiler} runner and kicks off * the map generation process. */ @@ -54,7 +54,7 @@ public class ConfiguredMapMain { configureSource(planetiler, sourcesDir, source); } - planetiler.overwriteOutput("mbtiles", Path.of("data", "output.mbtiles")).run(); + planetiler.overwriteOutput(Path.of("data", "output.mbtiles")).run(); } private static void configureSource(Planetiler planetiler, Path sourcesDir, Source source) { diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java index 9cb6e08e..57ca78c9 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java @@ -187,8 +187,6 @@ public class Contexts { argumentValues.put("minzoom", config.minzoom()); argumentValues.put("maxzoom", config.maxzoom()); argumentValues.put("render_maxzoom", config.maxzoomForRendering()); - argumentValues.put("skip_mbtiles_index_creation", config.skipIndexCreation()); - argumentValues.put("optimize_db", config.optimizeDb()); argumentValues.put("force", config.force()); argumentValues.put("gzip_temp", config.gzipTempStorage()); argumentValues.put("mmap_temp", config.mmapTempStorage()); @@ -209,7 +207,6 @@ public class Contexts { argumentValues.put("min_feature_size", config.minFeatureSizeBelowMaxZoom()); argumentValues.put("simplify_tolerance_at_max_zoom", config.simplifyToleranceAtMaxZoom()); argumentValues.put("simplify_tolerance", config.simplifyToleranceBelowMaxZoom()); - argumentValues.put("compact_db", config.compactDb()); argumentValues.put("skip_filled_tiles", config.skipFilledTiles()); argumentValues.put("tile_warning_size_mb", config.tileWarningSizeBytes()); builtInArgs = Set.copyOf(argumentValues.keySet()); diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredMapTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredMapTest.java index fe179519..d7c8527a 100644 --- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredMapTest.java +++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredMapTest.java @@ -47,7 +47,7 @@ class ConfiguredMapTest { "--tmp=" + tmpDir, // Override output location - "--mbtiles=" + dbPath + "--output=" + dbPath ); mbtiles = Mbtiles.newReadOnlyDatabase(dbPath); } @@ -59,7 +59,7 @@ class ConfiguredMapTest { @Test void testMetadata() { - Map metadata = mbtiles.metadata().getAll(); + Map metadata = mbtiles.metadataTable().getAll(); assertEquals("OWG Simple Schema", metadata.get("name")); assertEquals("0", metadata.get("minzoom")); assertEquals("14", metadata.get("maxzoom")); diff --git a/planetiler-examples/README.md b/planetiler-examples/README.md index 3978b59d..c6e39225 100644 --- a/planetiler-examples/README.md +++ b/planetiler-examples/README.md @@ -105,7 +105,7 @@ java -cp target/*-with-deps.jar com.onthegomap.planetiler.examples.MyProfile Then, to inspect the tiles: ```bash -tileserver-gl-light --mbtiles data/toilets.mbtiles +tileserver-gl-light data/toilets.mbtiles ``` Finally, open http://localhost:8080 to see your tiles. @@ -143,7 +143,7 @@ public void integrationTest(@TempDir Path tmpDir) throws Exception { MyProfile.main( "--osm_path=" + TestUtils.pathToResource("monaco-latest.osm.pbf"), "--tmp=" + tmpDir, - "--mbtiles=" + mbtilesPath, + "--output=" + mbtilesPath, )); try (Mbtiles mbtiles = Mbtiles.newReadOnlyDatabase(mbtilesPath)) { Map metadata = mbtiles.metadata().getAll(); diff --git a/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/BikeRouteOverlay.java b/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/BikeRouteOverlay.java index 5fed1516..719b5629 100644 --- a/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/BikeRouteOverlay.java +++ b/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/BikeRouteOverlay.java @@ -22,7 +22,7 @@ import java.util.List; *

  • then build the examples: {@code mvn clean package}
    • *
  • then run this example: * {@code java -cp target/*-with-deps.jar com.onthegomap.planetiler.examples.BikeRouteOverlay osm_path="path/to/data.osm.pbf" mbtiles="data/output.mbtiles"}
    • - *
  • then run the demo tileserver: {@code tileserver-gl-light --mbtiles data/bikeroutes.mbtiles}
    • + *
  • then run the demo tileserver: {@code tileserver-gl-light data/bikeroutes.mbtiles}
    • *
  • and view the output at localhost:8080
    • * */ @@ -175,7 +175,7 @@ public class BikeRouteOverlay implements Profile { // override this default with osm_path="path/to/data.osm.pbf" .addOsmSource("osm", Path.of("data", "sources", area + ".osm.pbf"), "geofabrik:" + area) // override this default with mbtiles="path/to/output.mbtiles" - .overwriteOutput("mbtiles", Path.of("data", "bikeroutes.mbtiles")) + .overwriteOutput(Path.of("data", "bikeroutes.mbtiles")) .run(); } } diff --git a/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/OsmQaTiles.java b/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/OsmQaTiles.java index 44df7001..4b149034 100644 --- a/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/OsmQaTiles.java +++ b/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/OsmQaTiles.java @@ -54,7 +54,7 @@ public class OsmQaTiles implements Profile { Path.of("data", "sources", area + ".osm.pbf"), "planet".equalsIgnoreCase(area) ? "aws:latest" : ("geofabrik:" + area) ) - .overwriteOutput("mbtiles", Path.of("data", "qa.mbtiles")) + .overwriteOutput(Path.of("data", "qa.mbtiles")) .run(); } diff --git a/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlay.java b/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlay.java index 35fe50b7..7763a40b 100644 --- a/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlay.java +++ b/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlay.java @@ -19,7 +19,7 @@ import java.util.concurrent.atomic.AtomicInteger; *
  • then build the examples: {@code mvn clean package}
    • *
  • then run this example: * {@code java -cp target/*-fatjar.jar com.onthegomap.planetiler.examples.ToiletsOverlay osm_path="path/to/data.osm.pbf" mbtiles="data/output.mbtiles"}
    • - *
  • then run the demo tileserver: {@code tileserver-gl-light --mbtiles=data/output.mbtiles}
    • + *
  • then run the demo tileserver: {@code tileserver-gl-light data/output.mbtiles}
    • *
  • and view the output at localhost:8080
    • * */ @@ -103,7 +103,7 @@ public class ToiletsOverlay implements Profile { // override this default with osm_path="path/to/data.osm.pbf" .addOsmSource("osm", Path.of("data", "sources", area + ".osm.pbf"), "geofabrik:" + area) // override this default with mbtiles="path/to/output.mbtiles" - .overwriteOutput("mbtiles", Path.of("data", "toilets.mbtiles")) + .overwriteOutput(Path.of("data", "toilets.mbtiles")) .run(); } } diff --git a/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApi.java b/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApi.java index 1c5a8e61..801489af 100644 --- a/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApi.java +++ b/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApi.java @@ -4,13 +4,14 @@ import com.onthegomap.planetiler.Planetiler; import com.onthegomap.planetiler.Profile; import com.onthegomap.planetiler.archive.TileArchiveMetadata; import com.onthegomap.planetiler.archive.TileArchiveWriter; +import com.onthegomap.planetiler.archive.TileArchives; +import com.onthegomap.planetiler.archive.WriteableTileArchive; import com.onthegomap.planetiler.collection.FeatureGroup; import com.onthegomap.planetiler.collection.LongLongMap; import com.onthegomap.planetiler.collection.LongLongMultimap; import com.onthegomap.planetiler.config.Arguments; import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.geo.TileOrder; -import com.onthegomap.planetiler.mbtiles.Mbtiles; import com.onthegomap.planetiler.reader.osm.OsmInputFile; import com.onthegomap.planetiler.reader.osm.OsmReader; import com.onthegomap.planetiler.stats.Stats; @@ -31,7 +32,7 @@ import org.slf4j.LoggerFactory; *
  • then build the examples: {@code mvn clean package}
    • *
  • then run this example: * {@code java -cp target/*-fatjar.jar com.onthegomap.planetiler.examples.ToiletsOverlayLowLevelApi}
    • - *
  • then run the demo tileserver: {@code tileserver-gl-light --mbtiles=data/toilets.mbtiles}
    • + *
  • then run the demo tileserver: {@code tileserver-gl-light data/toilets.mbtiles}
    • *
  • and view the output at localhost:8080
    • * */ @@ -57,7 +58,7 @@ public class ToiletsOverlayLowLevelApi { PlanetilerConfig config = PlanetilerConfig.from(Arguments.fromJvmProperties()); // extract mbtiles metadata from profile - TileArchiveMetadata tileArchiveMetadata = new TileArchiveMetadata(profile); + TileArchiveMetadata tileArchiveMetadata = new TileArchiveMetadata(profile, config); // overwrite output each time FileUtils.deleteFile(output); @@ -112,7 +113,7 @@ public class ToiletsOverlayLowLevelApi { // then process rendered features, grouped by tile, encoding them into binary vector tile format // and writing to the output mbtiles file. - try (Mbtiles db = Mbtiles.newWriteToFileDatabase(output, config.compactDb())) { + try (WriteableTileArchive db = TileArchives.newWriter(output, config)) { TileArchiveWriter.writeOutput(featureGroup, db, () -> FileUtils.fileSize(output), tileArchiveMetadata, config, stats); } catch (IOException e) { diff --git a/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/BikeRouteOverlayTest.java b/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/BikeRouteOverlayTest.java index 525c1278..7dbc25ef 100644 --- a/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/BikeRouteOverlayTest.java +++ b/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/BikeRouteOverlayTest.java @@ -98,10 +98,10 @@ class BikeRouteOverlayTest { // Override temp dir location "tmp", tmpDir.toString(), // Override output location - "mbtiles", dbPath.toString() + "output", dbPath.toString() )); try (Mbtiles mbtiles = Mbtiles.newReadOnlyDatabase(dbPath)) { - Map metadata = mbtiles.metadata().getAll(); + Map metadata = mbtiles.metadataTable().getAll(); assertEquals("Bike Paths Overlay", metadata.get("name")); assertContains("openstreetmap.org/copyright", metadata.get("attribution")); diff --git a/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/OsmQaTilesTest.java b/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/OsmQaTilesTest.java index a1b62eaa..65669508 100644 --- a/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/OsmQaTilesTest.java +++ b/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/OsmQaTilesTest.java @@ -93,10 +93,10 @@ class OsmQaTilesTest { // Override temp dir location "tmp", tmpDir.toString(), // Override output location - "mbtiles", dbPath.toString() + "output", dbPath.toString() )); try (Mbtiles mbtiles = Mbtiles.newReadOnlyDatabase(dbPath)) { - Map metadata = mbtiles.metadata().getAll(); + Map metadata = mbtiles.metadataTable().getAll(); assertEquals("osm qa", metadata.get("name")); assertContains("openstreetmap.org/copyright", metadata.get("attribution")); diff --git a/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApiTest.java b/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApiTest.java index 5b9cc15e..16c87280 100644 --- a/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApiTest.java +++ b/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApiTest.java @@ -27,7 +27,7 @@ class ToiletsOverlayLowLevelApiTest { dbPath ); try (Mbtiles mbtiles = Mbtiles.newReadOnlyDatabase(dbPath)) { - Map metadata = mbtiles.metadata().getAll(); + Map metadata = mbtiles.metadata().toMap(); assertEquals("Toilets Overlay", metadata.get("name")); assertContains("openstreetmap.org/copyright", metadata.get("attribution")); diff --git a/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/ToiletsProfileTest.java b/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/ToiletsProfileTest.java index 51c69997..691cbf99 100644 --- a/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/ToiletsProfileTest.java +++ b/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/ToiletsProfileTest.java @@ -58,10 +58,10 @@ class ToiletsProfileTest { // Override temp dir location "tmp", tmpDir.toString(), // Override output location - "mbtiles", dbPath.toString() + "output", dbPath.toString() )); try (Mbtiles mbtiles = Mbtiles.newReadOnlyDatabase(dbPath)) { - Map metadata = mbtiles.metadata().getAll(); + Map metadata = mbtiles.metadata().toMap(); assertEquals("Toilets Overlay", metadata.get("name")); assertContains("openstreetmap.org/copyright", metadata.get("attribution")); diff --git a/planetiler-openmaptiles b/planetiler-openmaptiles index 62de454c..292611de 160000 --- a/planetiler-openmaptiles +++ b/planetiler-openmaptiles @@ -1 +1 @@ -Subproject commit 62de454cf769e5bf2832f32d6b1f707860d442cf +Subproject commit 292611de84b69f0ddbd4b603ec1d0f5d13257c33 diff --git a/quickstart.sh b/quickstart.sh index de0911f9..82fe78e2 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -102,9 +102,10 @@ if [ "$DRY_RUN" == "true" ]; then fi function run() { - echo "$ $*" + command="${*//&/\&}" + echo "$ $command" if [ "$DRY_RUN" != "true" ]; then - eval "$*" + eval "$command" fi } diff --git a/scripts/test-release.sh b/scripts/test-release.sh index a8aab2bf..179c75b7 100755 --- a/scripts/test-release.sh +++ b/scripts/test-release.sh @@ -15,23 +15,23 @@ fi echo "Test java build" echo "::group::OpenMapTiles monaco (java)" rm -f data/out.mbtiles -java -jar planetiler-dist/target/*with-deps.jar --download --area=monaco --mbtiles=data/out.mbtiles +java -jar planetiler-dist/target/*with-deps.jar --download --area=monaco --output=data/out.mbtiles ./scripts/check-monaco.sh data/out.mbtiles echo "::endgroup::" echo "::group::Example (java)" rm -f data/out.mbtiles -java -jar planetiler-dist/target/*with-deps.jar example-toilets --download --area=monaco --mbtiles=data/out.mbtiles +java -jar planetiler-dist/target/*with-deps.jar example-toilets --download --area=monaco --output=data/out.mbtiles ./scripts/check-mbtiles.sh data/out.mbtiles echo "::endgroup::" echo "::endgroup::" echo "::group::OpenMapTiles monaco (docker)" rm -f data/out.mbtiles -docker run -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:"${version}" --area=monaco --mbtiles=data/out.mbtiles +docker run -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:"${version}" --area=monaco --output=data/out.mbtiles ./scripts/check-monaco.sh data/out.mbtiles echo "::endgroup::" echo "::group::Example (docker)" rm -f data/out.mbtiles -docker run -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:"${version}" example-toilets --area=monaco --mbtiles=data/out.mbtiles +docker run -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:"${version}" example-toilets --area=monaco --output=data/out.mbtiles ./scripts/check-mbtiles.sh data/out.mbtiles echo "::endgroup::"