diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 1e416a65..a4158207 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -18,10 +18,10 @@ jobs: - uses: actions/checkout@v4 with: submodules: true - - name: Set up JDK 17 - uses: actions/setup-java@v3 + - name: Set up JDK 21 + uses: actions/setup-java@v4 with: - java-version: 17 + java-version: 21 distribution: 'temurin' cache: 'maven' - name: Ensure code formatted with mvn spotless:apply @@ -34,13 +34,14 @@ jobs: fail-fast: false matrix: os: [ ubuntu-latest, macos-latest, windows-latest ] - jdk: [ 17 ] + jdk: [ 21 ] include: - os: ubuntu-latest - jdk: 17 + jdk: 21 args: "-DargLine='-Duser.language=fr -Duser.country=FR'" - os: ubuntu-latest - jdk: 20 + jdk: 21 + args: "" runs-on: ${{ matrix.os }} timeout-minutes: 15 steps: @@ -48,7 +49,7 @@ jobs: with: submodules: true - name: Set up JDK ${{ matrix.jdk }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.jdk }} distribution: 'temurin' @@ -71,10 +72,10 @@ jobs: - uses: actions/checkout@v4 with: submodules: true - - name: Set up JDK 17 - uses: actions/setup-java@v3 + - name: Set up JDK 21 + uses: actions/setup-java@v4 with: - java-version: 17 + java-version: 21 distribution: 'temurin' - name: Build and test run: mvn --batch-mode -no-transfer-progress package --file standalone.pom.xml @@ -100,9 +101,9 @@ jobs: - name: Cache data/sources uses: ./.github/cache-sources-action - name: Set up JDK - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: - java-version: 17 + java-version: 21 distribution: 'temurin' cache: 'maven' @@ -110,7 +111,7 @@ jobs: run: ./mvnw -DskipTests -Dimage.version=CI_ONLY --batch-mode -no-transfer-progress package jib:dockerBuild --file pom.xml - name: 'Upload artifact' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: planetiler-build path: planetiler-dist/target/*with-deps.jar @@ -137,9 +138,9 @@ jobs: - name: Cache data/sources uses: ./.github/cache-sources-action - name: Set up JDK - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: - java-version: 17 + java-version: 21 distribution: 'temurin' cache: 'maven' diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 56841ff0..88671eb7 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -42,12 +42,12 @@ jobs: with: basedir: branch - name: Set up JDK - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: - java-version: 17 + java-version: 21 distribution: 'temurin' cache: 'maven' - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 with: node-version: '14' - run: npm install -g strip-ansi-cli@3.0.2 @@ -88,7 +88,7 @@ jobs: cat log | strip-ansi > build-info/baselogs.txt - name: 'Upload build-info' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: build-info path: ./build-info diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e94bb712..c1c6eaa5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,65 +18,65 @@ jobs: contents: write packages: write steps: - - name: Ensure version does not start with 'v' - uses: actions/github-script@v6 - with: - github-token: ${{ github.token }} - script: | - version = context.payload.inputs.version; - if (/^v/.test(version)) throw new Error("Bad version number: " + version) - - uses: actions/checkout@v4 - with: - submodules: true - - name: Cache data/sources - uses: ./.github/cache-sources-action - - uses: actions/setup-java@v3 - with: - java-version: '17' - distribution: 'temurin' - cache: 'maven' - server-id: ossrh - server-username: MAVEN_USERNAME - server-password: MAVEN_PASSWORD + - name: Ensure version does not start with 'v' + uses: actions/github-script@v7 + with: + github-token: ${{ github.token }} + script: | + version = context.payload.inputs.version; + if (/^v/.test(version)) throw new Error("Bad version number: " + version) + - uses: actions/checkout@v4 + with: + submodules: true + - name: Cache data/sources + uses: ./.github/cache-sources-action + - uses: actions/setup-java@v4 + with: + java-version: '21' + distribution: 'temurin' + cache: 'maven' + server-id: ossrh + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD - - name: Check tag does not exist yet - run: if git rev-list "v${{ github.event.inputs.version }}"; then echo "Tag already exists. Aborting the release process."; exit 1; fi + - name: Check tag does not exist yet + run: if git rev-list "v${{ github.event.inputs.version }}"; then echo "Tag already exists. Aborting the release process."; exit 1; fi - - run: ./scripts/set-versions.sh "${{ github.event.inputs.version }}" - - run: ./scripts/build-release.sh - - run: ./scripts/test-release.sh "${{ github.event.inputs.version }}" - - name: Create tag - uses: actions/github-script@v6 - with: - github-token: ${{ github.token }} - script: | - github.rest.git.createRef({ - owner: context.repo.owner, - repo: context.repo.repo, - ref: "refs/tags/v${{ github.event.inputs.version }}", - sha: context.sha - }) - - run: mv planetiler-dist/target/*with-deps.jar planetiler.jar - - run: sha256sum planetiler.jar > planetiler.jar.sha256 - - run: md5sum planetiler.jar > planetiler.jar.md5 - - name: Install GPG Private Key - run: | - echo -n "${{ secrets.OSSRH_GPG_SECRET_KEY }}" | base64 --decode | gpg --batch --import - - name: Create Release - uses: softprops/action-gh-release@v1 - with: - fail_on_unmatched_files: true - tag_name: v${{ github.event.inputs.version }} - draft: true - files: | - planetiler.jar* - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - run: ./scripts/push-release.sh ${{ github.event.inputs.version }} - env: - GITHUB_ACTOR: ${{ github.actor }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - IMAGE_TAGS: ${{ github.event.inputs.image_tags }} - MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} - MAVEN_PASSWORD: ${{ secrets.OSSRH_PASSWORD }} - OSSRH_GPG_SECRET_KEY_PASSWORD: ${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} + - run: ./scripts/set-versions.sh "${{ github.event.inputs.version }}" + - run: ./scripts/build-release.sh + - run: ./scripts/test-release.sh "${{ github.event.inputs.version }}" + - name: Create tag + uses: actions/github-script@v7 + with: + github-token: ${{ github.token }} + script: | + github.rest.git.createRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: "refs/tags/v${{ github.event.inputs.version }}", + sha: context.sha + }) + - run: mv planetiler-dist/target/*with-deps.jar planetiler.jar + - run: sha256sum planetiler.jar > planetiler.jar.sha256 + - run: md5sum planetiler.jar > planetiler.jar.md5 + - name: Install GPG Private Key + run: | + echo -n "${{ secrets.OSSRH_GPG_SECRET_KEY }}" | base64 --decode | gpg --batch --import + - name: Create Release + uses: softprops/action-gh-release@v1 + with: + fail_on_unmatched_files: true + tag_name: v${{ github.event.inputs.version }} + draft: true + files: | + planetiler.jar* + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - run: ./scripts/push-release.sh ${{ github.event.inputs.version }} + env: + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + IMAGE_TAGS: ${{ github.event.inputs.image_tags }} + MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.OSSRH_PASSWORD }} + OSSRH_GPG_SECRET_KEY_PASSWORD: ${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} diff --git a/.github/workflows/snapshot.yml b/.github/workflows/snapshot.yml index 359180c6..26997230 100644 --- a/.github/workflows/snapshot.yml +++ b/.github/workflows/snapshot.yml @@ -21,37 +21,37 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Cache data/sources - uses: ./.github/cache-sources-action - - name: Set up JDK - uses: actions/setup-java@v3 - with: - java-version: 17 - distribution: 'temurin' - cache: 'maven' - server-id: ossrh - server-username: MAVEN_USERNAME - server-password: MAVEN_PASSWORD - - run: ./scripts/build-release.sh - - run: ./scripts/test-release.sh - - run: sha256sum planetiler-dist/target/*with-deps.jar - - run: md5sum planetiler-dist/target/*with-deps.jar - - name: 'Upload artifact' - uses: actions/upload-artifact@v3 - with: - name: planetiler-build - path: planetiler-dist/target/*with-deps.jar - - name: Install GPG Private Key - run: | - echo -n "${{ secrets.OSSRH_GPG_SECRET_KEY }}" | base64 --decode | gpg --batch --import - - run: ./scripts/push-release.sh - env: - GITHUB_ACTOR: ${{ github.actor }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - IMAGE_TAGS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tags || 'latest,snapshot' }} - MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} - MAVEN_PASSWORD: ${{ secrets.OSSRH_PASSWORD }} - OSSRH_GPG_SECRET_KEY_PASSWORD: ${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} + - uses: actions/checkout@v4 + with: + submodules: true + - name: Cache data/sources + uses: ./.github/cache-sources-action + - name: Set up JDK + uses: actions/setup-java@v4 + with: + java-version: 21 + distribution: 'temurin' + cache: 'maven' + server-id: ossrh + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + - run: ./scripts/build-release.sh + - run: ./scripts/test-release.sh + - run: sha256sum planetiler-dist/target/*with-deps.jar + - run: md5sum planetiler-dist/target/*with-deps.jar + - name: 'Upload artifact' + uses: actions/upload-artifact@v4 + with: + name: planetiler-build + path: planetiler-dist/target/*with-deps.jar + - name: Install GPG Private Key + run: | + echo -n "${{ secrets.OSSRH_GPG_SECRET_KEY }}" | base64 --decode | gpg --batch --import + - run: ./scripts/push-release.sh + env: + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + IMAGE_TAGS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tags || 'latest,snapshot' }} + MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.OSSRH_PASSWORD }} + OSSRH_GPG_SECRET_KEY_PASSWORD: ${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} diff --git a/.github/workflows/sonar.yml b/.github/workflows/sonar.yml index 67a05c18..5fc9bfd7 100644 --- a/.github/workflows/sonar.yml +++ b/.github/workflows/sonar.yml @@ -15,66 +15,66 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@v4 - with: - # Disabling shallow clone is recommended for improving relevancy of reporting - fetch-depth: 0 - submodules: true - - name: Set up JDK 17 - uses: actions/setup-java@v3 - with: - java-version: 17 - distribution: 'temurin' - cache: 'maven' - - name: Cache SonarCloud packages - uses: actions/cache@v3 - with: - path: ~/.sonar/cache - key: ${{ runner.os }}-sonar - restore-keys: ${{ runner.os }}-sonar - - name: Analyze with SonarCloud - run: | - mvn -Dspotless.check.skip -Pcoverage -B verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar - env: - # Needed to get some information about the pull request, if any - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # Read-only user, use this token to link SonarLint to SonarCloud as well - SONAR_TOKEN: c2cfe8bd7368ced07e84a620b7c2487846e220eb - - name: Wait for SonarCloud API to update... - run: "sleep 10" - - name: Upload annotations on PRs - if: ${{ github.event_name == 'pull_request' }} - uses: actions/github-script@v6 - with: - github-token: ${{ github.token }} - script: | - const pr = context.payload.pull_request.number; - const url = `https://sonarcloud.io/api/issues/search?pullRequest=${pr}&s=FILE_LINE&resolved=false&sinceLeakPeriod=true&ps=100&facets=severities%2Ctypes&componentKeys=onthegomap_planetiler&organization=onthegomap&additionalFields=_all`; - console.log("Fetching " + url); - const response = await github.request(url); - console.log("Got " + JSON.stringify(response.data)); - response.data.issues.forEach(issue => { - try { - if (issue.severity === 'INFO') return; - const textRange = issue.textRange; - const rule = encodeURIComponent(issue.rule); - const message = [ - issue.message, - '', - `rule: ${issue.rule} (https://sonarcloud.io/organizations/onthegomap/rules?open=${rule}&rule_key=${rule})`, - `issue url: https://sonarcloud.io/project/issues?pullRequest=${pr}&open=${encodeURIComponent(issue.key)}&id=onthegomap_planetiler` - ].join('\n'); - const args = { - title: `${issue.severity} ${issue.type}`, - file: issue.component.replace(/^[^:]*:/, ''), - startLine: textRange.startLine, - endLine: textRange.endLine, - startColumn: textRange.startOffset, - endColumn: textRange.endOffset - }; - core.warning(message, args); - console.log(args); - } catch (e) { - core.error(`Unable to parse sonar issue: ${JSON.stringify(issue)}`); - } - }); + - uses: actions/checkout@v4 + with: + # Disabling shallow clone is recommended for improving relevancy of reporting + fetch-depth: 0 + submodules: true + - name: Set up JDK 21 + uses: actions/setup-java@v4 + with: + java-version: 21 + distribution: 'temurin' + cache: 'maven' + - name: Cache SonarCloud packages + uses: actions/cache@v3 + with: + path: ~/.sonar/cache + key: ${{ runner.os }}-sonar + restore-keys: ${{ runner.os }}-sonar + - name: Analyze with SonarCloud + run: | + mvn -Dspotless.check.skip -Pcoverage -B verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar + env: + # Needed to get some information about the pull request, if any + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Read-only user, use this token to link SonarLint to SonarCloud as well + SONAR_TOKEN: c2cfe8bd7368ced07e84a620b7c2487846e220eb + - name: Wait for SonarCloud API to update... + run: "sleep 10" + - name: Upload annotations on PRs + if: ${{ github.event_name == 'pull_request' }} + uses: actions/github-script@v7 + with: + github-token: ${{ github.token }} + script: | + const pr = context.payload.pull_request.number; + const url = `https://sonarcloud.io/api/issues/search?pullRequest=${pr}&s=FILE_LINE&resolved=false&sinceLeakPeriod=true&ps=100&facets=severities%2Ctypes&componentKeys=onthegomap_planetiler&organization=onthegomap&additionalFields=_all`; + console.log("Fetching " + url); + const response = await github.request(url); + console.log("Got " + JSON.stringify(response.data)); + response.data.issues.forEach(issue => { + try { + if (issue.severity === 'INFO') return; + const textRange = issue.textRange; + const rule = encodeURIComponent(issue.rule); + const message = [ + issue.message, + '', + `rule: ${issue.rule} (https://sonarcloud.io/organizations/onthegomap/rules?open=${rule}&rule_key=${rule})`, + `issue url: https://sonarcloud.io/project/issues?pullRequest=${pr}&open=${encodeURIComponent(issue.key)}&id=onthegomap_planetiler` + ].join('\n'); + const args = { + title: `${issue.severity} ${issue.type}`, + file: issue.component.replace(/^[^:]*:/, ''), + startLine: textRange.startLine, + endLine: textRange.endLine, + startColumn: textRange.startOffset, + endColumn: textRange.endOffset + }; + core.warning(message, args); + console.log(args); + } catch (e) { + core.error(`Unable to parse sonar issue: ${JSON.stringify(issue)}`); + } + }); diff --git a/.github/workflows/update-pr.yml b/.github/workflows/update-pr.yml index 292d148c..e5bbf66c 100644 --- a/.github/workflows/update-pr.yml +++ b/.github/workflows/update-pr.yml @@ -22,7 +22,7 @@ jobs: with: submodules: true - name: 'Download branch build info' - uses: dawidd6/action-download-artifact@v2 + uses: dawidd6/action-download-artifact@v3 with: workflow: ${{ github.event.workflow_run.workflow_id }} run_id: ${{ github.event.workflow_run.id }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a55bb979..0bd8b1a2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,12 +11,12 @@ Pull requests are welcome! Any pull request should: To set up your local development environment: - Fork the repo [setup submodules](README.md#git-submodules) -- Install Java 17 or later. You can download Java manually from [Adoptium](https://adoptium.net/installation.html) or +- Install Java 21 or later. You can download Java manually from [Adoptium](https://adoptium.net/installation.html) or use: - [Windows installer](https://adoptium.net/installation.html#windows-msi) - [macOS installer](https://adoptium.net/installation.html#macos-pkg) (or `brew install --cask temurin`, - or `port install openjdk17-temurin`) - - [Linux installer](https://adoptium.net/installation/linux/) (or `apt-get install openjdk-17-jdk`) + or `port install openjdk21-temurin`) + - [Linux installer](https://adoptium.net/installation/linux/) (or `apt-get install openjdk-21-jdk`) - Build and run the tests ([mvnw](https://github.com/takari/maven-wrapper) automatically downloads maven the first time you run it): - on mac/linux: `./mvnw clean test` @@ -54,7 +54,7 @@ Troubleshooting: - If any java source files show "Cannot resolve symbol..." errors for Planetiler classes, you might need to select: `File -> Invalidate Caches... -> Just Restart`. -- If you see a "Project JDK is not defined" error, then choose `Setup SDK` and point IntelliJ at the Java 17 or later +- If you see a "Project JDK is not defined" error, then choose `Setup SDK` and point IntelliJ at the Java 21 or later installed on your system ### Visual Studio Code diff --git a/PLANET.md b/PLANET.md index 6c8bd268..b0fa28e6 100644 --- a/PLANET.md +++ b/PLANET.md @@ -3,9 +3,9 @@ To generate a map of the world using the built-in [OpenMapTiles profile](https://github.com/openmaptiles/planetiler-openmaptiles), you will need a machine with -Java 17 or later installed and at least 10x as much disk space and at least 0.5x as much RAM as the `planet.osm.pbf` +Java 21 or later installed and at least 10x as much disk space and at least 0.5x as much RAM as the `planet.osm.pbf` file you start from. All testing has been done using Digital Ocean droplets with dedicated -vCPUs ([referral link](https://m.do.co/c/a947e99aab25)) and OpenJDK 17 installed through `apt`. Planetiler splits work +vCPUs ([referral link](https://m.do.co/c/a947e99aab25)) and OpenJDK 21 installed through `apt`. Planetiler splits work among available CPUs so the more you have, the less time it takes. ### 1) Choose the Data Source @@ -84,10 +84,10 @@ To generate the tiles shown on https://onthegomap.github.io/planetiler-demo/ I u S3 snapshot, then ran Planetiler on a Digital Ocean Memory-Optimized droplet with 16 CPUs, 128GB RAM, and 1.17TB disk running Ubuntu 21.04 x64 in the nyc3 location. -First, I installed java 17 jre and screen: +First, I installed java 21 jre and screen: ```bash -apt-get update && apt-get install -y openjdk-17-jre-headless screen +apt-get update && apt-get install -y openjdk-21-jre-headless screen ``` Then I added a script `runworld.sh` to run with 100GB of RAM: diff --git a/README.md b/README.md index 25bfa2ec..95acf2a3 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ the [OpenStreetMap Americana Project](https://github.com/ZeLonewolf/openstreetma To generate a map of an area using the [OpenMapTiles profile](https://github.com/openmaptiles/planetiler-openmaptiles), you will need: -- Java 17+ (see [CONTRIBUTING.md](CONTRIBUTING.md)) or [Docker](https://docs.docker.com/get-docker/) +- Java 21+ (see [CONTRIBUTING.md](CONTRIBUTING.md)) or [Docker](https://docs.docker.com/get-docker/) - at least 1GB of free disk space plus 5-10x the size of the `.osm.pbf` file - at least 0.5x as much free RAM as the input `.osm.pbf` file size diff --git a/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesRead.java b/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesRead.java index b9255ffa..e0a19f94 100644 --- a/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesRead.java +++ b/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesRead.java @@ -47,7 +47,7 @@ public class BenchmarkMbtilesRead { List randomCoordsToFetchPerRepetition = new LinkedList<>(); do { - try (var db = Mbtiles.newReadOnlyDatabase(mbtilesPaths.get(0))) { + try (var db = Mbtiles.newReadOnlyDatabase(mbtilesPaths.getFirst())) { try (var statement = db.connection().prepareStatement(SELECT_RANDOM_COORDS)) { statement.setInt(1, nrTileReads - randomCoordsToFetchPerRepetition.size()); var rs = statement.executeQuery(); diff --git a/planetiler-core/pom.xml b/planetiler-core/pom.xml index 89c2e2a4..621a2199 100644 --- a/planetiler-core/pom.xml +++ b/planetiler-core/pom.xml @@ -16,11 +16,11 @@ - 30.0 - 2.20.0 + 30.1 + 2.22.0 0.16.0 - 3.24.4 - 6.6.3 + 3.25.1 + 6.6.4 @@ -32,7 +32,7 @@ org.roaringbitmap RoaringBitmap - 1.0.0 + 1.0.1 com.google.protobuf @@ -67,7 +67,7 @@ org.xerial sqlite-jdbc - 3.43.0.0 + 3.44.1.0 org.msgpack @@ -142,7 +142,7 @@ com.ibm.icu icu4j - 73.2 + 74.2 com.google.guava diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureCollector.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureCollector.java index 1ca64476..6dcba8a2 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureCollector.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureCollector.java @@ -15,6 +15,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.TreeMap; +import org.locationtech.jts.algorithm.construct.MaximumInscribedCircle; import org.locationtech.jts.geom.Geometry; /** @@ -22,16 +23,13 @@ import org.locationtech.jts.geom.Geometry; * feature. *

* For example to add a polygon feature for a lake and a center label point with its name: - * - * 

- * {@code
+ * {@snippet :
  * featureCollector.polygon("water")
  *   .setAttr("class", "lake");
  * featureCollector.centroid("water_name")
  *   .setAttr("class", "lake")
  *   .setAttr("name", element.getString("name"));
  * }
- *
*/ public class FeatureCollector implements Iterable { @@ -177,15 +175,55 @@ public class FeatureCollector implements Iterable { } } - public Feature innermostPoint(String layer) { + /** + * Starts building a new point map feature at the furthest interior point of a polygon from its edge using + * {@link MaximumInscribedCircle} (aka "pole of inaccessibility") of the source feature. + *

+ * NOTE: This is substantially more expensive to compute than {@link #centroid(String)} or + * {@link #pointOnSurface(String)}, especially for small {@code tolerance} values. + * + * @param layer the output vector tile layer this feature will be written to + * @param tolerance precision for calculating maximum inscribed circle. 0.01 means 1% of the square root of the area. + * Smaller values for a more precise tolerance become very expensive to compute. Values between 5% + * and 10% are a good compromise of performance vs. precision. + * @return a feature that can be configured further. + */ + public Feature innermostPoint(String layer, double tolerance) { try { - return geometry(layer, source.innermostPoint()); + return geometry(layer, source.innermostPoint(tolerance)); } catch (GeometryException e) { - e.log(stats, "feature_innermost_point", "Error getting innermost point for " + source.id() + " layer=" + layer); + e.log(stats, "feature_innermost_point", "Error constructing innermost point for " + source.id()); return new Feature(layer, EMPTY_GEOM, source.id()); } } + /** Alias for {@link #innermostPoint(String, double)} with a default tolerance of 10%. */ + public Feature innermostPoint(String layer) { + return innermostPoint(layer, 0.1); + } + + /** Returns the minimum zoom level at which this feature is at least {@code pixelSize} pixels large. */ + public int getMinZoomForPixelSize(double pixelSize) { + try { + return GeoUtils.minZoomForPixelSize(source.size(), pixelSize); + } catch (GeometryException e) { + e.log(stats, "min_zoom_for_size_failure", "Error getting min zoom for size from geometry " + source.id()); + return config.maxzoom(); + } + } + + + /** Returns the actual pixel size of the source feature at {@code zoom} (length if line, sqrt(area) if polygon). */ + public double getPixelSizeAtZoom(int zoom) { + try { + return source.size() * (256 << zoom); + } catch (GeometryException e) { + e.log(stats, "source_feature_pixel_size_at_zoom_failure", + "Error getting source feature pixel size at zoom from geometry " + source.id()); + return 0; + } + } + /** * Creates new feature collector instances for each source feature that we encounter. */ @@ -244,6 +282,10 @@ public class FeatureCollector implements Iterable { this.geom = geom; this.geometryType = GeometryType.typeOf(geom); this.id = id; + if (geometryType == GeometryType.POINT) { + minPixelSizeAtMaxZoom = 0; + defaultMinPixelSize = 0; + } } /** Returns the original ID of the source feature that this feature came from (i.e. OSM node/way ID). */ @@ -273,8 +315,8 @@ public class FeatureCollector implements Iterable { /** * Sets the value by which features are sorted within a layer in the output vector tile. Sort key gets packed into - * {@link FeatureGroup#SORT_KEY_BITS} bits so the range of this is limited to {@code -(2^(bits-1))} to {@code - * (2^(bits-1))-1}. + * {@link FeatureGroup#SORT_KEY_BITS} bits so the range of this is limited to {@code -(2^(bits-1))} to + * {@code (2^(bits-1))-1}. *

* Circles, lines, and polygons are rendered in the order they appear in each layer, so features that appear later * (higher sort key) show up on top of features with a lower sort key. @@ -685,6 +727,29 @@ public class FeatureCollector implements Iterable { return setAttr(key, ZoomFunction.minZoom(minzoom, value)); } + /** + * Sets the value for {@code key} only at zoom levels where the feature is at least {@code minPixelSize} pixels in + * size. + */ + public Feature setAttrWithMinSize(String key, Object value, double minPixelSize) { + return setAttrWithMinzoom(key, value, getMinZoomForPixelSize(minPixelSize)); + } + + /** + * Sets the value for {@code key} so that it always shows when {@code zoom_level >= minZoomToShowAlways} but only + * shows when {@code minZoomIfBigEnough <= zoom_level < minZoomToShowAlways} when it is at least + * {@code minPixelSize} pixels in size. + *

+ * If you need more flexibility, use {@link #getMinZoomForPixelSize(double)} directly, or create a + * {@link ZoomFunction} that calculates {@link #getPixelSizeAtZoom(int)} and applies a custom threshold based on the + * zoom level. + */ + public Feature setAttrWithMinSize(String key, Object value, double minPixelSize, int minZoomIfBigEnough, + int minZoomToShowAlways) { + return setAttrWithMinzoom(key, value, + Math.clamp(getMinZoomForPixelSize(minPixelSize), minZoomIfBigEnough, minZoomToShowAlways)); + } + /** * Inserts all key/value pairs in {@code attrs} into the set of attribute to emit on the output feature at or above * {@code minzoom}. @@ -720,6 +785,14 @@ public class FeatureCollector implements Iterable { return this; } + /** + * Returns the attribute key that the renderer should use to store the number of points in the simplified geometry + * before slicing it into tiles. + */ + public String getNumPointsAttr() { + return numPointsAttr; + } + /** * Sets a special attribute key that the renderer will use to store the number of points in the simplified geometry * before slicing it into tiles. @@ -729,14 +802,6 @@ public class FeatureCollector implements Iterable { return this; } - /** - * Returns the attribute key that the renderer should use to store the number of points in the simplified geometry - * before slicing it into tiles. - */ - public String getNumPointsAttr() { - return numPointsAttr; - } - @Override public String toString() { return "Feature{" + @@ -745,5 +810,10 @@ public class FeatureCollector implements Iterable { ", attrs=" + attrs + '}'; } + + /** Returns the actual pixel size of the source feature at {@code zoom} (length if line, sqrt(area) if polygon). */ + public double getSourceFeaturePixelSizeAtZoom(int zoom) { + return getPixelSizeAtZoom(zoom); + } } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java index 202178d8..6e0a5482 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java @@ -28,6 +28,8 @@ import org.locationtech.jts.geom.LineString; import org.locationtech.jts.geom.LinearRing; import org.locationtech.jts.geom.Polygon; import org.locationtech.jts.geom.Polygonal; +import org.locationtech.jts.geom.TopologyException; +import org.locationtech.jts.geom.util.GeometryFixer; import org.locationtech.jts.index.strtree.STRtree; import org.locationtech.jts.operation.buffer.BufferOp; import org.locationtech.jts.operation.buffer.BufferParameters; @@ -124,7 +126,7 @@ public class FeatureMerge { List result = new ArrayList<>(features.size()); var groupedByAttrs = groupByAttrs(features, result, geometryType); for (List groupedFeatures : groupedByAttrs) { - VectorTile.Feature feature1 = groupedFeatures.get(0); + VectorTile.Feature feature1 = groupedFeatures.getFirst(); if (groupedFeatures.size() == 1) { result.add(feature1); } else { @@ -158,7 +160,7 @@ public class FeatureMerge { List result = new ArrayList<>(features.size()); var groupedByAttrs = groupByAttrs(features, result, GeometryType.LINE); for (List groupedFeatures : groupedByAttrs) { - VectorTile.Feature feature1 = groupedFeatures.get(0); + VectorTile.Feature feature1 = groupedFeatures.getFirst(); double lengthLimit = lengthLimitCalculator.apply(feature1.attrs()); // as a shortcut, can skip line merging only if: @@ -300,7 +302,7 @@ public class FeatureMerge { Collection> groupedByAttrs = groupByAttrs(features, result, GeometryType.POLYGON); for (List groupedFeatures : groupedByAttrs) { List outPolygons = new ArrayList<>(); - VectorTile.Feature feature1 = groupedFeatures.get(0); + VectorTile.Feature feature1 = groupedFeatures.getFirst(); List geometries = new ArrayList<>(groupedFeatures.size()); for (var feature : groupedFeatures) { try { @@ -322,7 +324,7 @@ public class FeatureMerge { // spinning for a very long time on very dense tiles. // TODO use some heuristic to choose bufferUnbuffer vs. bufferUnionUnbuffer based on the number small // polygons in the group? - merged = bufferUnionUnbuffer(buffer, polygonGroup); + merged = bufferUnionUnbuffer(buffer, polygonGroup, stats); } else { merged = buffer(buffer, GeoUtils.createGeometryCollection(polygonGroup)); } @@ -331,7 +333,7 @@ public class FeatureMerge { } merged = GeoUtils.snapAndFixPolygon(merged, stats, "merge").reverse(); } else { - merged = polygonGroup.get(0); + merged = polygonGroup.getFirst(); if (!(merged instanceof Polygonal) || merged.getEnvelopeInternal().getArea() < minArea) { continue; } @@ -410,7 +412,7 @@ public class FeatureMerge { * Merges nearby polygons by expanding each individual polygon by {@code buffer}, unioning them, and contracting the * result. */ - private static Geometry bufferUnionUnbuffer(double buffer, List polygonGroup) { + static Geometry bufferUnionUnbuffer(double buffer, List polygonGroup, Stats stats) { /* * A simpler alternative that might initially appear faster would be: * @@ -424,11 +426,20 @@ public class FeatureMerge { * The following approach is slower most of the time, but faster on average because it does * not choke on dense nearby polygons: */ - for (int i = 0; i < polygonGroup.size(); i++) { - polygonGroup.set(i, buffer(buffer, polygonGroup.get(i))); + List buffered = new ArrayList<>(polygonGroup.size()); + for (Geometry geometry : polygonGroup) { + buffered.add(buffer(buffer, geometry)); + } + Geometry merged = GeoUtils.createGeometryCollection(buffered); + try { + merged = union(merged); + } catch (TopologyException e) { + // buffer result is sometimes invalid, which makes union throw so fix + // it and try again (see #700) + stats.dataError("buffer_union_unbuffer_union_failed"); + merged = GeometryFixer.fix(merged); + merged = union(merged); } - Geometry merged = GeoUtils.createGeometryCollection(polygonGroup); - merged = union(merged); merged = unbuffer(buffer, merged); return merged; } @@ -572,5 +583,5 @@ public class FeatureMerge { return result; } - private record WithIndex (T feature, int hilbert) {} + private record WithIndex(T feature, int hilbert) {} } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java index ef3e7709..c53583f0 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java @@ -907,7 +907,7 @@ public class Planetiler { private void download() { var timer = stats.startStage("download"); - Downloader downloader = Downloader.create(config(), stats()); + Downloader downloader = Downloader.create(config()); for (ToDownload toDownload : toDownload) { if (profile.caresAboutSource(toDownload.id)) { downloader.add(toDownload.id, toDownload.url, toDownload.path); @@ -920,7 +920,7 @@ public class Planetiler { private void ensureInputFilesExist() { for (InputPath inputPath : inputPaths) { if (profile.caresAboutSource(inputPath.id) && !Files.exists(inputPath.path)) { - throw new IllegalArgumentException(inputPath.path + " does not exist"); + throw new IllegalArgumentException(inputPath.path + " does not exist. Run with --download to fetch it"); } } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java index 6e3d7df4..2ba97126 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java @@ -27,6 +27,7 @@ import com.onthegomap.planetiler.geo.GeometryException; import com.onthegomap.planetiler.geo.GeometryType; import com.onthegomap.planetiler.geo.MutableCoordinateSequence; import com.onthegomap.planetiler.util.Hilbert; +import com.onthegomap.planetiler.util.LayerAttrStats; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -80,6 +81,7 @@ public class VectorTile { private static final int EXTENT = 4096; private static final double SIZE = 256d; private final Map layers = new LinkedHashMap<>(); + private LayerAttrStats.Updater.ForZoom layerStatsTracker = LayerAttrStats.Updater.ForZoom.NOOP; private static int[] getCommands(Geometry input, int scale) { var encoder = new CommandEncoder(scale); @@ -263,7 +265,7 @@ public class VectorTile { lineStrings.add(gf.createLineString(coordSeq)); } if (lineStrings.size() == 1) { - geometry = lineStrings.get(0); + geometry = lineStrings.getFirst(); } else if (lineStrings.size() > 1) { geometry = gf.createMultiLineString(lineStrings.toArray(new LineString[0])); } @@ -305,12 +307,12 @@ public class VectorTile { } List polygons = new ArrayList<>(); for (List rings : polygonRings) { - LinearRing shell = rings.get(0); + LinearRing shell = rings.getFirst(); LinearRing[] holes = rings.subList(1, rings.size()).toArray(new LinearRing[rings.size() - 1]); polygons.add(gf.createPolygon(shell, holes)); } if (polygons.size() == 1) { - geometry = polygons.get(0); + geometry = polygons.getFirst(); } if (polygons.size() > 1) { geometry = gf.createMultiPolygon(GeometryFactory.toPolygonArray(polygons)); @@ -376,7 +378,7 @@ public class VectorTile { for (VectorTileProto.Tile.Feature feature : layer.getFeaturesList()) { int tagsCount = feature.getTagsCount(); - Map attrs = new HashMap<>(tagsCount / 2); + Map attrs = HashMap.newHashMap(tagsCount / 2); int tagIdx = 0; while (tagIdx < feature.getTagsCount()) { String key = keys.get(feature.getTags(tagIdx++)); @@ -467,12 +469,12 @@ public class VectorTile { if (features.isEmpty()) { return this; } - Layer layer = layers.get(layerName); if (layer == null) { layer = new Layer(); layers.put(layerName, layer); } + var statsTracker = layerStatsTracker.forLayer(layerName); for (Feature inFeature : features) { if (inFeature != null && inFeature.geometry().commands().length > 0) { @@ -481,8 +483,11 @@ public class VectorTile { for (Map.Entry e : inFeature.attrs().entrySet()) { // skip attribute without value if (e.getValue() != null) { - outFeature.tags.add(layer.key(e.getKey())); - outFeature.tags.add(layer.value(e.getValue())); + String key = e.getKey(); + Object value = e.getValue(); + outFeature.tags.add(layer.key(key)); + outFeature.tags.add(layer.value(value)); + statsTracker.accept(key, value); } } @@ -509,20 +514,14 @@ public class VectorTile { for (Object value : layer.values()) { VectorTileProto.Tile.Value.Builder tileValue = VectorTileProto.Tile.Value.newBuilder(); - if (value instanceof String stringValue) { - tileValue.setStringValue(stringValue); - } else if (value instanceof Integer intValue) { - tileValue.setSintValue(intValue); - } else if (value instanceof Long longValue) { - tileValue.setSintValue(longValue); - } else if (value instanceof Float floatValue) { - tileValue.setFloatValue(floatValue); - } else if (value instanceof Double doubleValue) { - tileValue.setDoubleValue(doubleValue); - } else if (value instanceof Boolean booleanValue) { - tileValue.setBoolValue(booleanValue); - } else { - tileValue.setStringValue(value.toString()); + switch (value) { + case String stringValue -> tileValue.setStringValue(stringValue); + case Integer intValue -> tileValue.setSintValue(intValue); + case Long longValue -> tileValue.setSintValue(longValue); + case Float floatValue -> tileValue.setFloatValue(floatValue); + case Double doubleValue -> tileValue.setDoubleValue(doubleValue); + case Boolean booleanValue -> tileValue.setBoolValue(booleanValue); + case Object other -> tileValue.setStringValue(other.toString()); } tileLayer.addValues(tileValue.build()); } @@ -600,6 +599,15 @@ public class VectorTile { return layers.values().stream().allMatch(v -> v.encodedFeatures.isEmpty()) || containsOnlyFillsOrEdges(); } + /** + * Call back to {@code layerStats} as vector tile features are being encoded in + * {@link #addLayerFeatures(String, List)} to track attribute types present on features in each layer, for example to + * emit in tilejson metadata stats. + */ + public void trackLayerStats(LayerAttrStats.Updater.ForZoom layerStats) { + this.layerStatsTracker = layerStats; + } + enum Command { MOVE_TO(1), LINE_TO(2), @@ -1072,31 +1080,32 @@ public class VectorTile { } void accept(Geometry geometry) { - if (geometry instanceof MultiLineString multiLineString) { - for (int i = 0; i < multiLineString.getNumGeometries(); i++) { - encode(((LineString) multiLineString.getGeometryN(i)).getCoordinateSequence(), false, GeometryType.LINE); + switch (geometry) { + case MultiLineString multiLineString -> { + for (int i = 0; i < multiLineString.getNumGeometries(); i++) { + encode(((LineString) multiLineString.getGeometryN(i)).getCoordinateSequence(), false, GeometryType.LINE); + } } - } else if (geometry instanceof Polygon polygon) { - LineString exteriorRing = polygon.getExteriorRing(); - encode(exteriorRing.getCoordinateSequence(), true, GeometryType.POLYGON); - - for (int i = 0; i < polygon.getNumInteriorRing(); i++) { - LineString interiorRing = polygon.getInteriorRingN(i); - encode(interiorRing.getCoordinateSequence(), true, GeometryType.LINE); + case Polygon polygon -> { + LineString exteriorRing = polygon.getExteriorRing(); + encode(exteriorRing.getCoordinateSequence(), true, GeometryType.POLYGON); + for (int i = 0; i < polygon.getNumInteriorRing(); i++) { + LineString interiorRing = polygon.getInteriorRingN(i); + encode(interiorRing.getCoordinateSequence(), true, GeometryType.LINE); + } } - } else if (geometry instanceof MultiPolygon multiPolygon) { - for (int i = 0; i < multiPolygon.getNumGeometries(); i++) { - accept(multiPolygon.getGeometryN(i)); + case MultiPolygon multiPolygon -> { + for (int i = 0; i < multiPolygon.getNumGeometries(); i++) { + accept(multiPolygon.getGeometryN(i)); + } } - } else if (geometry instanceof LineString lineString) { - encode(lineString.getCoordinateSequence(), shouldClosePath(geometry), GeometryType.LINE); - } else if (geometry instanceof Point point) { - encode(point.getCoordinateSequence(), false, GeometryType.POINT); - } else if (geometry instanceof Puntal) { - encode(new CoordinateArraySequence(geometry.getCoordinates()), shouldClosePath(geometry), + case LineString lineString -> + encode(lineString.getCoordinateSequence(), shouldClosePath(geometry), GeometryType.LINE); + case Point point -> encode(point.getCoordinateSequence(), false, GeometryType.POINT); + case Puntal ignored -> encode(new CoordinateArraySequence(geometry.getCoordinates()), shouldClosePath(geometry), geometry instanceof MultiPoint, GeometryType.POINT); - } else { - LOGGER.warn("Unrecognized geometry type: " + geometry.getGeometryType()); + case null -> LOGGER.warn("Null geometry type"); + default -> LOGGER.warn("Unrecognized geometry type: " + geometry.getGeometryType()); } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java index afa6048b..ad67d041 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java @@ -15,6 +15,7 @@ import com.onthegomap.planetiler.stats.Timer; import com.onthegomap.planetiler.util.DiskBacked; import com.onthegomap.planetiler.util.Format; import com.onthegomap.planetiler.util.Hashing; +import com.onthegomap.planetiler.util.LayerAttrStats; import com.onthegomap.planetiler.util.TileSizeStats; import com.onthegomap.planetiler.util.TileWeights; import com.onthegomap.planetiler.util.TilesetSummaryStatistics; @@ -59,6 +60,7 @@ public class TileArchiveWriter { private final AtomicReference lastTileWritten = new AtomicReference<>(); private final TileArchiveMetadata tileArchiveMetadata; private final TilesetSummaryStatistics tileStats; + private final LayerAttrStats layerAttrStats = new LayerAttrStats(); private TileArchiveWriter(Iterable inputTiles, WriteableTileArchive archive, PlanetilerConfig config, TileArchiveMetadata tileArchiveMetadata, Stats stats) { @@ -105,9 +107,7 @@ public class TileArchiveWriter { readWorker = reader.readWorker(); } - TileArchiveWriter writer = - new TileArchiveWriter(inputTiles, output, config, tileArchiveMetadata.withLayerStats(features.layerStats() - .getTileStats()), stats); + TileArchiveWriter writer = new TileArchiveWriter(inputTiles, output, config, tileArchiveMetadata, stats); var pipeline = WorkerPipeline.start("archive", stats); @@ -260,6 +260,7 @@ public class TileArchiveWriter { boolean skipFilled = config.skipFilledTiles(); var tileStatsUpdater = tileStats.threadLocalUpdater(); + var layerAttrStatsUpdater = layerAttrStats.handlerForThread(); for (TileBatch batch : prev) { List result = new ArrayList<>(batch.size()); FeatureGroup.TileFeatures last = null; @@ -277,7 +278,7 @@ public class TileArchiveWriter { layerStats = lastLayerStats; memoizedTiles.inc(); } else { - VectorTile tile = tileFeatures.getVectorTile(); + VectorTile tile = tileFeatures.getVectorTile(layerAttrStatsUpdater); if (skipFilled && (lastIsFill = tile.containsOnlyFills())) { encoded = null; layerStats = null; @@ -333,7 +334,7 @@ public class TileArchiveWriter { var f = NumberFormat.getNumberInstance(Locale.getDefault()); f.setMaximumFractionDigits(5); - archive.initialize(tileArchiveMetadata); + archive.initialize(); var order = archive.tileOrder(); TileCoord lastTile = null; @@ -371,7 +372,7 @@ public class TileArchiveWriter { LOGGER.info("Finished z{} in {}", currentZ, time.stop()); } - archive.finish(tileArchiveMetadata); + archive.finish(tileArchiveMetadata.withLayerStats(layerAttrStats.getTileStats())); } @SuppressWarnings("java:S2629") diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java index 26c2a3bc..d3852b08 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java @@ -30,7 +30,7 @@ public interface WriteableTileArchive extends Closeable { * Called before any tiles are written into {@link TileWriter}. Implementations of TileArchive should set up any * required state here. */ - default void initialize(TileArchiveMetadata metadata) {} + default void initialize() {} /** * Implementations should return a object that implements {@link TileWriter} The specific TileWriter returned might diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ArrayLongLongMapMmap.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ArrayLongLongMapMmap.java index e208248b..17791e6e 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ArrayLongLongMapMmap.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ArrayLongLongMapMmap.java @@ -95,7 +95,7 @@ class ArrayLongLongMapMmap implements LongLongMap.ParallelWrites { int minChunks = 1; int maxChunks = (int) (MAX_BYTES_TO_USE / chunkSize); int targetChunks = (int) (ProcessInfo.getMaxMemoryBytes() * 0.5d / chunkSize); - return Math.min(maxChunks, Math.max(minChunks, targetChunks)); + return Math.clamp(targetChunks, minChunks, maxChunks); } public void init() { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ExternalMergeSort.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ExternalMergeSort.java index c63269e7..11a63af5 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ExternalMergeSort.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ExternalMergeSort.java @@ -185,7 +185,7 @@ class ExternalMergeSort implements FeatureSort { .sinkToConsumer("worker", workers, group -> { try { readSemaphore.acquire(); - var chunk = group.get(0); + var chunk = group.getFirst(); var others = group.stream().skip(1).toList(); var toSort = time(reading, () -> { // merge all chunks into first one, and remove the others diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java index d990f62b..ba074ef7 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java @@ -27,7 +27,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Consumer; import java.util.function.Function; import javax.annotation.concurrent.NotThreadSafe; import org.msgpack.core.MessageBufferPacker; @@ -59,7 +58,6 @@ public final class FeatureGroup implements Iterable, private final CommonStringEncoder.AsByte commonLayerStrings = new CommonStringEncoder.AsByte(); private final CommonStringEncoder commonValueStrings = new CommonStringEncoder(100_000); private final Stats stats; - private final LayerAttrStats layerStats = new LayerAttrStats(); private final PlanetilerConfig config; private volatile boolean prepared = false; private final TileOrder tileOrder; @@ -141,14 +139,6 @@ public final class FeatureGroup implements Iterable, return (byte) ((geometry.geomType().asByte() & 0xff) | (geometry.scale() << 3)); } - /** - * Returns statistics about each layer written through {@link #newRenderedFeatureEncoder()} including min/max zoom, - * features on elements in that layer, and their types. - */ - public LayerAttrStats layerStats() { - return layerStats; - } - public long numFeaturesWritten() { return sorter.numFeaturesWritten(); } @@ -159,16 +149,13 @@ public final class FeatureGroup implements Iterable, // This method gets called billions of times when generating the planet, so these optimizations make a big difference: // 1) Re-use the same buffer packer to avoid allocating and resizing new byte arrays for every feature. private final MessageBufferPacker packer = MessagePack.newDefaultBufferPacker(); - // 2) Avoid a ThreadLocal lookup on every layer stats call by getting the handler for this thread once - private final Consumer threadLocalLayerStats = layerStats.handlerForThread(); - // 3) Avoid re-encoding values for identical filled geometries (i.e. ocean) by memoizing the encoded values + // 2) Avoid re-encoding values for identical filled geometries (i.e. ocean) by memoizing the encoded values // FeatureRenderer ensures that a separate VectorTileEncoder.Feature is used for each zoom level private VectorTile.Feature lastFeature = null; private byte[] lastEncodedValue = null; @Override public SortableFeature apply(RenderedFeature feature) { - threadLocalLayerStats.accept(feature); var group = feature.group().orElse(null); var thisFeature = feature.vectorTileFeature(); byte[] encodedValue; @@ -217,24 +204,18 @@ public final class FeatureGroup implements Iterable, var attrs = vectorTileFeature.attrs(); packer.packMapHeader((int) attrs.values().stream().filter(Objects::nonNull).count()); for (Map.Entry entry : attrs.entrySet()) { - if (entry.getValue() != null) { + Object value = entry.getValue(); + if (value != null) { packer.packInt(commonValueStrings.encode(entry.getKey())); - Object value = entry.getValue(); - if (value instanceof String string) { - packer.packValue(ValueFactory.newString(string)); - } else if (value instanceof Integer integer) { - packer.packValue(ValueFactory.newInteger(integer.longValue())); - } else if (value instanceof Long longValue) { - packer.packValue(ValueFactory.newInteger(longValue)); - } else if (value instanceof Float floatValue) { - packer.packValue(ValueFactory.newFloat(floatValue)); - } else if (value instanceof Double doubleValue) { - packer.packValue(ValueFactory.newFloat(doubleValue)); - } else if (value instanceof Boolean booleanValue) { - packer.packValue(ValueFactory.newBoolean(booleanValue)); - } else { - packer.packValue(ValueFactory.newString(value.toString())); - } + packer.packValue(switch (value) { + case String string -> ValueFactory.newString(string); + case Integer integer -> ValueFactory.newInteger(integer.longValue()); + case Long longValue -> ValueFactory.newInteger(longValue); + case Float floatValue -> ValueFactory.newFloat(floatValue); + case Double doubleValue -> ValueFactory.newFloat(doubleValue); + case Boolean booleanValue -> ValueFactory.newBoolean(booleanValue); + case Object other -> ValueFactory.newString(other.toString()); + }); } } // Use the same binary format for encoding geometries in output vector tiles. Benchmarking showed @@ -423,7 +404,7 @@ public final class FeatureGroup implements Iterable, GeometryType geomType = decodeGeomType(geomTypeAndScale); int scale = decodeScale(geomTypeAndScale); int mapSize = unpacker.unpackMapHeader(); - Map attrs = new HashMap<>(mapSize); + Map attrs = HashMap.newHashMap(mapSize); for (int i = 0; i < mapSize; i++) { String key = commonValueStrings.decode(unpacker.unpackInt()); Value v = unpacker.unpackValue(); @@ -456,7 +437,14 @@ public final class FeatureGroup implements Iterable, } public VectorTile getVectorTile() { + return getVectorTile(null); + } + + public VectorTile getVectorTile(LayerAttrStats.Updater layerStats) { VectorTile tile = new VectorTile(); + if (layerStats != null) { + tile.trackLayerStats(layerStats.forZoom(tileCoord.z())); + } List items = new ArrayList<>(entries.size()); String currentLayer = null; for (SortableFeature entry : entries) { @@ -494,7 +482,7 @@ public final class FeatureGroup implements Iterable, // log failures, only throwing when it's a fatal error if (e instanceof GeometryException geoe) { geoe.log(stats, "postprocess_layer", - "Caught error postprocessing features for " + layer + " layer on " + tileCoord); + "Caught error postprocessing features for " + layer + " layer on " + tileCoord, config.logJtsExceptions()); } else if (e instanceof Error err) { LOGGER.error("Caught fatal error postprocessing features {} {}", layer, tileCoord, e); throw err; diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Bounds.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Bounds.java index a6243a8c..2f322def 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Bounds.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Bounds.java @@ -17,6 +17,7 @@ import org.slf4j.LoggerFactory; public class Bounds { private static final Logger LOGGER = LoggerFactory.getLogger(Bounds.class); + public static final Bounds WORLD = new Bounds(null); private Envelope latLon; private Envelope world; @@ -24,14 +25,10 @@ public class Bounds { private Geometry shape; - Bounds(Envelope latLon) { + public Bounds(Envelope latLon) { set(latLon); } - public boolean isWorld() { - return latLon == null || latLon.contains(GeoUtils.WORLD_LAT_LON_BOUNDS); - } - public Envelope latLon() { return latLon == null ? GeoUtils.WORLD_LAT_LON_BOUNDS : latLon; } @@ -40,6 +37,10 @@ public class Bounds { return world == null ? GeoUtils.WORLD_BOUNDS : world; } + public boolean isWorld() { + return latLon == null || latLon.equals(GeoUtils.WORLD_LAT_LON_BOUNDS); + } + public TileExtents tileExtents() { if (tileExtents == null) { tileExtents = TileExtents.computeFromWorldBounds(PlanetilerConfig.MAX_MAXZOOM, world(), shape); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java index 0ca34704..b9a14630 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java @@ -58,7 +58,8 @@ public record PlanetilerConfig( String debugUrlPattern, Path tmpDir, Path tileWeights, - double maxPointBuffer + double maxPointBuffer, + boolean logJtsExceptions ) { public static final int MIN_MINZOOM = 0; @@ -208,7 +209,8 @@ public record PlanetilerConfig( "Max tile pixels to include points outside tile bounds. Set to a lower value to reduce tile size for " + "clients that handle label collisions across tiles (most web and native clients). NOTE: Do not reduce if you need to support " + "raster tile rendering", - Double.POSITIVE_INFINITY) + Double.POSITIVE_INFINITY), + arguments.getBoolean("log_jts_exceptions", "Emit verbose details to debug JTS geometry errors", false) ); } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/DataType.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/DataType.java index ab93fdef..de3fdac9 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/DataType.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/DataType.java @@ -34,19 +34,14 @@ public enum DataType implements BiFunction { /** Returns the data type associated with {@code value}, or {@link #GET_TAG} as a fallback. */ public static DataType typeOf(Object value) { - if (value instanceof String) { - return GET_STRING; - } else if (value instanceof Integer) { - return GET_INT; - } else if (value instanceof Long) { - return GET_LONG; - } else if (value instanceof Double) { - return GET_DOUBLE; - } else if (value instanceof Boolean) { - return GET_BOOLEAN; - } else { - return GET_TAG; - } + return switch (value) { + case String ignored -> GET_STRING; + case Integer ignored -> GET_INT; + case Long ignored -> GET_LONG; + case Double ignored -> GET_DOUBLE; + case Boolean ignored -> GET_BOOLEAN; + default -> GET_TAG; + }; } /** Returns the data type associated with {@code id}, or {@link #GET_TAG} as a fallback. */ diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/Expression.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/Expression.java index 6fbe641b..22fa9eb0 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/Expression.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/Expression.java @@ -23,12 +23,9 @@ import org.slf4j.LoggerFactory; *

* Calling {@code toString()} on any expression will generate code that can be used to recreate an identical copy of the * original expression, assuming that the generated code includes: - * - * 

- * {@code
+ * {@snippet :
  * import static com.onthegomap.planetiler.expression.Expression.*;
  * }
- *
*/ // TODO rename to BooleanExpression public interface Expression extends Simplifiable { @@ -141,14 +138,13 @@ public interface Expression extends Simplifiable { default Expression replace(Predicate replace, Expression b) { if (replace.test(this)) { return b; - } else if (this instanceof Not not) { - return new Not(not.child.replace(replace, b)); - } else if (this instanceof Or or) { - return new Or(or.children.stream().map(child -> child.replace(replace, b)).toList()); - } else if (this instanceof And and) { - return new And(and.children.stream().map(child -> child.replace(replace, b)).toList()); } else { - return this; + return switch (this) { + case Not(var child) -> new Not(child.replace(replace, b)); + case Or(var children) -> new Or(children.stream().map(child -> child.replace(replace, b)).toList()); + case And(var children) -> new And(children.stream().map(child -> child.replace(replace, b)).toList()); + default -> this; + }; } } @@ -156,14 +152,13 @@ public interface Expression extends Simplifiable { default boolean contains(Predicate filter) { if (filter.test(this)) { return true; - } else if (this instanceof Not not) { - return not.child.contains(filter); - } else if (this instanceof Or or) { - return or.children.stream().anyMatch(child -> child.contains(filter)); - } else if (this instanceof And and) { - return and.children.stream().anyMatch(child -> child.contains(filter)); } else { - return false; + return switch (this) { + case Not(var child) -> child.contains(filter); + case Or(var children) -> children.stream().anyMatch(child -> child.contains(filter)); + case And(var children) -> children.stream().anyMatch(child -> child.contains(filter)); + default -> false; + }; } } @@ -234,7 +229,7 @@ public interface Expression extends Simplifiable { return TRUE; } if (children.size() == 1) { - return children.get(0).simplifyOnce(); + return children.getFirst().simplifyOnce(); } if (children.contains(FALSE)) { return FALSE; @@ -288,7 +283,7 @@ public interface Expression extends Simplifiable { return FALSE; } if (children.size() == 1) { - return children.get(0).simplifyOnce(); + return children.getFirst().simplifyOnce(); } if (children.contains(TRUE)) { return TRUE; diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/MultiExpression.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/MultiExpression.java index 97bc6c77..484c5693 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/MultiExpression.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/MultiExpression.java @@ -50,19 +50,15 @@ public record MultiExpression (List> expressions) implements Simplif * when a particular key is present on the input. */ private static boolean mustAlwaysEvaluate(Expression expression) { - if (expression instanceof Expression.Or or) { - return or.children().stream().anyMatch(MultiExpression::mustAlwaysEvaluate); - } else if (expression instanceof Expression.And and) { - return and.children().stream().allMatch(MultiExpression::mustAlwaysEvaluate); - } else if (expression instanceof Expression.Not not) { - return !mustAlwaysEvaluate(not.child()); - } else if (expression instanceof Expression.MatchAny any && any.matchWhenMissing()) { - return true; - } else { - return !(expression instanceof Expression.MatchAny) && - !(expression instanceof Expression.MatchField) && - !FALSE.equals(expression); - } + return switch (expression) { + case Expression.Or(var children) -> children.stream().anyMatch(MultiExpression::mustAlwaysEvaluate); + case Expression.And(var children) -> children.stream().allMatch(MultiExpression::mustAlwaysEvaluate); + case Expression.Not(var child) -> !mustAlwaysEvaluate(child); + case Expression.MatchAny any when any.matchWhenMissing() -> true; + case null, default -> !(expression instanceof Expression.MatchAny) && + !(expression instanceof Expression.MatchField) && + !FALSE.equals(expression); + }; } /** Calls {@code acceptKey} for every tag that could possibly cause {@code exp} to match an input element. */ @@ -176,7 +172,7 @@ public record MultiExpression (List> expressions) implements Simplif */ default O getOrElse(WithTags input, O defaultValue) { List matches = getMatches(input); - return matches.isEmpty() ? defaultValue : matches.get(0); + return matches.isEmpty() ? defaultValue : matches.getFirst(); } /** @@ -184,7 +180,7 @@ public record MultiExpression (List> expressions) implements Simplif */ default O getOrElse(Map tags, O defaultValue) { List matches = getMatches(WithTags.from(tags)); - return matches.isEmpty() ? defaultValue : matches.get(0); + return matches.isEmpty() ? defaultValue : matches.getFirst(); } /** Returns true if any expression matches that tags from an input element. */ diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/GeoUtils.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/GeoUtils.java index c71dbd6e..f6d0aceb 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/GeoUtils.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/GeoUtils.java @@ -1,6 +1,7 @@ package com.onthegomap.planetiler.geo; import com.onthegomap.planetiler.collection.LongLongMap; +import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.stats.Stats; import java.util.ArrayList; import java.util.List; @@ -51,6 +52,7 @@ public class GeoUtils { public static final double WORLD_CIRCUMFERENCE_METERS = Math.PI * 2 * WORLD_RADIUS_METERS; private static final double RADIANS_PER_DEGREE = Math.PI / 180; private static final double DEGREES_PER_RADIAN = 180 / Math.PI; + private static final double LOG2 = Math.log(2); /** * Transform web mercator coordinates where top-left corner of the planet is (0,0) and bottom-right is (1,1) to * latitude/longitude coordinates. @@ -281,15 +283,15 @@ public class GeoUtils { } public static Geometry combineLineStrings(List lineStrings) { - return lineStrings.size() == 1 ? lineStrings.get(0) : createMultiLineString(lineStrings); + return lineStrings.size() == 1 ? lineStrings.getFirst() : createMultiLineString(lineStrings); } public static Geometry combinePolygons(List polys) { - return polys.size() == 1 ? polys.get(0) : createMultiPolygon(polys); + return polys.size() == 1 ? polys.getFirst() : createMultiPolygon(polys); } public static Geometry combinePoints(List points) { - return points.size() == 1 ? points.get(0) : createMultiPoint(points); + return points.size() == 1 ? points.getFirst() : createMultiPoint(points); } /** @@ -383,29 +385,29 @@ public class GeoUtils { if (lineStrings.isEmpty()) { throw new GeometryException("polygon_to_linestring_empty", "No line strings"); } else if (lineStrings.size() == 1) { - return lineStrings.get(0); + return lineStrings.getFirst(); } else { return createMultiLineString(lineStrings); } } private static void getLineStrings(Geometry input, List output) throws GeometryException { - if (input instanceof LinearRing linearRing) { - output.add(JTS_FACTORY.createLineString(linearRing.getCoordinateSequence())); - } else if (input instanceof LineString lineString) { - output.add(lineString); - } else if (input instanceof Polygon polygon) { - getLineStrings(polygon.getExteriorRing(), output); - for (int i = 0; i < polygon.getNumInteriorRing(); i++) { - getLineStrings(polygon.getInteriorRingN(i), output); + switch (input) { + case LinearRing linearRing -> output.add(JTS_FACTORY.createLineString(linearRing.getCoordinateSequence())); + case LineString lineString -> output.add(lineString); + case Polygon polygon -> { + getLineStrings(polygon.getExteriorRing(), output); + for (int i = 0; i < polygon.getNumInteriorRing(); i++) { + getLineStrings(polygon.getInteriorRingN(i), output); + } } - } else if (input instanceof GeometryCollection gc) { - for (int i = 0; i < gc.getNumGeometries(); i++) { - getLineStrings(gc.getGeometryN(i), output); + case GeometryCollection gc -> { + for (int i = 0; i < gc.getNumGeometries(); i++) { + getLineStrings(gc.getGeometryN(i), output); + } } - } else { - throw new GeometryException("get_line_strings_bad_type", - "unrecognized geometry type: " + input.getGeometryType()); + case null, default -> throw new GeometryException("get_line_strings_bad_type", + "unrecognized geometry type: " + (input == null ? "null" : input.getGeometryType())); } } @@ -416,7 +418,7 @@ public class GeoUtils { /** Returns a point approximately {@code ratio} of the way from start to end and {@code offset} units to the right. */ public static Point pointAlongOffset(LineString lineString, double ratio, double offset) { int numPoints = lineString.getNumPoints(); - int middle = Math.max(0, Math.min(numPoints - 2, (int) (numPoints * ratio))); + int middle = Math.clamp((int) (numPoints * ratio), 0, numPoints - 2); Coordinate a = lineString.getCoordinateN(middle); Coordinate b = lineString.getCoordinateN(middle + 1); LineSegment segment = new LineSegment(a, b); @@ -530,10 +532,22 @@ public class GeoUtils { innerGeometries.add(geom); } } - return innerGeometries.size() == 1 ? innerGeometries.get(0) : + return innerGeometries.size() == 1 ? innerGeometries.getFirst() : JTS_FACTORY.createGeometryCollection(innerGeometries.toArray(Geometry[]::new)); } + /** + * For a feature of size {@code worldGeometrySize} (where 1=full planet), determine the minimum zoom level at which + * the feature appears at least {@code minPixelSize} pixels large. + *

+ * The result will be clamped to the range [0, {@link PlanetilerConfig#MAX_MAXZOOM}]. + */ + public static int minZoomForPixelSize(double worldGeometrySize, double minPixelSize) { + double worldPixels = worldGeometrySize * 256; + return Math.clamp((int) Math.ceil(Math.log(minPixelSize / worldPixels) / LOG2), 0, + PlanetilerConfig.MAX_MAXZOOM); + } + /** Helper class to sort polygons by area of their outer shell. */ private record PolyAndArea(Polygon poly, double area) implements Comparable { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/GeometryException.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/GeometryException.java index 765bdc91..9461b250 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/GeometryException.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/GeometryException.java @@ -1,6 +1,12 @@ package com.onthegomap.planetiler.geo; import com.onthegomap.planetiler.stats.Stats; +import java.util.ArrayList; +import java.util.Base64; +import java.util.function.Supplier; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.io.WKBWriter; +import org.locationtech.jts.io.WKTWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -14,6 +20,7 @@ public class GeometryException extends Exception { private final String stat; private final boolean nonFatal; + private final ArrayList> detailsSuppliers = new ArrayList<>(); /** * Constructs a new exception with a detailed error message caused by {@code cause}. @@ -51,6 +58,11 @@ public class GeometryException extends Exception { this.nonFatal = nonFatal; } + public GeometryException addDetails(Supplier detailsSupplier) { + this.detailsSuppliers.add(detailsSupplier); + return this; + } + /** Returns the unique code for this error condition to use for counting the number of occurrences in stats. */ public String stat() { return stat; @@ -72,6 +84,38 @@ public class GeometryException extends Exception { assert nonFatal : log; // make unit tests fail if fatal } + + /** Logs the error but if {@code logDetails} is true, then also prints detailed debugging info. */ + public void log(Stats stats, String statPrefix, String logPrefix, boolean logDetails) { + if (logDetails) { + stats.dataError(statPrefix + "_" + stat()); + StringBuilder log = new StringBuilder(logPrefix + ": " + getMessage()); + for (var details : detailsSuppliers) { + log.append("\n").append(details.get()); + } + var str = log.toString(); + LOGGER.warn(str, this.getCause() == null ? this : this.getCause()); + assert nonFatal : log.toString(); // make unit tests fail if fatal + } else { + log(stats, statPrefix, logPrefix); + } + } + + public GeometryException addGeometryDetails(String original, Geometry geometryCollection) { + return addDetails(() -> { + var wktWriter = new WKTWriter(); + var wkbWriter = new WKBWriter(); + var base64 = Base64.getEncoder(); + return """ + %s (wkt): %s + %s (wkb): %s + """.formatted( + original, wktWriter.write(geometryCollection), + original, base64.encodeToString(wkbWriter.write(geometryCollection)) + ).strip(); + }); + } + /** * An error that we expect to encounter often so should only be logged at {@code TRACE} level. */ diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/PolygonIndex.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/PolygonIndex.java index 6a2bfd82..0d06d903 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/PolygonIndex.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/PolygonIndex.java @@ -19,7 +19,7 @@ import org.locationtech.jts.index.strtree.STRtree; @ThreadSafe public class PolygonIndex { - private record GeomWithData (Polygon poly, T data) {} + private record GeomWithData(Polygon poly, T data) {} private final STRtree index = new STRtree(); @@ -45,7 +45,7 @@ public class PolygonIndex { /** Returns the data associated with the first polygon containing {@code point}. */ public T getOnlyContaining(Point point) { List result = getContaining(point); - return result.isEmpty() ? null : result.get(0); + return result.isEmpty() ? null : result.getFirst(); } /** Returns the data associated with all polygons containing {@code point}. */ @@ -77,7 +77,7 @@ public class PolygonIndex { List items = index.query(point.getEnvelopeInternal()); // optimization: if there's only one then skip checking contains/distance if (items.size() == 1) { - if (items.get(0)instanceof GeomWithData value) { + if (items.getFirst() instanceof GeomWithData value) { @SuppressWarnings("unchecked") T t = (T) value.data; return List.of(t); } @@ -108,7 +108,7 @@ public class PolygonIndex { /** Returns the data associated with a polygon that contains {@code point} or nearest polygon if none are found. */ public T get(Point point) { List nearests = getContainingOrNearest(point); - return nearests.isEmpty() ? null : nearests.get(0); + return nearests.isEmpty() ? null : nearests.getFirst(); } /** Indexes {@code item} for all polygons contained in {@code geom}. */ diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileExtents.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileExtents.java index f6313cf2..911be3c7 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileExtents.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileExtents.java @@ -21,11 +21,11 @@ public class TileExtents implements Predicate { } private static int quantizeDown(double value, int levels) { - return Math.max(0, Math.min(levels, (int) Math.floor(value * levels))); + return Math.clamp((int) Math.floor(value * levels), 0, levels); } private static int quantizeUp(double value, int levels) { - return Math.max(0, Math.min(levels, (int) Math.ceil(value * levels))); + return Math.clamp((int) Math.ceil(value * levels), 0, levels); } /** Returns a filter to tiles that intersect {@code worldBounds} (specified in world web mercator coordinates). */ diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java index b33181e2..1479c5eb 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java @@ -220,7 +220,7 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive } @Override - public void initialize(TileArchiveMetadata tileArchiveMetadata) { + public void initialize() { if (skipIndexCreation) { createTablesWithoutIndexes(); if (LOGGER.isInfoEnabled()) { @@ -230,12 +230,11 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive } else { createTablesWithIndexes(); } - - metadataTable().set(tileArchiveMetadata); } @Override public void finish(TileArchiveMetadata tileArchiveMetadata) { + metadataTable().set(tileArchiveMetadata); if (vacuumAnalyze) { vacuumAnalyze(); } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/overture/Overture.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/overture/Overture.java index 8863cea5..f8913ef0 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/overture/Overture.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/overture/Overture.java @@ -74,7 +74,7 @@ public class Overture implements Profile { } private static void downloadFiles(Path base, Planetiler pt, String release, boolean sample) { - var d = Downloader.create(pt.config(), pt.stats()); + var d = Downloader.create(pt.config()); var urls = sample ? OvertureUrls.sampleSmallest(pt.config(), "release/" + release) : OvertureUrls.getAll(pt.config(), "release/" + release); @@ -149,8 +149,8 @@ public class Overture implements Profile { String clazz = sourceFeature.getStruct().get("class").asString(); createAnyFeature(sourceFeature, features) .setMinZoom(sourceFeature.isPoint() ? 14 : switch (clazz) { - case "residential" -> 6; - default -> 9; + case "residential" -> 6; + default -> 9; }) .inheritAttrFromSource("subType") .inheritAttrFromSource("class") @@ -254,24 +254,24 @@ public class Overture implements Profile { } int minzoom = switch (subtype) { case ROAD -> switch (roadClass) { - case MOTORWAY -> 4; - case TRUNK -> 5; - case PRIMARY -> 7; - case SECONDARY -> 9; - case TERTIARY -> 11; - case RESIDENTIAL -> 12; - case LIVINGSTREET -> 13; - case UNCLASSIFIED -> 14; - case PARKINGAISLE -> 14; - case DRIVEWAY -> 14; - case PEDESTRIAN -> 14; - case FOOTWAY -> 14; - case STEPS -> 14; - case TRACK -> 14; - case CYCLEWAY -> 14; - case BRIDLEWAY -> 14; - case UNKNOWN -> 14; - }; + case MOTORWAY -> 4; + case TRUNK -> 5; + case PRIMARY -> 7; + case SECONDARY -> 9; + case TERTIARY -> 11; + case RESIDENTIAL -> 12; + case LIVINGSTREET -> 13; + case UNCLASSIFIED -> 14; + case PARKINGAISLE -> 14; + case DRIVEWAY -> 14; + case PEDESTRIAN -> 14; + case FOOTWAY -> 14; + case STEPS -> 14; + case TRACK -> 14; + case CYCLEWAY -> 14; + case BRIDLEWAY -> 14; + case UNKNOWN -> 14; + }; case RAIL -> 8; case WATER -> 10; }; @@ -433,7 +433,7 @@ public class Overture implements Profile { private static final Range FULL_LENGTH = Range.closedOpen(0.0, 1.0); - record Partial (T value, Range at) {} + record Partial(T value, Range at) {} private void processConnector(AvroParquetFeature sourceFeature, FeatureCollector features) { if (connectors) { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/GeoPackageReader.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/GeoPackageReader.java index fff8b20c..91882acb 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/GeoPackageReader.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/GeoPackageReader.java @@ -153,7 +153,7 @@ public class GeoPackageReader extends SimpleReader { Geometry latLonGeom = (transform.isIdentity()) ? featureGeom : JTS.transform(featureGeom, transform); FeatureColumns columns = feature.getColumns(); - SimpleFeature geom = SimpleFeature.create(latLonGeom, new HashMap<>(columns.columnCount()), + SimpleFeature geom = SimpleFeature.create(latLonGeom, HashMap.newHashMap(columns.columnCount()), sourceName, featureName, ++id); for (int i = 0; i < columns.columnCount(); ++i) { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/NaturalEarthReader.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/NaturalEarthReader.java index c8fe0a93..14ae862a 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/NaturalEarthReader.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/NaturalEarthReader.java @@ -177,7 +177,7 @@ public class NaturalEarthReader extends SimpleReader { // create the feature and pass to next stage Geometry latLonGeometry = GeoUtils.WKB_READER.read(geometry); - SimpleFeature readerGeometry = SimpleFeature.create(latLonGeometry, new HashMap<>(column.length - 1), + SimpleFeature readerGeometry = SimpleFeature.create(latLonGeometry, HashMap.newHashMap(column.length - 1), sourceName, table, ++id); for (int c = 0; c < column.length; c++) { if (c != geometryColumn) { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/ShapefileReader.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/ShapefileReader.java index d48f6f44..299ce80d 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/ShapefileReader.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/ShapefileReader.java @@ -2,6 +2,7 @@ package com.onthegomap.planetiler.reader; import com.onthegomap.planetiler.Profile; import com.onthegomap.planetiler.collection.FeatureGroup; +import com.onthegomap.planetiler.config.Bounds; import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.stats.Stats; import java.io.IOException; @@ -19,9 +20,13 @@ import org.geotools.api.referencing.operation.MathTransform; import org.geotools.api.referencing.operation.OperationNotFoundException; import org.geotools.api.referencing.operation.TransformException; import org.geotools.data.shapefile.ShapefileDataStore; +import org.geotools.factory.CommonFactoryFinder; import org.geotools.feature.FeatureCollection; import org.geotools.geometry.jts.JTS; +import org.geotools.geometry.jts.ReferencedEnvelope; import org.geotools.referencing.CRS; +import org.geotools.util.factory.GeoTools; +import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,6 +50,10 @@ public class ShapefileReader extends SimpleReader { private MathTransform transformToLatLon; public ShapefileReader(String sourceProjection, String sourceName, Path input) { + this(sourceProjection, sourceName, input, Bounds.WORLD); + } + + public ShapefileReader(String sourceProjection, String sourceName, Path input, Bounds bounds) { super(sourceName); this.layer = input.getFileName().toString().replaceAll("\\.shp$", ""); dataStore = open(input); @@ -52,8 +61,6 @@ public class ShapefileReader extends SimpleReader { String typeName = dataStore.getTypeNames()[0]; FeatureSource source = dataStore .getFeatureSource(typeName); - - inputSource = source.getFeatures(Filter.INCLUDE); CoordinateReferenceSystem src = sourceProjection == null ? source.getSchema().getCoordinateReferenceSystem() : CRS.decode(sourceProjection); CoordinateReferenceSystem dest = CRS.decode("EPSG:4326", true); @@ -61,6 +68,26 @@ public class ShapefileReader extends SimpleReader { if (transformToLatLon.isIdentity()) { transformToLatLon = null; } + + Filter filter = Filter.INCLUDE; + + Envelope env = bounds.latLon(); + if (!bounds.isWorld()) { + var ff = CommonFactoryFinder.getFilterFactory(GeoTools.getDefaultHints()); + var schema = source.getSchema(); + + String geometryPropertyName = schema.getGeometryDescriptor().getLocalName(); + + var bbox = new ReferencedEnvelope(env.getMinX(), env.getMaxX(), env.getMinY(), env.getMaxY(), dest); + try { + var bbox2 = bbox.transform(schema.getGeometryDescriptor().getCoordinateReferenceSystem(), true); + filter = ff.bbox(ff.property(geometryPropertyName), bbox2); + } catch (TransformException e) { + // just use include filter + } + } + + inputSource = source.getFeatures(filter); attributeNames = new String[inputSource.getSchema().getAttributeCount()]; for (int i = 0; i < attributeNames.length; i++) { attributeNames[i] = inputSource.getSchema().getDescriptor(i).getLocalName(); @@ -105,7 +132,7 @@ public class ShapefileReader extends SimpleReader { SourceFeatureProcessor.processFiles( sourceName, sourcePaths, - path -> new ShapefileReader(sourceProjection, sourceName, path), + path -> new ShapefileReader(sourceProjection, sourceName, path, config.bounds()), writer, config, profile, stats ); } @@ -137,7 +164,7 @@ public class ShapefileReader extends SimpleReader { latLonGeometry = JTS.transform(source, transformToLatLon); } if (latLonGeometry != null) { - SimpleFeature geom = SimpleFeature.create(latLonGeometry, new HashMap<>(attributeNames.length), + SimpleFeature geom = SimpleFeature.create(latLonGeometry, HashMap.newHashMap(attributeNames.length), sourceName, layer, ++id); for (int i = 1; i < attributeNames.length; i++) { geom.setTag(attributeNames[i], feature.getAttribute(i)); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/SimpleFeature.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/SimpleFeature.java index 5f117267..675a8549 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/SimpleFeature.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/SimpleFeature.java @@ -1,8 +1,10 @@ package com.onthegomap.planetiler.reader; import com.onthegomap.planetiler.geo.GeoUtils; +import com.onthegomap.planetiler.reader.osm.OsmElement; import com.onthegomap.planetiler.reader.osm.OsmReader; import com.onthegomap.planetiler.reader.osm.OsmRelationInfo; +import com.onthegomap.planetiler.reader.osm.OsmSourceFeature; import java.util.List; import java.util.Map; import java.util.Objects; @@ -76,29 +78,87 @@ public class SimpleFeature extends SourceFeature { return new SimpleFeature(latLonGeometry, null, tags, null, null, idGenerator.incrementAndGet(), null); } + private static class SimpleOsmFeature extends SimpleFeature implements OsmSourceFeature { + + private final String area; + private final OsmElement.Info info; + + private SimpleOsmFeature(Geometry latLonGeometry, Geometry worldGeometry, Map tags, String source, + String sourceLayer, long id, List> relations, OsmElement.Info info) { + super(latLonGeometry, worldGeometry, tags, source, sourceLayer, id, relations); + this.area = (String) tags.get("area"); + this.info = info; + } + + @Override + public boolean canBePolygon() { + return latLonGeometry() instanceof Polygonal || (latLonGeometry() instanceof LineString line && + OsmReader.canBePolygon(line.isClosed(), area, latLonGeometry().getNumPoints())); + } + + @Override + public boolean canBeLine() { + return latLonGeometry() instanceof MultiLineString || (latLonGeometry() instanceof LineString line && + OsmReader.canBeLine(line.isClosed(), area, latLonGeometry().getNumPoints())); + } + + @Override + protected Geometry computePolygon() { + var geom = worldGeometry(); + return geom instanceof LineString line ? GeoUtils.JTS_FACTORY.createPolygon(line.getCoordinates()) : geom; + } + + + @Override + public OsmElement originalElement() { + return new OsmElement() { + @Override + public long id() { + return SimpleOsmFeature.this.id(); + } + + @Override + public Info info() { + return info; + } + + @Override + public int cost() { + return 1; + } + + @Override + public Map tags() { + return tags(); + } + }; + } + + @Override + public boolean equals(Object o) { + return this == o || (o instanceof SimpleOsmFeature other && super.equals(other) && + Objects.equals(area, other.area) && Objects.equals(info, other.info)); + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + (area != null ? area.hashCode() : 0); + result = 31 * result + (info != null ? info.hashCode() : 0); + return result; + } + } + /** Returns a new feature with OSM relation info. Useful for setting up inputs for OSM unit tests. */ public static SimpleFeature createFakeOsmFeature(Geometry latLonGeometry, Map tags, String source, String sourceLayer, long id, List> relations) { - String area = (String) tags.get("area"); - return new SimpleFeature(latLonGeometry, null, tags, source, sourceLayer, id, relations) { - @Override - public boolean canBePolygon() { - return latLonGeometry instanceof Polygonal || (latLonGeometry instanceof LineString line && - OsmReader.canBePolygon(line.isClosed(), area, latLonGeometry.getNumPoints())); - } + return createFakeOsmFeature(latLonGeometry, tags, source, sourceLayer, id, relations, null); + } - @Override - public boolean canBeLine() { - return latLonGeometry instanceof MultiLineString || (latLonGeometry instanceof LineString line && - OsmReader.canBeLine(line.isClosed(), area, latLonGeometry.getNumPoints())); - } - - @Override - protected Geometry computePolygon() { - var geom = worldGeometry(); - return geom instanceof LineString line ? GeoUtils.JTS_FACTORY.createPolygon(line.getCoordinates()) : geom; - } - }; + /** Returns a new feature with OSM relation info and metadata. Useful for setting up inputs for OSM unit tests. */ + public static SimpleFeature createFakeOsmFeature(Geometry latLonGeometry, Map tags, String source, + String sourceLayer, long id, List> relations, OsmElement.Info info) { + return new SimpleOsmFeature(latLonGeometry, null, tags, source, sourceLayer, id, relations, info); } @Override diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/SourceFeature.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/SourceFeature.java index de73f9b0..5634207b 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/SourceFeature.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/SourceFeature.java @@ -35,13 +35,15 @@ public abstract class SourceFeature implements WithTags, WithGeometryType { private final long id; private Geometry centroid = null; private Geometry pointOnSurface = null; - private Geometry innermostPoint = null; private Geometry centroidIfConvex = null; + private double innermostPointTolerance = Double.NaN; + private Geometry innermostPoint = null; private Geometry linearGeometry = null; private Geometry polygonGeometry = null; private Geometry validPolygon = null; private double area = Double.NaN; private double length = Double.NaN; + private double size = Double.NaN; // slight optimization: replace default implementation with direct access to the tags // map to get slightly improved performance when matching elements against expressions @@ -127,18 +129,30 @@ public abstract class SourceFeature implements WithTags, WithGeometryType { worldGeometry().getInteriorPoint()); } - public final Geometry innermostPoint() throws GeometryException { - if (innermostPoint == null) { - Geometry polygon = polygon(); - innermostPoint = MaximumInscribedCircle.getCenter(polygon(), Math.sqrt(polygon.getArea() / 100d)); + /** + * Returns {@link MaximumInscribedCircle#getCenter()} of this geometry in world web mercator coordinates. + * + * @param tolerance precision for calculating maximum inscribed circle. 0.01 means 1% of the square root of the area. + * Smaller values for a more precise tolerance become very expensive to compute. Values between + * 0.05-0.1 are a good compromise of performance vs. precision. + */ + public final Geometry innermostPoint(double tolerance) throws GeometryException { + if (canBePolygon()) { + // cache as long as the tolerance hasn't changed + if (tolerance != innermostPointTolerance || innermostPoint == null) { + innermostPoint = MaximumInscribedCircle.getCenter(polygon(), Math.sqrt(area()) * tolerance); + innermostPointTolerance = tolerance; + } + return innermostPoint; + } else { + return pointOnSurface(); } - return innermostPoint; } private Geometry computeCentroidIfConvex() throws GeometryException { if (!canBePolygon()) { return centroid(); - } else if (polygon()instanceof Polygon poly && + } else if (polygon() instanceof Polygon poly && poly.getNumInteriorRing() == 0 && GeoUtils.isConvex(poly.getExteriorRing())) { return centroid(); @@ -255,6 +269,14 @@ public abstract class SourceFeature implements WithTags, WithGeometryType { (isPoint() || canBePolygon() || canBeLine()) ? worldGeometry().getLength() : 0) : length; } + /** + * Returns and caches sqrt of {@link #area()} if polygon or {@link #length()} if a line string. + */ + public double size() throws GeometryException { + return Double.isNaN(size) ? (size = canBePolygon() ? Math.sqrt(Math.abs(area())) : canBeLine() ? length() : 0) : + size; + } + /** Returns the ID of the source that this feature came from. */ public String getSource() { return source; @@ -302,4 +324,5 @@ public abstract class SourceFeature implements WithTags, WithGeometryType { public boolean hasRelationInfo() { return relationInfos != null && !relationInfos.isEmpty(); } + } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmMultipolygon.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmMultipolygon.java index 4e18f594..31ca60a3 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmMultipolygon.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmMultipolygon.java @@ -232,7 +232,7 @@ public class OsmMultipolygon { if (numPolygons == 0) { return shells; } - shells.add(polygons.get(0)); + shells.add(polygons.getFirst()); if (numPolygons == 1) { return shells; } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/PbfDecoder.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/PbfDecoder.java index dd4b6976..dc6c9255 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/PbfDecoder.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/PbfDecoder.java @@ -133,7 +133,7 @@ public class PbfDecoder implements Iterable { private Map buildTags(int num, IntUnaryOperator key, IntUnaryOperator value) { if (num > 0) { - Map tags = new HashMap<>(num); + Map tags = HashMap.newHashMap(num); for (int i = 0; i < num; i++) { String k = fieldDecoder.decodeString(key.applyAsInt(i)); String v = fieldDecoder.decodeString(value.applyAsInt(i)); @@ -366,7 +366,7 @@ public class PbfDecoder implements Iterable { if (tags == null) { // divide by 2 as key&value, multiply by 2 because of the better approximation - tags = new HashMap<>(Math.max(3, 2 * (nodes.getKeysValsCount() / 2) / nodes.getKeysValsCount())); + tags = HashMap.newHashMap(Math.max(3, 2 * (nodes.getKeysValsCount() / 2) / nodes.getKeysValsCount())); } tags.put(fieldDecoder.decodeString(keyIndex), fieldDecoder.decodeString(valueIndex)); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/render/FeatureRenderer.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/render/FeatureRenderer.java index 98df7cf6..545dcad9 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/render/FeatureRenderer.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/render/FeatureRenderer.java @@ -97,6 +97,10 @@ public class FeatureRenderer implements Consumer, Clos coords[i] = origCoords[i].copy(); } for (int zoom = feature.getMaxZoom(); zoom >= feature.getMinZoom(); zoom--) { + double minSize = feature.getMinPixelSizeAtZoom(zoom); + if (minSize > 0 && feature.getSourceFeaturePixelSizeAtZoom(zoom) < minSize) { + continue; + } Map attrs = feature.getAttrsAtZoom(zoom); double buffer = feature.getBufferPixelsAtZoom(zoom) / 256; int tilesAtZoom = 1 << zoom; @@ -207,7 +211,7 @@ public class FeatureRenderer implements Consumer, Clos } Map attrs = feature.getAttrsAtZoom(sliced.zoomLevel()); if (numPointsAttr != null) { - // if profile wants the original number of points that the simplified but untiled geometry started with + // if profile wants the original number off points that the simplified but untiled geometry started with attrs = new HashMap<>(attrs); attrs.put(numPointsAttr, geom.getNumPoints()); } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/render/GeometryCoordinateSequences.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/render/GeometryCoordinateSequences.java index 26112918..60b117f8 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/render/GeometryCoordinateSequences.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/render/GeometryCoordinateSequences.java @@ -122,7 +122,7 @@ class GeometryCoordinateSequences { static Geometry reassemblePolygons(List> groups) throws GeometryException { int numGeoms = groups.size(); if (numGeoms == 1) { - return reassemblePolygon(groups.get(0)); + return reassemblePolygon(groups.getFirst()); } else { Polygon[] polygons = new Polygon[numGeoms]; for (int i = 0; i < numGeoms; i++) { @@ -135,7 +135,7 @@ class GeometryCoordinateSequences { /** Returns a {@link Polygon} built from all outer/inner rings in {@code group}, reversing all inner rings. */ private static Polygon reassemblePolygon(List group) throws GeometryException { try { - LinearRing first = GeoUtils.JTS_FACTORY.createLinearRing(group.get(0)); + LinearRing first = GeoUtils.JTS_FACTORY.createLinearRing(group.getFirst()); LinearRing[] rest = new LinearRing[group.size() - 1]; for (int j = 1; j < group.size(); j++) { CoordinateSequence seq = group.get(j); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/render/TiledGeometry.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/render/TiledGeometry.java index d5b9469a..03ffbfc9 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/render/TiledGeometry.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/render/TiledGeometry.java @@ -258,7 +258,7 @@ public class TiledGeometry { TileCoord tile = TileCoord.ofXYZ(wrappedX, y, z); double tileY = worldY - y; tileContents.computeIfAbsent(tile, t -> List.of(new ArrayList<>())) - .get(0) + .getFirst() .add(GeoUtils.coordinateSequence(tileX * 256, tileY * 256)); } } @@ -384,7 +384,7 @@ public class TiledGeometry { for (var entry : inProgressShapes.entrySet()) { TileCoord tileID = entry.getKey(); List inSeqs = entry.getValue(); - if (area && inSeqs.get(0).size() < 4) { + if (area && inSeqs.getFirst().size() < 4) { // not enough points in outer polygon, ignore continue; } @@ -573,20 +573,20 @@ public class TiledGeometry { } /* A tile is inside a filled region when there is an odd number of vertical edges to the left and right - + for example a simple shape: --------- out | in | out (0/2) | (1/1) | (2/0) --------- - + or a more complex shape --------- --------- out | in | out | in | (0/4) | (1/3) | (2/2) | (3/1) | | --------- | ------------------------- - + So we keep track of this number by xor'ing the left and right fills repeatedly, then and'ing them together at the end. */ diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/ProcessInfo.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/ProcessInfo.java index 4dda00a3..29736ef4 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/ProcessInfo.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/ProcessInfo.java @@ -38,7 +38,7 @@ public class ProcessInfo { for (GarbageCollectorMXBean garbageCollectorMXBean : ManagementFactory.getGarbageCollectorMXBeans()) { if (garbageCollectorMXBean instanceof NotificationEmitter emitter) { emitter.addNotificationListener((notification, handback) -> { - if (notification.getUserData()instanceof CompositeData compositeData) { + if (notification.getUserData() instanceof CompositeData compositeData) { var info = GarbageCollectionNotificationInfo.from(compositeData); GcInfo gcInfo = info.getGcInfo(); postGcMemoryUsage.set(gcInfo.getMemoryUsageAfterGc().entrySet().stream() @@ -142,7 +142,7 @@ public class ProcessInfo { * Returns the total amount of memory available on the system if available. */ public static OptionalLong getSystemMemoryBytes() { - if (ManagementFactory.getOperatingSystemMXBean()instanceof com.sun.management.OperatingSystemMXBean osBean) { + if (ManagementFactory.getOperatingSystemMXBean() instanceof com.sun.management.OperatingSystemMXBean osBean) { return OptionalLong.of(osBean.getTotalMemorySize()); } else { return OptionalLong.empty(); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/ProcessTime.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/ProcessTime.java index 16645bbf..7e8cb0b6 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/ProcessTime.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/ProcessTime.java @@ -10,15 +10,12 @@ import javax.annotation.concurrent.Immutable; * A utility for measuring the wall and CPU time that this JVM consumes between snapshots. *

* For example: - * - * 

- * {@code
+ * {@snippet :
  * var start = ProcessTime.now();
  * // do expensive work...
- * var end - ProcessTime.now();
+ * var end = ProcessTime.now();
  * LOGGER.log("Expensive work took " + end.minus(start));
  * }
- *
*/ @Immutable public record ProcessTime(Duration wall, Optional cpu, Duration gc) { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/ProgressLoggers.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/ProgressLoggers.java index 3d7535ce..44dce911 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/ProgressLoggers.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/ProgressLoggers.java @@ -275,7 +275,7 @@ public class ProgressLoggers { /** Adds the CPU utilization of every thread starting with {@code prefix} since the last log to output. */ public ProgressLoggers addThreadPoolStats(String name, String prefix) { - boolean first = loggers.isEmpty() || !(loggers.get(loggers.size() - 1) instanceof WorkerPipelineLogger); + boolean first = loggers.isEmpty() || !(loggers.getLast() instanceof WorkerPipelineLogger); try { Map lastThreads = ProcessInfo.getThreadStats(); AtomicLong lastTime = new AtomicLong(System.nanoTime()); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchive.java index 40f389ce..30d683ee 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchive.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchive.java @@ -76,11 +76,11 @@ public final class WriteableJsonStreamArchive extends WriteableStreamArchive { } @Override - public void initialize(TileArchiveMetadata metadata) { + public void initialize() { if (writeTilesOnly) { return; } - writeEntryFlush(new InitializationEntry(metadata)); + writeEntryFlush(new InitializationEntry()); } @Override @@ -204,22 +204,19 @@ public final class WriteableJsonStreamArchive extends WriteableStreamArchive { } } - record InitializationEntry(TileArchiveMetadata metadata) implements Entry {} + record InitializationEntry() implements Entry {} record FinishEntry(TileArchiveMetadata metadata) implements Entry {} - private interface TileArchiveMetadataMixin { + private record TileArchiveMetadataMixin( - @JsonIgnore(false) - Envelope bounds(); + @JsonIgnore(false) Envelope bounds, - @JsonIgnore(false) - CoordinateXY center(); + @JsonIgnore(false) CoordinateXY center, - @JsonIgnore(false) - List vectorLayers(); - } + @JsonIgnore(false) List vectorLayers + ) {} @JsonIncludeProperties({"minX", "maxX", "minY", "maxY"}) private abstract static class EnvelopeMixin { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchive.java index ebe9f965..2c193a8c 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchive.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchive.java @@ -50,14 +50,8 @@ public final class WriteableProtoStreamArchive extends WriteableStreamArchive { } @Override - public void initialize(TileArchiveMetadata metadata) { - writeEntry( - StreamArchiveProto.Entry.newBuilder() - .setInitialization( - StreamArchiveProto.InitializationEntry.newBuilder().setMetadata(toExportData(metadata)).build() - ) - .build() - ); + public void initialize() { + writeEntry(StreamArchiveProto.Entry.newBuilder().build()); } @Override diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/AwsOsm.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/AwsOsm.java index 95f1665e..23168509 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/AwsOsm.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/AwsOsm.java @@ -106,7 +106,7 @@ public class AwsOsm { } else if (results.size() > 1) { throw new IllegalArgumentException("Found multiple AWS osm download URLs for " + searchQuery + ": " + results); } - return results.get(0); + return results.getFirst(); } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Downloader.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Downloader.java index a538f89b..3139f4da 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Downloader.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Downloader.java @@ -5,12 +5,12 @@ import static java.nio.file.StandardOpenOption.WRITE; import com.google.common.util.concurrent.RateLimiter; import com.onthegomap.planetiler.config.PlanetilerConfig; +import com.onthegomap.planetiler.stats.Counter; import com.onthegomap.planetiler.stats.ProgressLoggers; -import com.onthegomap.planetiler.stats.Stats; -import com.onthegomap.planetiler.worker.WorkerPipeline; +import com.onthegomap.planetiler.worker.RunnableThatThrows; +import com.onthegomap.planetiler.worker.Worker; import java.io.IOException; import java.io.InputStream; -import java.io.UncheckedIOException; import java.net.URI; import java.net.URLConnection; import java.net.http.HttpClient; @@ -18,9 +18,7 @@ import java.net.http.HttpHeaders; import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.nio.ByteBuffer; -import java.nio.channels.Channels; import java.nio.channels.FileChannel; -import java.nio.channels.ReadableByteChannel; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -30,9 +28,9 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicLong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,15 +41,12 @@ import org.slf4j.LoggerFactory; * changes. *

* For example: - * - * 

- * {@code
+ * {@snippet :
  * Downloader.create(PlanetilerConfig.defaults())
  *   .add("natural_earth", "http://url/of/natural_earth.zip", Path.of("natural_earth.zip"))
  *   .add("osm", "http://url/of/file.osm.pbf", Path.of("file.osm.pbf"))
  *   .run();
  * }
- *
*

* As a shortcut to find the URL of a file to download from the Geofabrik * download site, you can use "geofabrik:extract name" (i.e. "geofabrik:monaco" or "geofabrik:australia") to look up @@ -69,32 +64,26 @@ public class Downloader { private static final Logger LOGGER = LoggerFactory.getLogger(Downloader.class); private final PlanetilerConfig config; private final List toDownloadList = new ArrayList<>(); - private final HttpClient client = HttpClient.newBuilder() - // explicitly follow redirects to capture final redirect url - .followRedirects(HttpClient.Redirect.NEVER).build(); + private final HttpClient client; private final ExecutorService executor; - private final Stats stats; private final long chunkSizeBytes; private final ResourceUsage diskSpaceCheck = new ResourceUsage("download"); private final RateLimiter rateLimiter; - Downloader(PlanetilerConfig config, Stats stats, long chunkSizeBytes) { + Downloader(PlanetilerConfig config, long chunkSizeBytes) { this.rateLimiter = config.downloadMaxBandwidth() == 0 ? null : RateLimiter.create(config.downloadMaxBandwidth()); this.chunkSizeBytes = chunkSizeBytes; this.config = config; - this.stats = stats; - this.executor = Executors.newSingleThreadExecutor(runnable -> { - Thread thread = new Thread(() -> { - LogUtil.setStage("download"); - runnable.run(); - }); - thread.setDaemon(true); - return thread; - }); + this.executor = Executors.newVirtualThreadPerTaskExecutor(); + this.client = HttpClient.newBuilder() + // explicitly follow redirects to capture final redirect url + .followRedirects(HttpClient.Redirect.NEVER) + .executor(executor) + .build(); } - public static Downloader create(PlanetilerConfig config, Stats stats) { - return new Downloader(config, stats, config.downloadChunkSizeMB() * 1_000_000L); + public static Downloader create(PlanetilerConfig config) { + return new Downloader(config, config.downloadChunkSizeMB() * 1_000_000L); } public static URLConnection getUrlConnection(String urlString, PlanetilerConfig config) throws IOException { @@ -191,145 +180,117 @@ public class Downloader { } CompletableFuture downloadIfNecessary(ResourceToDownload resourceToDownload) { - long existingSize = FileUtils.size(resourceToDownload.output); - - return httpHeadFollowRedirects(resourceToDownload.url, 0) - .whenComplete((metadata, err) -> { - if (metadata != null) { - resourceToDownload.metadata.complete(metadata); - } else { - resourceToDownload.metadata.completeExceptionally(err); - } - }) - .thenComposeAsync(metadata -> { - if (metadata.size == existingSize) { - LOGGER.info("Skipping {}: {} already up-to-date", resourceToDownload.id, resourceToDownload.output); - return CompletableFuture.completedFuture(null); - } else { - String redirectInfo = metadata.canonicalUrl.equals(resourceToDownload.url) ? "" : - " (redirected to " + metadata.canonicalUrl + ")"; - LOGGER.info("Downloading {}{} to {}", resourceToDownload.url, redirectInfo, resourceToDownload.output); - FileUtils.delete(resourceToDownload.output); - FileUtils.createParentDirectories(resourceToDownload.output); - Path tmpPath = resourceToDownload.tmpPath(); - FileUtils.delete(tmpPath); - FileUtils.deleteOnExit(tmpPath); - diskSpaceCheck.addDisk(tmpPath, metadata.size, resourceToDownload.id); - diskSpaceCheck.checkAgainstLimits(config.force(), false); - return httpDownload(resourceToDownload, tmpPath) - .thenCompose(result -> { - try { - Files.move(tmpPath, resourceToDownload.output); - return CompletableFuture.completedFuture(null); - } catch (IOException e) { - return CompletableFuture.failedFuture(e); - } - }) - .whenCompleteAsync((result, error) -> { - if (error != null) { - LOGGER.error("Error downloading {} to {}", resourceToDownload.url, resourceToDownload.output, error); - } else { - LOGGER.info("Finished downloading {} to {}", resourceToDownload.url, resourceToDownload.output); - } - FileUtils.delete(tmpPath); - }, executor); - } - }, executor); + return CompletableFuture.runAsync(RunnableThatThrows.wrap(() -> { + LogUtil.setStage("download", resourceToDownload.id); + long existingSize = FileUtils.size(resourceToDownload.output); + var metadata = httpHeadFollowRedirects(resourceToDownload.url, 0); + Path tmpPath = resourceToDownload.tmpPath(); + resourceToDownload.metadata.complete(metadata); + if (metadata.size == existingSize) { + LOGGER.info("Skipping {}: {} already up-to-date", resourceToDownload.id, resourceToDownload.output); + return; + } + try { + String redirectInfo = metadata.canonicalUrl.equals(resourceToDownload.url) ? "" : + " (redirected to " + metadata.canonicalUrl + ")"; + LOGGER.info("Downloading {}{} to {}", resourceToDownload.url, redirectInfo, resourceToDownload.output); + FileUtils.delete(resourceToDownload.output); + FileUtils.createParentDirectories(resourceToDownload.output); + FileUtils.delete(tmpPath); + FileUtils.deleteOnExit(tmpPath); + diskSpaceCheck.addDisk(tmpPath, metadata.size, resourceToDownload.id); + diskSpaceCheck.checkAgainstLimits(config.force(), false); + httpDownload(resourceToDownload, tmpPath); + Files.move(tmpPath, resourceToDownload.output); + LOGGER.info("Finished downloading {} to {}", resourceToDownload.url, resourceToDownload.output); + } catch (Exception e) { // NOSONAR + LOGGER.error("Error downloading {} to {}", resourceToDownload.url, resourceToDownload.output, e); + throw e; + } finally { + FileUtils.delete(tmpPath); + } + }), executor); } - private CompletableFuture httpHeadFollowRedirects(String url, int redirects) { + private ResourceMetadata httpHeadFollowRedirects(String url, int redirects) throws IOException, InterruptedException { if (redirects > MAX_REDIRECTS) { throw new IllegalStateException("Exceeded " + redirects + " redirects for " + url); } - return httpHead(url).thenComposeAsync(response -> response.redirect.isPresent() ? - httpHeadFollowRedirects(response.redirect.get(), redirects + 1) : CompletableFuture.completedFuture(response)); + var response = httpHead(url); + return response.redirect.isPresent() ? httpHeadFollowRedirects(response.redirect.get(), redirects + 1) : response; } - CompletableFuture httpHead(String url) { - return client - .sendAsync(newHttpRequest(url).method("HEAD", HttpRequest.BodyPublishers.noBody()).build(), - responseInfo -> { - int status = responseInfo.statusCode(); - Optional location = Optional.empty(); - long contentLength = 0; - HttpHeaders headers = responseInfo.headers(); - if (status >= 300 && status < 400) { - location = responseInfo.headers().firstValue(LOCATION); - if (location.isEmpty()) { - throw new IllegalStateException("Received " + status + " but no location header from " + url); - } - } else if (responseInfo.statusCode() != 200) { - throw new IllegalStateException("Bad response: " + responseInfo.statusCode()); - } else { - contentLength = headers.firstValueAsLong(CONTENT_LENGTH).orElseThrow(); + ResourceMetadata httpHead(String url) throws IOException, InterruptedException { + return client.send(newHttpRequest(url).HEAD().build(), + responseInfo -> { + int status = responseInfo.statusCode(); + Optional location = Optional.empty(); + long contentLength = 0; + HttpHeaders headers = responseInfo.headers(); + if (status >= 300 && status < 400) { + location = responseInfo.headers().firstValue(LOCATION); + if (location.isEmpty()) { + throw new IllegalStateException("Received " + status + " but no location header from " + url); } - boolean supportsRangeRequest = headers.allValues(ACCEPT_RANGES).contains("bytes"); - ResourceMetadata metadata = new ResourceMetadata(location, url, contentLength, supportsRangeRequest); - return HttpResponse.BodyHandlers.replacing(metadata).apply(responseInfo); - }) - .thenApply(HttpResponse::body); - } - - private CompletableFuture httpDownload(ResourceToDownload resource, Path tmpPath) { - /* - * Alternative using async HTTP client: - * - * return client.sendAsync(newHttpRequest(url).GET().build(), responseInfo -> { - * assertOK(responseInfo); - * return HttpResponse.BodyHandlers.ofFile(path).apply(responseInfo); - * - * But it is slower on large files - */ - return resource.metadata.thenCompose(metadata -> { - String canonicalUrl = metadata.canonicalUrl; - record Range(long start, long end) { - - long size() { - return end - start; + } else if (responseInfo.statusCode() != 200) { + throw new IllegalStateException("Bad response: " + responseInfo.statusCode()); + } else { + contentLength = headers.firstValueAsLong(CONTENT_LENGTH).orElseThrow(); } - } - List chunks = new ArrayList<>(); - boolean ranges = metadata.acceptRange && config.downloadThreads() > 1; - long chunkSize = ranges ? chunkSizeBytes : metadata.size; - for (long start = 0; start < metadata.size; start += chunkSize) { - long end = Math.min(start + chunkSize, metadata.size); - chunks.add(new Range(start, end)); - } - // create an empty file - try { - Files.createFile(tmpPath); - } catch (IOException e) { - return CompletableFuture.failedFuture(new IOException("Failed to create " + resource.output, e)); - } - return WorkerPipeline.start("download-" + resource.id, stats) - .readFromTiny("chunks", chunks) - .sinkToConsumer("chunk-downloader", Math.min(config.downloadThreads(), chunks.size()), range -> { - try (var fileChannel = FileChannel.open(tmpPath, WRITE)) { - while (range.size() > 0) { - try ( - var inputStream = (ranges || range.start > 0) ? openStreamRange(canonicalUrl, range.start, range.end) : - openStream(canonicalUrl); - var input = new ProgressChannel(Channels.newChannel(inputStream), resource.progress, rateLimiter) - ) { - // ensure this file has been allocated up to the start of this block - fileChannel.write(ByteBuffer.allocate(1), range.start); - fileChannel.position(range.start); - long transferred = fileChannel.transferFrom(input, range.start, range.size()); - if (transferred == 0) { - throw new IOException("Transferred 0 bytes but " + range.size() + " expected: " + canonicalUrl); - } else if (transferred != range.size() && !metadata.acceptRange) { - throw new IOException( - "Transferred " + transferred + " bytes but " + range.size() + " expected: " + canonicalUrl + - " and server does not support range requests"); - } - range = new Range(range.start + transferred, range.end); - } - } - } catch (IOException e) { - throw new UncheckedIOException(e); + boolean supportsRangeRequest = headers.allValues(ACCEPT_RANGES).contains("bytes"); + ResourceMetadata metadata = new ResourceMetadata(location, url, contentLength, supportsRangeRequest); + return HttpResponse.BodyHandlers.replacing(metadata).apply(responseInfo); + }).body(); + } + + private void httpDownload(ResourceToDownload resource, Path tmpPath) + throws ExecutionException, InterruptedException { + var metadata = resource.metadata().get(); + String canonicalUrl = metadata.canonicalUrl(); + record Range(long start, long end) {} + List chunks = new ArrayList<>(); + boolean ranges = metadata.acceptRange && config.downloadThreads() > 1; + long chunkSize = ranges ? chunkSizeBytes : metadata.size; + for (long start = 0; start < metadata.size; start += chunkSize) { + long end = Math.min(start + chunkSize, metadata.size); + chunks.add(new Range(start, end)); + } + FileUtils.setLength(tmpPath, metadata.size); + Semaphore perFileLimiter = new Semaphore(config.downloadThreads()); + Worker.joinFutures(chunks.stream().map(range -> CompletableFuture.runAsync(RunnableThatThrows.wrap(() -> { + LogUtil.setStage("download", resource.id); + perFileLimiter.acquire(); + var counter = resource.progress.counterForThread(); + try ( + var fc = FileChannel.open(tmpPath, WRITE); + var inputStream = (ranges || range.start > 0) ? + openStreamRange(canonicalUrl, range.start, range.end) : + openStream(canonicalUrl); + ) { + long offset = range.start; + byte[] buffer = new byte[16384]; + int read; + while (offset < range.end && (read = inputStream.read(buffer, 0, 16384)) >= 0) { + counter.incBy(read); + if (rateLimiter != null) { + rateLimiter.acquire(read); } - }).done(); - }); + int position = 0; + int remaining = read; + while (remaining > 0) { + int written = fc.write(ByteBuffer.wrap(buffer, position, remaining), offset); + if (written <= 0) { + throw new IOException("Failed to write to " + tmpPath); + } + position += written; + remaining -= written; + offset += written; + } + } + } finally { + perFileLimiter.release(); + } + }), executor)).toArray(CompletableFuture[]::new)).get(); } private HttpRequest.Builder newHttpRequest(String url) { @@ -341,11 +302,12 @@ public class Downloader { record ResourceMetadata(Optional redirect, String canonicalUrl, long size, boolean acceptRange) {} record ResourceToDownload( - String id, String url, Path output, CompletableFuture metadata, AtomicLong progress + String id, String url, Path output, CompletableFuture metadata, + Counter.MultiThreadCounter progress ) { ResourceToDownload(String id, String url, Path output) { - this(id, url, output, new CompletableFuture<>(), new AtomicLong(0)); + this(id, url, output, new CompletableFuture<>(), Counter.newMultiThreadCounter()); } public Path tmpPath() { @@ -356,33 +318,4 @@ public class Downloader { return progress.get(); } } - - /** - * Wrapper for a {@link ReadableByteChannel} that captures progress information. - */ - private record ProgressChannel(ReadableByteChannel inner, AtomicLong progress, RateLimiter rateLimiter) - implements ReadableByteChannel { - - @Override - public int read(ByteBuffer dst) throws IOException { - int n = inner.read(dst); - if (n > 0) { - if (rateLimiter != null) { - rateLimiter.acquire(n); - } - progress.addAndGet(n); - } - return n; - } - - @Override - public boolean isOpen() { - return inner.isOpen(); - } - - @Override - public void close() throws IOException { - inner.close(); - } - } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/FileUtils.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/FileUtils.java index cbd60373..1cbbe138 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/FileUtils.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/FileUtils.java @@ -1,8 +1,13 @@ package com.onthegomap.planetiler.util; +import static java.nio.file.StandardOpenOption.CREATE; +import static java.nio.file.StandardOpenOption.WRITE; + import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import java.nio.file.ClosedFileSystemException; import java.nio.file.FileStore; import java.nio.file.FileSystem; @@ -263,7 +268,7 @@ public class FileUtils { * @throws UncheckedIOException if an IO exception occurs */ public static void safeCopy(InputStream inputStream, Path destPath) { - try (var outputStream = Files.newOutputStream(destPath, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) { + try (var outputStream = Files.newOutputStream(destPath, StandardOpenOption.CREATE, WRITE)) { int totalSize = 0; int nBytes; @@ -310,7 +315,7 @@ public class FileUtils { try ( var out = Files.newOutputStream(destination, StandardOpenOption.CREATE_NEW, - StandardOpenOption.WRITE) + WRITE) ) { totalEntryArchive++; while ((nBytes = zip.read(buffer)) > 0) { @@ -366,4 +371,16 @@ public class FileUtils { return true; } } + + /** Expands the file at {@code path} to {@code size} bytes. */ + public static void setLength(Path path, long size) { + try (var fc = FileChannel.open(path, CREATE, WRITE)) { + int written = fc.write(ByteBuffer.allocate(1), size - 1); + if (written != 1) { + throw new IOException("Unable to expand " + path + " to " + size); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Geofabrik.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Geofabrik.java index fe7bf2c0..07e514ad 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Geofabrik.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Geofabrik.java @@ -106,7 +106,7 @@ public class Geofabrik { "Multiple " + name + " for '" + searchQuery + "': " + values.stream().map(d -> d.id).collect( Collectors.joining(", "))); } else if (values.size() == 1) { - return values.get(0).urls.get("pbf"); + return values.getFirst().urls.get("pbf"); } else { return null; } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/LayerAttrStats.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/LayerAttrStats.java index 9685f0fe..11b10340 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/LayerAttrStats.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/LayerAttrStats.java @@ -3,7 +3,6 @@ package com.onthegomap.planetiler.util; import com.fasterxml.jackson.annotation.JsonProperty; import com.onthegomap.planetiler.archive.WriteableTileArchive; import com.onthegomap.planetiler.mbtiles.Mbtiles; -import com.onthegomap.planetiler.render.RenderedFeature; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -11,7 +10,6 @@ import java.util.Optional; import java.util.OptionalInt; import java.util.TreeMap; import java.util.concurrent.CopyOnWriteArrayList; -import java.util.function.Consumer; import javax.annotation.concurrent.NotThreadSafe; import javax.annotation.concurrent.ThreadSafe; @@ -27,7 +25,7 @@ import javax.annotation.concurrent.ThreadSafe; * @see MBtiles spec */ @ThreadSafe -public class LayerAttrStats implements Consumer { +public class LayerAttrStats { /* * This utility is called for billions of features by multiple threads when processing the planet which can make * access to shared data structures a bottleneck. So give each thread an individual ThreadLocalLayerStatsHandler to @@ -63,6 +61,11 @@ public class LayerAttrStats implements Consumer { .toList(); } + /** Shortcut for tests */ + void accept(String layer, int zoom, String key, Object value) { + handlerForThread().forZoom(zoom).forLayer(layer).accept(key, value); + } + public enum FieldType { @JsonProperty("Number") NUMBER, @@ -114,7 +117,7 @@ public class LayerAttrStats implements Consumer { /** Accepts features from a single thread that will be combined across all threads in {@link #getTileStats()}. */ @NotThreadSafe - private class ThreadLocalHandler implements Consumer { + private class ThreadLocalHandler implements Updater { private final Map layers = new TreeMap<>(); @@ -123,42 +126,50 @@ public class LayerAttrStats implements Consumer { } @Override - public void accept(RenderedFeature feature) { - var vectorTileFeature = feature.vectorTileFeature(); - var stats = layers.computeIfAbsent(vectorTileFeature.layer(), StatsForLayer::new); - stats.expandZoomRangeToInclude(feature.tile().z()); - for (var entry : vectorTileFeature.attrs().entrySet()) { - String key = entry.getKey(); - Object value = entry.getValue(); - - FieldType fieldType = null; - if (value instanceof Number) { - fieldType = FieldType.NUMBER; - } else if (value instanceof Boolean) { - fieldType = FieldType.BOOLEAN; - } else if (value != null) { - fieldType = FieldType.STRING; - } - if (fieldType != null) { - // widen different types to string - stats.fields.merge(key, fieldType, FieldType::merge); - } - } + public Updater.ForZoom forZoom(int zoom) { + return layer -> { + var stats = layers.computeIfAbsent(layer, StatsForLayer::new); + stats.expandZoomRangeToInclude(zoom); + return (key, value) -> { + FieldType fieldType = null; + if (value instanceof Number) { + fieldType = FieldType.NUMBER; + } else if (value instanceof Boolean) { + fieldType = FieldType.BOOLEAN; + } else if (value != null) { + fieldType = FieldType.STRING; + } + if (fieldType != null) { + // widen different types to string + stats.fields.merge(key, fieldType, FieldType::merge); + } + }; + }; } } /** * Returns a handler optimized for accepting features from a single thread. - *

- * Use this instead of {@link #accept(RenderedFeature)} */ - public Consumer handlerForThread() { + public Updater handlerForThread() { return layerStats.get(); } - @Override - public void accept(RenderedFeature feature) { - handlerForThread().accept(feature); + public interface Updater { + + ForZoom forZoom(int zoom); + + interface ForZoom { + + ForZoom NOOP = layer -> (key, value) -> { + }; + + ForLayer forLayer(String layer); + + interface ForLayer { + void accept(String key, Object value); + } + } } private static class StatsForLayer { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/LogUtil.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/LogUtil.java index 67d76190..11669dd8 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/LogUtil.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/LogUtil.java @@ -14,7 +14,7 @@ public class LogUtil { /** Prepends {@code [stage]} to all subsequent logs from this thread. */ public static void setStage(String stage) { - MDC.put(STAGE_KEY, stage); + MDC.put(STAGE_KEY, "[%s] ".formatted(stage)); } /** Removes {@code [stage]} from subsequent logs from this thread. */ @@ -24,7 +24,8 @@ public class LogUtil { /** Returns the current {@code [stage]} value prepended to log for this thread. */ public static String getStage() { - return MDC.get(STAGE_KEY); + // strip out the "[stage] " wrapper + return MDC.get(STAGE_KEY) instanceof String s ? s.substring(1, s.length() - 2) : null; } /** Prepends {@code [parent:child]} to all subsequent logs from this thread. */ diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/SortKey.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/SortKey.java index 3cb9a843..fb659485 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/SortKey.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/SortKey.java @@ -13,19 +13,15 @@ import com.onthegomap.planetiler.collection.FeatureGroup; * To sort by a field descending, specify its range from high to low. *

* For example this SQL ordering: - * - * 

- * {@code
+ *
+ * {@snippet lang = "sql" :
  * ORDER BY rank ASC,
  * population DESC,
  * length(name) ASC
  * }
- *
*

* would become: - * - * 

- * {@code
+ * {@snippet :
  * feature.setSortKey(
  *   SortKey
  *     .orderByInt(rank, MIN_RANK, MAX_RANK)
@@ -125,7 +121,7 @@ public class SortKey {
     }
     int levels = end + 1 - start;
     if (value < start || value > end) {
-      value = Math.max(start, Math.min(end, value));
+      value = Math.clamp(value, start, end);
     }
     return accumulate(value, start, levels);
   }
@@ -141,7 +137,7 @@ public class SortKey {
       return thenByDouble(start - value, end, start, levels);
     }
     if (value < start || value > end) {
-      value = Math.max(start, Math.min(end, value));
+      value = Math.clamp(value, start, end);
     }
 
     int intVal = doubleRangeToInt(value, start, end, levels);
@@ -160,7 +156,7 @@ public class SortKey {
     }
     assert start > 0 : "log thresholds must be > 0 got [" + start + ", " + end + "]";
     if (value < start || value > end) {
-      value = Math.max(start, Math.min(end, value));
+      value = Math.clamp(value, start, end);
     }
 
     int intVal = doubleRangeToInt(Math.log(value), Math.log(start), Math.log(end), levels);
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TopOsmTiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TopOsmTiles.java
index e66e8cbc..9f893efd 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TopOsmTiles.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TopOsmTiles.java
@@ -62,7 +62,7 @@ public class TopOsmTiles {
   TopOsmTiles(PlanetilerConfig config, Stats stats) {
     this.config = config;
     this.stats = stats;
-    downloader = Downloader.create(config, stats);
+    downloader = Downloader.create(config);
   }
 
   Reader fetch(LocalDate date) throws IOException {
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Translations.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Translations.java
index c06a0fce..7e3c60b0 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Translations.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Translations.java
@@ -128,7 +128,7 @@ public class Translations {
       Map result = new HashMap<>();
       for (var entry : tags.entrySet()) {
         String key = entry.getKey();
-        if (key.startsWith("name:") && entry.getValue()instanceof String stringVal) {
+        if (key.startsWith("name:") && entry.getValue() instanceof String stringVal) {
           result.put(key, stringVal);
         }
       }
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/worker/RunnableThatThrows.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/worker/RunnableThatThrows.java
index d2c0ea73..a3ed4ae9 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/worker/RunnableThatThrows.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/worker/RunnableThatThrows.java
@@ -18,4 +18,8 @@ public interface RunnableThatThrows {
       throwFatalException(e);
     }
   }
+
+  static Runnable wrap(RunnableThatThrows thrower) {
+    return thrower::runAndWrapException;
+  }
 }
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/worker/WorkerPipeline.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/worker/WorkerPipeline.java
index 9bdeede6..22abcacd 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/worker/WorkerPipeline.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/worker/WorkerPipeline.java
@@ -17,24 +17,21 @@ import java.util.function.Consumer;
  * A mini-framework for chaining sequential steps that run in dedicated threads with a queue between each.
  * 

  * For example:
- *
- * 
- * {@code
+ * {@snippet :
  * WorkerPipeline.start("name", stats)
  *   .readFrom("reader", List.of(1, 2, 3))
  *   .addBuffer("reader_queue", 10)
- *   .addWorker("process", 2, (i, next) -> next.accept(doExpensiveWork(i))
+ *   .addWorker("process", 2, (i, next) -> next.accept(doExpensiveWork(i)))
  *   .addBuffer("writer_queue", 10)
  *   .sinkToConsumer("writer", 1, result -> writeToDisk(result))
  *   .await();
  * }
- * 

  * 

  * NOTE: to do any forking/joining, you must construct and wire-up queues and each sequence of steps manually.
  *
  * @param  input type of this pipeline
  */
-public record WorkerPipeline (
+public record WorkerPipeline(
   String name,
   WorkerPipeline previous,
   WorkQueue inputQueue,
@@ -219,7 +216,7 @@ public record WorkerPipeline (
    *
    * @param  type of elements that the next step must process
    */
-  public record Builder (
+  public record Builder(
     String prefix,
     String name,
     // keep track of previous elements so that build can wire-up the computation graph
diff --git a/planetiler-core/src/main/proto/stream_archive_proto.proto b/planetiler-core/src/main/proto/stream_archive_proto.proto
index b33d94ae..17fd6961 100644
--- a/planetiler-core/src/main/proto/stream_archive_proto.proto
+++ b/planetiler-core/src/main/proto/stream_archive_proto.proto
@@ -1,4 +1,3 @@
-
 syntax = "proto3";
 
 package com.onthegomap.planetiler.proto;
@@ -19,7 +18,6 @@ message TileEntry {
 }
 
 message InitializationEntry {
-  Metadata metadata = 1;
 }
 
 message FinishEntry {
diff --git a/planetiler-core/src/main/resources/log4j2.properties b/planetiler-core/src/main/resources/log4j2.properties
index 03a0b83e..9c9af0c3 100644
--- a/planetiler-core/src/main/resources/log4j2.properties
+++ b/planetiler-core/src/main/resources/log4j2.properties
@@ -2,7 +2,7 @@ appenders=console
 appender.console.type=Console
 appender.console.name=STDOUT
 appender.console.layout.type=PatternLayout
-appender.console.layout.pattern=%highlight{$${uptime:now} %level{length=3} %notEmpty{[%X{stage}] }- %msg%n%throwable}{FATAL=red, ERROR=red, WARN=YELLOW, INFO=normal, DEBUG=normal, TRACE=normal}
+appender.console.layout.pattern=%highlight{$${uptime:now} %level{length=3} %X{stage}- %msg%n%throwable}{FATAL=red, ERROR=red, WARN=YELLOW, INFO=normal, DEBUG=normal, TRACE=normal}
 packages=com.onthegomap.planetiler.util.log4j
 rootLogger.level=debug
 rootLogger.appenderRefs=stdout
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureCollectorTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureCollectorTest.java
index 672fe9ef..678a29fd 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureCollectorTest.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureCollectorTest.java
@@ -493,6 +493,34 @@ class FeatureCollectorTest {
     assertFalse(iter.hasNext());
   }
 
+  @Test
+  void testInnermostPoint() {
+    /*
+      _____
+     | · __|
+     |__|
+     */
+    var sourceLine = newReaderFeature(newPolygon(worldToLatLon(
+      0, 0,
+      1, 0,
+      1, 0.5,
+      0.5, 0.5,
+      0.5, 1,
+      0, 1,
+      0, 0
+    )), Map.of());
+
+    var fc = factory.get(sourceLine);
+    fc.innermostPoint("layer").setZoomRange(0, 10);
+    var iter = fc.iterator();
+
+    var item = iter.next();
+    assertEquals(GeometryType.POINT, item.getGeometryType());
+    assertEquals(round(newPoint(0.28, 0.28)), round(item.getGeometry(), 1e2));
+
+    assertFalse(iter.hasNext());
+  }
+
   @Test
   void testMultiPolygonCoercion() throws GeometryException {
     var sourceLine = newReaderFeature(newMultiPolygon(
@@ -614,5 +642,4 @@ class FeatureCollectorTest {
       )
     ), collector);
   }
-
 }
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureMergeTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureMergeTest.java
index 32b6527e..0fd1628c 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureMergeTest.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureMergeTest.java
@@ -10,7 +10,9 @@ import com.onthegomap.planetiler.collection.Hppc;
 import com.onthegomap.planetiler.geo.GeometryException;
 import com.onthegomap.planetiler.geo.GeometryType;
 import com.onthegomap.planetiler.mbtiles.Mbtiles;
+import com.onthegomap.planetiler.stats.Stats;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -20,11 +22,14 @@ import java.util.function.UnaryOperator;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
 import org.locationtech.jts.geom.Geometry;
 import org.locationtech.jts.geom.GeometryCollection;
 import org.locationtech.jts.geom.LineString;
 import org.locationtech.jts.geom.Point;
 import org.locationtech.jts.geom.Polygon;
+import org.locationtech.jts.io.ParseException;
+import org.locationtech.jts.io.WKBReader;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -853,4 +858,27 @@ class FeatureMergeTest {
       )
     );
   }
+
+  @ParameterizedTest
+  @ValueSource(strings = {
+    "/issue_700/exception_1.wkb",
+    "/issue_700/exception_2.wkb",
+    "/issue_700/exception_3.wkb",
+    "/issue_700/exception_4.wkb",
+    "/issue_700/exception_5.wkb",
+    "/issue_700/exception_6.wkb",
+    "/issue_700/exception_7.wkb",
+    "/issue_700/exception_8.wkb",
+    "/issue_700/exception_9.wkb",
+  })
+  void testIssue700BufferUnionUnbufferFailure(String path) throws IOException, ParseException {
+    try (var is = getClass().getResource(path).openStream()) {
+      GeometryCollection collection = (GeometryCollection) new WKBReader().read(is.readAllBytes());
+      List geometries = new ArrayList<>();
+      for (int i = 0; i < collection.getNumGeometries(); i++) {
+        geometries.add(collection.getGeometryN(i));
+      }
+      FeatureMerge.bufferUnionUnbuffer(0.5, geometries, Stats.inMemory());
+    }
+  }
 }
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java
index d1964b29..d1224780 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java
@@ -87,6 +87,8 @@ class PlanetilerTests {
   private static final double Z13_WIDTH = 1d / Z13_TILES;
   private static final int Z12_TILES = 1 << 12;
   private static final double Z12_WIDTH = 1d / Z12_TILES;
+  private static final int Z11_TILES = 1 << 11;
+  private static final double Z11_WIDTH = 1d / Z11_TILES;
   private static final int Z4_TILES = 1 << 4;
   private static final Polygon WORLD_POLYGON = newPolygon(
     worldCoordinateList(
@@ -592,6 +594,15 @@ class PlanetilerTests {
     return points;
   }
 
+  public List z14PixelRectangle(double min, double max) {
+    List points = rectangleCoordList(min / 256d, max / 256d);
+    points.forEach(c -> {
+      c.x = GeoUtils.getWorldLon(0.5 + c.x * Z14_WIDTH);
+      c.y = GeoUtils.getWorldLat(0.5 + c.y * Z14_WIDTH);
+    });
+    return points;
+  }
+
   public List z14CoordinatePixelList(double... coords) {
     return z14CoordinateList(DoubleStream.of(coords).map(c -> c / 256d).toArray());
   }
@@ -827,7 +838,7 @@ class PlanetilerTests {
 
     var tileContents = results.tiles.get(TileCoord.ofXYZ(0, 0, 0));
     assertEquals(1, tileContents.size());
-    Geometry geom = tileContents.get(0).geometry().geom();
+    Geometry geom = tileContents.getFirst().geometry().geom();
     assertTrue(geom instanceof MultiPolygon, geom.toString());
     MultiPolygon multiPolygon = (MultiPolygon) geom;
     assertSameNormalizedFeature(newPolygon(
@@ -1884,7 +1895,7 @@ class PlanetilerTests {
       var point = newPoint(tileX, tileY);
 
       assertEquals(1, problematicTile.size());
-      var geomCompare = problematicTile.get(0).geometry();
+      var geomCompare = problematicTile.getFirst().geometry();
       geomCompare.validate();
       var geom = geomCompare.geom();
 
@@ -2341,6 +2352,158 @@ class PlanetilerTests {
     assertEquals(bboxResult.tiles, polyResult.tiles);
   }
 
+  @Test
+  void testSimplePolygon() throws Exception {
+    List points = z14PixelRectangle(0, 40);
+
+    var results = runWithReaderFeatures(
+      Map.of("threads", "1"),
+      List.of(
+        newReaderFeature(newPolygon(points), Map.of())
+      ),
+      (in, features) -> features.polygon("layer")
+        .setZoomRange(0, 14)
+        .setBufferPixels(0)
+        .setMinPixelSize(10) // should only show up z14 (40) z13 (20) and z12 (10)
+    );
+
+    assertEquals(Map.ofEntries(
+      newTileEntry(Z12_TILES / 2, Z12_TILES / 2, 12, List.of(
+        feature(newPolygon(rectangleCoordList(0, 10)), Map.of())
+      )),
+      newTileEntry(Z13_TILES / 2, Z13_TILES / 2, 13, List.of(
+        feature(newPolygon(rectangleCoordList(0, 20)), Map.of())
+      )),
+      newTileEntry(Z14_TILES / 2, Z14_TILES / 2, 14, List.of(
+        feature(newPolygon(rectangleCoordList(0, 40)), Map.of())
+      ))
+    ), results.tiles);
+  }
+
+  @Test
+  void testCentroidWithPolygonMinSize() throws Exception {
+    List points = z14PixelRectangle(0, 40);
+
+    var results = runWithReaderFeatures(
+      Map.of("threads", "1"),
+      List.of(
+        newReaderFeature(newPolygon(points), Map.of())
+      ),
+      (in, features) -> features.centroid("layer")
+        .setZoomRange(0, 14)
+        .setBufferPixels(0)
+        .setMinPixelSize(10) // should only show up z14 (40) z13 (20) and z12 (10)
+    );
+
+    assertEquals(Map.ofEntries(
+      newTileEntry(Z12_TILES / 2, Z12_TILES / 2, 12, List.of(
+        feature(newPoint(5, 5), Map.of())
+      )),
+      newTileEntry(Z13_TILES / 2, Z13_TILES / 2, 13, List.of(
+        feature(newPoint(10, 10), Map.of())
+      )),
+      newTileEntry(Z14_TILES / 2, Z14_TILES / 2, 14, List.of(
+        feature(newPoint(20, 20), Map.of())
+      ))
+    ), results.tiles);
+  }
+
+  @Test
+  void testCentroidWithLineMinSize() throws Exception {
+    List points = z14CoordinatePixelList(0, 4, 40, 4);
+
+    var results = runWithReaderFeatures(
+      Map.of("threads", "1"),
+      List.of(
+        newReaderFeature(newLineString(points), Map.of())
+      ),
+      (in, features) -> features.centroid("layer")
+        .setZoomRange(0, 14)
+        .setBufferPixels(0)
+        .setMinPixelSize(10) // should only show up z14 (40) z13 (20) and z12 (10)
+    );
+
+    assertEquals(Map.ofEntries(
+      newTileEntry(Z12_TILES / 2, Z12_TILES / 2, 12, List.of(
+        feature(newPoint(5, 1), Map.of())
+      )),
+      newTileEntry(Z13_TILES / 2, Z13_TILES / 2, 13, List.of(
+        feature(newPoint(10, 2), Map.of())
+      )),
+      newTileEntry(Z14_TILES / 2, Z14_TILES / 2, 14, List.of(
+        feature(newPoint(20, 4), Map.of())
+      ))
+    ), results.tiles);
+  }
+
+  @Test
+  void testAttributeMinSizeLine() throws Exception {
+    List points = z14CoordinatePixelList(0, 4, 40, 4);
+
+    var results = runWithReaderFeatures(
+      Map.of("threads", "1"),
+      List.of(
+        newReaderFeature(newLineString(points), Map.of())
+      ),
+      (in, features) -> features.line("layer")
+        .setZoomRange(11, 14)
+        .setBufferPixels(0)
+        .setAttrWithMinSize("a", "1", 10)
+        .setAttrWithMinSize("b", "2", 20)
+        .setAttrWithMinSize("c", "3", 40)
+        .setAttrWithMinSize("d", "4", 40, 0, 13) // should show up at z13 and above
+    );
+
+    assertEquals(Map.ofEntries(
+      newTileEntry(Z11_TILES / 2, Z11_TILES / 2, 11, List.of(
+        feature(newLineString(0, 0.5, 5, 0.5), Map.of())
+      )),
+      newTileEntry(Z12_TILES / 2, Z12_TILES / 2, 12, List.of(
+        feature(newLineString(0, 1, 10, 1), Map.of("a", "1"))
+      )),
+      newTileEntry(Z13_TILES / 2, Z13_TILES / 2, 13, List.of(
+        feature(newLineString(0, 2, 20, 2), Map.of("a", "1", "b", "2", "d", "4"))
+      )),
+      newTileEntry(Z14_TILES / 2, Z14_TILES / 2, 14, List.of(
+        feature(newLineString(0, 4, 40, 4), Map.of("a", "1", "b", "2", "c", "3", "d", "4"))
+      ))
+    ), results.tiles);
+  }
+
+  @Test
+  void testAttributeMinSizePoint() throws Exception {
+    List points = z14CoordinatePixelList(0, 4, 40, 4);
+
+    var results = runWithReaderFeatures(
+      Map.of("threads", "1"),
+      List.of(
+        newReaderFeature(newLineString(points), Map.of())
+      ),
+      (in, features) -> features.centroid("layer")
+        .setZoomRange(11, 14)
+        .setBufferPixels(0)
+        .setAttrWithMinSize("a", "1", 10)
+        .setAttrWithMinSize("b", "2", 20)
+        .setAttrWithMinSize("c", "3", 40)
+        .setAttrWithMinSize("d", "4", 40, 0, 13) // should show up at z13 and above
+    );
+
+    assertEquals(Map.ofEntries(
+      newTileEntry(Z11_TILES / 2, Z11_TILES / 2, 11, List.of(
+        feature(newPoint(2.5, 0.5), Map.of())
+      )),
+      newTileEntry(Z12_TILES / 2, Z12_TILES / 2, 12, List.of(
+        feature(newPoint(5, 1), Map.of("a", "1"))
+      )),
+      newTileEntry(Z13_TILES / 2, Z13_TILES / 2, 13, List.of(
+        feature(newPoint(10, 2), Map.of("a", "1", "b", "2", "d", "4"))
+      )),
+      newTileEntry(Z14_TILES / 2, Z14_TILES / 2, 14, List.of(
+        feature(newPoint(20, 4), Map.of("a", "1", "b", "2", "c", "3", "d", "4"))
+      ))
+    ), results.tiles);
+  }
+
   @Test
   void testBoundFiltersFill() throws Exception {
     var polyResultz8 = runForBoundsTest(8, 8, "polygon", TestUtils.pathToResource("bottomrightearth.poly").toString());
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/VectorTileTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/VectorTileTest.java
index 84a9f8b4..9f065ae4 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/VectorTileTest.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/VectorTileTest.java
@@ -160,7 +160,7 @@ class VectorTileTest {
 
     List decoded = VectorTile.decode(encoded);
     assertEquals(1, decoded.size());
-    Map decodedAttributes = decoded.get(0).attrs();
+    Map decodedAttributes = decoded.getFirst().attrs();
     assertEquals("value1", decodedAttributes.get("key1"));
     assertEquals(123L, decodedAttributes.get("key2"));
     assertEquals(234.1f, decodedAttributes.get("key3"));
@@ -220,7 +220,7 @@ class VectorTileTest {
 
     var features = VectorTile.decode(encoded);
     assertEquals(1, features.size());
-    MultiPolygon mp2 = (MultiPolygon) decodeSilently(features.get(0).geometry());
+    MultiPolygon mp2 = (MultiPolygon) decodeSilently(features.getFirst().geometry());
     assertEquals(mp.getNumGeometries(), mp2.getNumGeometries());
   }
 
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/geo/GeoUtilsTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/geo/GeoUtilsTest.java
index 81cff04c..dead3141 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/geo/GeoUtilsTest.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/geo/GeoUtilsTest.java
@@ -8,7 +8,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import com.onthegomap.planetiler.stats.Stats;
 import java.util.List;
-import org.geotools.geometry.jts.WKTReader2;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.CsvSource;
@@ -18,6 +17,7 @@ import org.locationtech.jts.geom.LinearRing;
 import org.locationtech.jts.geom.Point;
 import org.locationtech.jts.geom.util.AffineTransformation;
 import org.locationtech.jts.io.ParseException;
+import org.locationtech.jts.io.WKTReader;
 
 class GeoUtilsTest {
 
@@ -367,7 +367,7 @@ class GeoUtilsTest {
 
   @Test
   void testSnapAndFixIssue511() throws ParseException, GeometryException {
-    var result = GeoUtils.snapAndFixPolygon(new WKTReader2().read(
+    var result = GeoUtils.snapAndFixPolygon(new WKTReader().read(
       """
         MULTIPOLYGON (((198.83750000000003 46.07500000000004, 199.0625 46.375, 199.4375 46.0625, 199.5 46.43750000000001, 199.5625 46, 199.3125 45.5, 198.8912037037037 46.101851851851876, 198.83750000000003 46.07500000000004)), ((198.43750000000003 46.49999999999999, 198.5625 46.43750000000001, 198.6875 46.25, 198.1875 46.25, 198.43750000000003 46.49999999999999)), ((198.6875 46.25, 198.81249999999997 46.062500000000014, 198.6875 46.00000000000002, 198.6875 46.25)), ((196.55199579831933 46.29359243697479, 196.52255639097743 46.941259398496236, 196.5225563909774 46.941259398496236, 196.49999999999997 47.43750000000001, 196.875 47.125, 197 47.5625, 197.47880544905414 46.97729334004497, 197.51505401161464 46.998359569801956, 197.25 47.6875, 198.0625 47.6875, 198.5 46.625, 198.34375 46.546875, 198.34375000000003 46.54687499999999, 197.875 46.3125, 197.875 46.25, 197.875 46.0625, 197.82894736842107 46.20065789473683, 197.25 46.56250000000001, 197.3125 46.125, 196.9375 46.1875, 196.9375 46.21527777777778, 196.73250000000002 46.26083333333334, 196.5625 46.0625, 196.55199579831933 46.29359243697479)), ((196.35213414634146 45.8170731707317, 197.3402027027027 45.93108108108108, 197.875 45.99278846153846, 197.875 45.93750000000002, 197.93749999999997 45.99999999999999, 197.9375 46, 197.90625 45.96874999999999, 197.90625 45.96875, 196.75000000000006 44.81250000000007, 197.1875 45.4375, 196.3125 45.8125, 196.35213414634146 45.8170731707317)), ((195.875 46.124999999999986, 195.8125 46.5625, 196.5 46.31250000000001, 195.9375 46.4375, 195.875 46.124999999999986)), ((196.49999999999997 46.93749999999999, 196.125 46.875, 196.3125 47.125, 196.49999999999997 46.93749999999999)))
         """),
@@ -377,7 +377,7 @@ class GeoUtilsTest {
 
   @Test
   void testSnapAndFixIssue546() throws GeometryException, ParseException {
-    var orig = new WKTReader2().read(
+    var orig = new WKTReader().read(
       """
         POLYGON(
           (
@@ -404,7 +404,7 @@ class GeoUtilsTest {
 
   @Test
   void testSnapAndFixIssue546_2() throws GeometryException, ParseException {
-    var orig = new WKTReader2().read(
+    var orig = new WKTReader().read(
       """
         POLYGON(
           (
@@ -423,7 +423,7 @@ class GeoUtilsTest {
 
   @Test
   void testSnapAndFixIssue546_3() throws GeometryException, ParseException {
-    var orig = new WKTReader2().read(
+    var orig = new WKTReader().read(
       """
         POLYGON(
           (
@@ -447,4 +447,30 @@ class GeoUtilsTest {
     assertTrue(result.isValid());
     assertFalse(result.contains(point));
   }
+
+  @ParameterizedTest
+  @CsvSource({
+    "1,0,0",
+    "1,10,0",
+    "1,255,0",
+
+    "0.5,0,0",
+    "0.5,128,0",
+    "0.5,129,1",
+    "0.5,256,1",
+
+    "0.25,0,0",
+    "0.25,128,1",
+    "0.25,129,2",
+    "0.25,256,2",
+  })
+  void minZoomForPixelSize(double worldGeometrySize, double minPixelSize, int expectedMinZoom) {
+    assertEquals(expectedMinZoom, GeoUtils.minZoomForPixelSize(worldGeometrySize, minPixelSize));
+  }
+
+  @Test
+  void minZoomForPixelSizesAtZ9_10() {
+    assertEquals(10, GeoUtils.minZoomForPixelSize(3.1 / (256 << 10), 3));
+    assertEquals(9, GeoUtils.minZoomForPixelSize(6.1 / (256 << 10), 3));
+  }
 }
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/pmtiles/PmtilesTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/pmtiles/PmtilesTest.java
index 08384a94..48c09cea 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/pmtiles/PmtilesTest.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/pmtiles/PmtilesTest.java
@@ -188,7 +188,7 @@ class PmtilesTest {
 
     var config = PlanetilerConfig.defaults();
     var metadata = new TileArchiveMetadata(new Profile.NullProfile(), config);
-    in.initialize(metadata);
+    in.initialize();
     var writer = in.newTileWriter();
     writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 1), new byte[]{0xa, 0x2}, OptionalLong.empty()));
 
@@ -259,7 +259,7 @@ class PmtilesTest {
       var channel = new SeekableInMemoryByteChannel(0);
       var in = WriteablePmtiles.newWriteToMemory(channel)
     ) {
-      in.initialize(input);
+      in.initialize();
       var writer = in.newTileWriter();
       writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0xa, 0x2}, OptionalLong.empty()));
 
@@ -299,7 +299,7 @@ class PmtilesTest {
 
     var config = PlanetilerConfig.defaults();
     var metadata = new TileArchiveMetadata(new Profile.NullProfile(), config);
-    in.initialize(metadata);
+    in.initialize();
     var writer = in.newTileWriter();
     writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0xa, 0x2}, OptionalLong.of(42)));
     writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 1), new byte[]{0xa, 0x2}, OptionalLong.of(42)));
@@ -337,7 +337,7 @@ class PmtilesTest {
 
     var config = PlanetilerConfig.defaults();
     var metadata = new TileArchiveMetadata(new Profile.NullProfile(), config);
-    in.initialize(metadata);
+    in.initialize();
     var writer = in.newTileWriter();
     writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 1), new byte[]{0xa, 0x2}, OptionalLong.of(42)));
     writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0xa, 0x2}, OptionalLong.of(42)));
@@ -372,7 +372,7 @@ class PmtilesTest {
 
     var config = PlanetilerConfig.defaults();
     var metadata = new TileArchiveMetadata(new Profile.NullProfile(), config);
-    in.initialize(metadata);
+    in.initialize();
     var writer = in.newTileWriter();
 
     int ENTRIES = 20000;
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/ShapefileReaderTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/ShapefileReaderTest.java
index 1cb2d209..26b1c97e 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/ShapefileReaderTest.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/ShapefileReaderTest.java
@@ -1,9 +1,11 @@
 package com.onthegomap.planetiler.reader;
 
+import static com.onthegomap.planetiler.TestUtils.newPoint;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import com.onthegomap.planetiler.TestUtils;
+import com.onthegomap.planetiler.config.Bounds;
 import com.onthegomap.planetiler.geo.GeoUtils;
 import com.onthegomap.planetiler.stats.Stats;
 import com.onthegomap.planetiler.util.FileUtils;
@@ -11,9 +13,9 @@ import com.onthegomap.planetiler.worker.WorkerPipeline;
 import java.io.IOException;
 import java.nio.file.FileSystems;
 import java.nio.file.Path;
-import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.CopyOnWriteArrayList;
 import org.geotools.api.data.SimpleFeatureStore;
 import org.geotools.api.referencing.FactoryException;
 import org.geotools.api.referencing.operation.TransformException;
@@ -29,12 +31,18 @@ import org.junit.jupiter.api.Timeout;
 import org.junit.jupiter.api.condition.DisabledOnOs;
 import org.junit.jupiter.api.condition.OS;
 import org.junit.jupiter.api.io.TempDir;
+import org.locationtech.jts.geom.Envelope;
 import org.locationtech.jts.geom.Geometry;
 import org.locationtech.jts.geom.Point;
 
 class ShapefileReaderTest {
   @TempDir
   private Path tempDir;
+  private static final Envelope env = newPoint(-77.12911152370515, 38.79930767201779).getEnvelopeInternal();
+  private static final int numInEnv = 18;
+  static {
+    env.expandBy(0.1);
+  }
 
   @Test
   @Timeout(30)
@@ -55,6 +63,35 @@ class ShapefileReaderTest {
     testReadShapefile(dest.resolve("shapefile").resolve("stations.shp"));
   }
 
+  @Test
+  @Timeout(30)
+  void testReadShapefileWithBoundingBox() {
+    var dest = tempDir.resolve("shapefile.zip");
+    FileUtils.unzipResource("/shapefile.zip", dest);
+    try (
+      var reader = new ShapefileReader(null, "test", dest.resolve("shapefile").resolve("stations.shp"), new Bounds(env))
+    ) {
+      for (int i = 1; i <= 2; i++) {
+        assertEquals(numInEnv, reader.getFeatureCount());
+        List points = new CopyOnWriteArrayList<>();
+        WorkerPipeline.start("test", Stats.inMemory())
+          .fromGenerator("source", reader::readFeatures)
+          .addBuffer("reader_queue", 100, 1)
+          .sinkToConsumer("counter", 1, elem -> {
+            assertTrue(elem.getTag("name") instanceof String);
+            assertEquals("test", elem.getSource());
+            assertEquals("stations", elem.getSourceLayer());
+            points.add(elem.latLonGeometry());
+          }).await();
+        assertEquals(numInEnv, points.size());
+        var gc = GeoUtils.JTS_FACTORY.createGeometryCollection(points.toArray(new Geometry[0]));
+        var centroid = gc.getCentroid();
+        assertEquals(-77.0934256, centroid.getX(), 1e-5, "iter " + i);
+        assertEquals(38.8509022, centroid.getY(), 1e-5, "iter " + i);
+      }
+    }
+  }
+
   @Test
   void testReadShapefileLeniently(@TempDir Path dir) throws IOException, TransformException, FactoryException {
     var shpPath = dir.resolve("test.shp");
@@ -82,7 +119,7 @@ class ShapefileReaderTest {
       featureStore.setTransaction(transaction);
       var collection = new DefaultFeatureCollection();
       var featureBuilder = new SimpleFeatureBuilder(type);
-      featureBuilder.add(TestUtils.newPoint(1, 2));
+      featureBuilder.add(newPoint(1, 2));
       featureBuilder.add(3);
       var feature = featureBuilder.buildFeature(null);
       collection.add(feature);
@@ -92,11 +129,11 @@ class ShapefileReaderTest {
 
     try (var reader = new ShapefileReader(null, "test", shpPath)) {
       assertEquals(1, reader.getFeatureCount());
-      List features = new ArrayList<>();
+      List features = new CopyOnWriteArrayList<>();
       reader.readFeatures(features::add);
-      assertEquals(10.5113, features.get(0).latLonGeometry().getCentroid().getX(), 1e-4);
-      assertEquals(0, features.get(0).latLonGeometry().getCentroid().getY(), 1e-4);
-      assertEquals(3, features.get(0).getTag("value"));
+      assertEquals(10.5113, features.getFirst().latLonGeometry().getCentroid().getX(), 1e-4);
+      assertEquals(0, features.getFirst().latLonGeometry().getCentroid().getY(), 1e-4);
+      assertEquals(3, features.getFirst().getTag("value"));
     }
   }
 
@@ -105,8 +142,8 @@ class ShapefileReaderTest {
 
       for (int i = 1; i <= 2; i++) {
         assertEquals(86, reader.getFeatureCount());
-        List points = new ArrayList<>();
-        List names = new ArrayList<>();
+        List points = new CopyOnWriteArrayList<>();
+        List names = new CopyOnWriteArrayList<>();
         WorkerPipeline.start("test", Stats.inMemory())
           .fromGenerator("source", reader::readFeatures)
           .addBuffer("reader_queue", 100, 1)
@@ -117,12 +154,13 @@ class ShapefileReaderTest {
             points.add(elem.latLonGeometry());
             names.add(elem.getTag("name").toString());
           }).await();
+        assertEquals(numInEnv, points.stream().filter(point -> env.contains(point.getCoordinate())).count());
         assertEquals(86, points.size());
         assertTrue(names.contains("Van Dörn Street"));
         var gc = GeoUtils.JTS_FACTORY.createGeometryCollection(points.toArray(new Geometry[0]));
         var centroid = gc.getCentroid();
-        assertEquals(-77.0297995, centroid.getX(), 5, "iter " + i);
-        assertEquals(38.9119684, centroid.getY(), 5, "iter " + i);
+        assertEquals(-77.0297995, centroid.getX(), 1e-5, "iter " + i);
+        assertEquals(38.9119684, centroid.getY(), 1e-5, "iter " + i);
       }
     }
   }
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/render/FeatureRendererTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/render/FeatureRendererTest.java
index fab95501..549f7569 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/render/FeatureRendererTest.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/render/FeatureRendererTest.java
@@ -47,7 +47,7 @@ class FeatureRendererTest {
   private FeatureCollector collector(Geometry worldGeom) {
     var latLonGeom = GeoUtils.worldToLatLonCoords(worldGeom);
     return new FeatureCollector.Factory(config, stats)
-      .get(SimpleFeature.create(latLonGeom, new HashMap<>(0), null, null,
+      .get(SimpleFeature.create(latLonGeom, HashMap.newHashMap(0), null, null,
         1));
   }
 
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableCsvArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableCsvArchiveTest.java
index aef43850..39fd244a 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableCsvArchiveTest.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableCsvArchiveTest.java
@@ -32,7 +32,7 @@ class WriteableCsvArchiveTest {
     final Path csvFile = tempDir.resolve("out.csv");
 
     try (var archive = WriteableCsvArchive.newWriteToFile(format, csvFile, defaultConfig)) {
-      archive.initialize(defaultMetadata); // ignored
+      archive.initialize();
       try (var tileWriter = archive.newTileWriter()) {
         tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}, OptionalLong.empty()));
         tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(1, 2, 3), new byte[]{1}, OptionalLong.of(1)));
@@ -67,7 +67,7 @@ class WriteableCsvArchiveTest {
     try (
       var archive = WriteableCsvArchive.newWriteToFile(TileArchiveConfig.Format.CSV, csvFilePrimary, defaultConfig)
     ) {
-      archive.initialize(defaultMetadata); // ignored
+      archive.initialize();
       try (var tileWriter = archive.newTileWriter()) {
         tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(11, 12, 1), new byte[]{0}, OptionalLong.empty()));
         tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(21, 22, 2), new byte[]{1}, OptionalLong.empty()));
@@ -159,7 +159,7 @@ class WriteableCsvArchiveTest {
     final Path csvFile = tempDir.resolve("out.csv");
 
     try (var archive = WriteableCsvArchive.newWriteToFile(TileArchiveConfig.Format.CSV, csvFile, config)) {
-      archive.initialize(defaultMetadata);
+      archive.initialize();
       try (var tileWriter = archive.newTileWriter()) {
         tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0, 1}, OptionalLong.empty()));
         tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(1, 1, 1), new byte[]{2, 3}, OptionalLong.empty()));
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchiveTest.java
index 21005e4f..0a9dd22f 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchiveTest.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchiveTest.java
@@ -32,7 +32,7 @@ import org.locationtech.jts.geom.Envelope;
 class WriteableJsonStreamArchiveTest {
 
   private static final StreamArchiveConfig defaultConfig = new StreamArchiveConfig(false, Arguments.of());
-  private static final TileArchiveMetadata maxMetadataIn =
+  private static final TileArchiveMetadata MAX_METADATA_IN =
     new TileArchiveMetadata("name", "description", "attribution", "version", "type", "format", new Envelope(0, 1, 2, 3),
       new CoordinateXY(1.3, 3.7), 1.0, 2, 3,
       List.of(
@@ -46,7 +46,7 @@ class WriteableJsonStreamArchiveTest {
       ),
       ImmutableMap.of("a", "b", "c", "d"),
       TileCompression.GZIP);
-  private static final String maxMetadataOut = """
+  private static final String MAX_METADATA_OUT = """
     {
       "name":"name",
       "description":"description",
@@ -88,7 +88,7 @@ class WriteableJsonStreamArchiveTest {
       "c":"d"
     }""".lines().map(String::trim).collect(Collectors.joining(""));
 
-  private static final TileArchiveMetadata minMetadataIn =
+  private static final TileArchiveMetadata MIN_METADATA_IN =
     new TileArchiveMetadata(null, null, null, null, null, null, null, null, null, null, null, null, null, null);
   private static final String MIN_METADATA_OUT = "{}";
 
@@ -98,21 +98,21 @@ class WriteableJsonStreamArchiveTest {
     final Path csvFile = tempDir.resolve("out.json");
 
     try (var archive = WriteableJsonStreamArchive.newWriteToFile(csvFile, defaultConfig)) {
-      archive.initialize(maxMetadataIn);
+      archive.initialize();
       try (var tileWriter = archive.newTileWriter()) {
         tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}, OptionalLong.empty()));
         tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(1, 2, 3), new byte[]{1}, OptionalLong.of(1)));
       }
-      archive.finish(minMetadataIn);
+      archive.finish(MIN_METADATA_IN);
     }
 
     assertEqualsDelimitedJson(
       """
-        {"type":"initialization","metadata":%s}
+        {"type":"initialization"}
         {"type":"tile","x":0,"y":0,"z":0,"encodedData":"AA=="}
         {"type":"tile","x":1,"y":2,"z":3,"encodedData":"AQ=="}
         {"type":"finish","metadata":%s}
-        """.formatted(maxMetadataOut, MIN_METADATA_OUT),
+        """.formatted(MIN_METADATA_OUT),
       Files.readString(csvFile)
     );
 
@@ -132,7 +132,7 @@ class WriteableJsonStreamArchiveTest {
     final var tile3 = new TileEncodingResult(TileCoord.ofXYZ(41, 42, 4), new byte[]{3}, OptionalLong.empty());
     final var tile4 = new TileEncodingResult(TileCoord.ofXYZ(51, 52, 5), new byte[]{4}, OptionalLong.empty());
     try (var archive = WriteableJsonStreamArchive.newWriteToFile(csvFilePrimary, defaultConfig)) {
-      archive.initialize(minMetadataIn);
+      archive.initialize();
       try (var tileWriter = archive.newTileWriter()) {
         tileWriter.write(tile0);
         tileWriter.write(tile1);
@@ -144,16 +144,16 @@ class WriteableJsonStreamArchiveTest {
       try (var tileWriter = archive.newTileWriter()) {
         tileWriter.write(tile4);
       }
-      archive.finish(maxMetadataIn);
+      archive.finish(MAX_METADATA_IN);
     }
 
     assertEqualsDelimitedJson(
       """
-        {"type":"initialization","metadata":%s}
+        {"type":"initialization"}
         {"type":"tile","x":11,"y":12,"z":1,"encodedData":"AA=="}
         {"type":"tile","x":21,"y":22,"z":2,"encodedData":"AQ=="}
         {"type":"finish","metadata":%s}
-        """.formatted(MIN_METADATA_OUT, maxMetadataOut),
+        """.formatted(MAX_METADATA_OUT),
       Files.readString(csvFilePrimary)
     );
 
@@ -199,11 +199,11 @@ class WriteableJsonStreamArchiveTest {
 
     final String expectedJson =
       """
-        {"type":"initialization","metadata":%s}
+        {"type":"initialization"}
         {"type":"tile","x":0,"y":0,"z":0,"encodedData":"AA=="}
         {"type":"tile","x":1,"y":2,"z":3,"encodedData":"AQ=="}
         {"type":"finish","metadata":%s}
-        """.formatted(MIN_METADATA_OUT, maxMetadataOut)
+        """.formatted(MAX_METADATA_OUT)
         .replace('\n', ' ');
 
     testTileOptions(tempDir, config, expectedJson);
@@ -216,12 +216,12 @@ class WriteableJsonStreamArchiveTest {
     final Path csvFile = tempDir.resolve("out.json");
 
     try (var archive = WriteableJsonStreamArchive.newWriteToFile(csvFile, config)) {
-      archive.initialize(minMetadataIn);
+      archive.initialize();
       try (var tileWriter = archive.newTileWriter()) {
         tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}, OptionalLong.empty()));
         tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(1, 2, 3), new byte[]{1}, OptionalLong.empty()));
       }
-      archive.finish(maxMetadataIn);
+      archive.finish(MAX_METADATA_IN);
     }
 
     assertEqualsDelimitedJson(expectedJson, Files.readString(csvFile));
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchiveTest.java
index b69f3a50..dd7299ef 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchiveTest.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchiveTest.java
@@ -76,7 +76,7 @@ class WriteableProtoStreamArchiveTest {
     final var tile0 = new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}, OptionalLong.empty());
     final var tile1 = new TileEncodingResult(TileCoord.ofXYZ(1, 2, 3), new byte[]{1}, OptionalLong.of(1));
     try (var archive = WriteableProtoStreamArchive.newWriteToFile(csvFile, defaultConfig)) {
-      archive.initialize(maxMetadataIn);
+      archive.initialize();
       try (var tileWriter = archive.newTileWriter()) {
         tileWriter.write(tile0);
         tileWriter.write(tile1);
@@ -86,7 +86,7 @@ class WriteableProtoStreamArchiveTest {
 
     try (InputStream in = Files.newInputStream(csvFile)) {
       assertEquals(
-        List.of(wrapInit(maxMetadataOut), toEntry(tile0), toEntry(tile1), wrapFinish(minMetadataOut)),
+        List.of(wrapInit(), toEntry(tile0), toEntry(tile1), wrapFinish(minMetadataOut)),
         readAllEntries(in)
       );
     }
@@ -105,7 +105,7 @@ class WriteableProtoStreamArchiveTest {
     final var tile3 = new TileEncodingResult(TileCoord.ofXYZ(41, 42, 4), new byte[]{3}, OptionalLong.empty());
     final var tile4 = new TileEncodingResult(TileCoord.ofXYZ(51, 52, 5), new byte[]{4}, OptionalLong.empty());
     try (var archive = WriteableProtoStreamArchive.newWriteToFile(csvFilePrimary, defaultConfig)) {
-      archive.initialize(minMetadataIn);
+      archive.initialize();
       try (var tileWriter = archive.newTileWriter()) {
         tileWriter.write(tile0);
         tileWriter.write(tile1);
@@ -122,7 +122,7 @@ class WriteableProtoStreamArchiveTest {
 
     try (InputStream in = Files.newInputStream(csvFilePrimary)) {
       assertEquals(
-        List.of(wrapInit(minMetadataOut), toEntry(tile0), toEntry(tile1), wrapFinish(maxMetadataOut)),
+        List.of(wrapInit(), toEntry(tile0), toEntry(tile1), wrapFinish(maxMetadataOut)),
         readAllEntries(in)
       );
     }
@@ -167,10 +167,8 @@ class WriteableProtoStreamArchiveTest {
       .build();
   }
 
-  private static StreamArchiveProto.Entry wrapInit(StreamArchiveProto.Metadata metadata) {
-    return StreamArchiveProto.Entry.newBuilder()
-      .setInitialization(StreamArchiveProto.InitializationEntry.newBuilder().setMetadata(metadata).build())
-      .build();
+  private static StreamArchiveProto.Entry wrapInit() {
+    return StreamArchiveProto.Entry.newBuilder().build();
   }
 
   private static StreamArchiveProto.Entry wrapFinish(StreamArchiveProto.Metadata metadata) {
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/DownloaderTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/DownloaderTest.java
index f7143977..f5b285b2 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/DownloaderTest.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/DownloaderTest.java
@@ -3,18 +3,16 @@ package com.onthegomap.planetiler.util;
 import static org.junit.jupiter.api.Assertions.*;
 
 import com.onthegomap.planetiler.config.PlanetilerConfig;
-import com.onthegomap.planetiler.stats.Stats;
 import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.Arrays;
 import java.util.Map;
 import java.util.Optional;
-import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicLong;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
 import org.junit.jupiter.params.ParameterizedTest;
@@ -25,26 +23,25 @@ class DownloaderTest {
   @TempDir
   Path path;
   private final PlanetilerConfig config = PlanetilerConfig.defaults();
-  private final Stats stats = Stats.inMemory();
-  private long downloads = 0;
+  private AtomicLong downloads = new AtomicLong(0);
 
-  private Downloader mockDownloader(Map resources, boolean supportsRange, int maxLength) {
-    return new Downloader(config, stats, 2L) {
+  private Downloader mockDownloader(Map resources, boolean supportsRange) {
+    return new Downloader(config, 2L) {
 
       @Override
       InputStream openStream(String url) {
-        downloads++;
+        downloads.incrementAndGet();
         assertTrue(resources.containsKey(url), "no resource for " + url);
         byte[] bytes = resources.get(url);
-        return new ByteArrayInputStream(maxLength < bytes.length ? Arrays.copyOf(bytes, maxLength) : bytes);
+        return new ByteArrayInputStream(bytes);
       }
 
       @Override
       InputStream openStreamRange(String url, long start, long end) {
         assertTrue(supportsRange, "does not support range");
-        downloads++;
+        downloads.incrementAndGet();
         assertTrue(resources.containsKey(url), "no resource for " + url);
-        byte[] result = new byte[Math.min(maxLength, (int) (end - start))];
+        byte[] result = new byte[(int) (end - start)];
         byte[] bytes = resources.get(url);
         for (int i = (int) start; i < start + result.length; i++) {
           result[(int) (i - start)] = bytes[i];
@@ -53,31 +50,28 @@ class DownloaderTest {
       }
 
       @Override
-      CompletableFuture httpHead(String url) {
+      ResourceMetadata httpHead(String url) {
         String[] parts = url.split("#");
         if (parts.length > 1) {
           int redirectNum = Integer.parseInt(parts[1]);
           String next = redirectNum <= 1 ? parts[0] : (parts[0] + "#" + (redirectNum - 1));
-          return CompletableFuture.supplyAsync(
-            () -> new ResourceMetadata(Optional.of(next), url, 0, supportsRange));
+          return new ResourceMetadata(Optional.of(next), url, 0, supportsRange);
         }
         byte[] bytes = resources.get(url);
-        return CompletableFuture.supplyAsync(
-          () -> new ResourceMetadata(Optional.empty(), url, bytes.length, supportsRange));
+        return new ResourceMetadata(Optional.empty(), url, bytes.length, supportsRange);
       }
     };
   }
 
   @ParameterizedTest
   @CsvSource({
-    "false,100,0",
-    "true,100,0",
-    "true,2,0",
-    "false,100,1",
-    "false,100,2",
-    "true,2,4",
+    "false,0",
+    "true,0",
+    "false,1",
+    "false,2",
+    "true,4",
   })
-  void testDownload(boolean range, int maxLength, int redirects) throws Exception {
+  void testDownload(boolean range, int redirects) throws Exception {
     Path dest = path.resolve("out");
     String string = "0123456789";
     String url = "http://url";
@@ -85,7 +79,7 @@ class DownloaderTest {
     Map resources = new ConcurrentHashMap<>();
 
     byte[] bytes = string.getBytes(StandardCharsets.UTF_8);
-    Downloader downloader = mockDownloader(resources, range, maxLength);
+    Downloader downloader = mockDownloader(resources, range);
 
     // fails if no data
     var resource1 = new Downloader.ResourceToDownload("resource", initialUrl, dest);
@@ -102,10 +96,10 @@ class DownloaderTest {
     assertEquals(10, resource2.bytesDownloaded());
 
     // does not re-request if size is the same
-    downloads = 0;
+    downloads.set(0);
     var resource3 = new Downloader.ResourceToDownload("resource", initialUrl, dest);
     downloader.downloadIfNecessary(resource3).get();
-    assertEquals(0, downloads);
+    assertEquals(0, downloads.get());
     assertEquals(string, Files.readString(dest));
     assertEquals(FileUtils.size(path), FileUtils.size(dest));
     assertEquals(0, resource3.bytesDownloaded());
@@ -115,7 +109,7 @@ class DownloaderTest {
     String newContent = "54321";
     resources.put(url, newContent.getBytes(StandardCharsets.UTF_8));
     downloader.downloadIfNecessary(resource4).get();
-    assertTrue(downloads > 0, "downloads were " + downloads);
+    assertTrue(downloads.get() > 0, "downloads were " + downloads);
     assertEquals(newContent, Files.readString(dest));
     assertEquals(FileUtils.size(path), FileUtils.size(dest));
     assertEquals(5, resource4.bytesDownloaded());
@@ -123,7 +117,7 @@ class DownloaderTest {
 
   @Test
   void testDownloadFailsIfTooBig() {
-    var downloader = new Downloader(config, stats, 2L) {
+    var downloader = new Downloader(config, 2L) {
 
       @Override
       InputStream openStream(String url) {
@@ -136,8 +130,8 @@ class DownloaderTest {
       }
 
       @Override
-      CompletableFuture httpHead(String url) {
-        return CompletableFuture.completedFuture(new ResourceMetadata(Optional.empty(), url, Long.MAX_VALUE, true));
+      ResourceMetadata httpHead(String url) {
+        return new ResourceMetadata(Optional.empty(), url, Long.MAX_VALUE, true);
       }
     };
 
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/FileUtilsTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/FileUtilsTest.java
index 0dbb5b71..2f480cbd 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/FileUtilsTest.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/FileUtilsTest.java
@@ -152,4 +152,11 @@ class FileUtilsTest {
       List.of("/shapefile/stations.shp", "/shapefile/stations.shx"),
       matchingPaths.stream().map(Path::toString).sorted().toList());
   }
+
+  @Test
+  void testExpandFile() throws IOException {
+    Path path = tmpDir.resolve("toExpand");
+    FileUtils.setLength(path, 1000);
+    assertEquals(1000, Files.size(path));
+  }
 }
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/LayerAttrStatsTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/LayerAttrStatsTest.java
index 8835d68c..d3f567fa 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/LayerAttrStatsTest.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/LayerAttrStatsTest.java
@@ -2,13 +2,8 @@ package com.onthegomap.planetiler.util;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
-import com.onthegomap.planetiler.VectorTile;
-import com.onthegomap.planetiler.geo.GeoUtils;
-import com.onthegomap.planetiler.geo.TileCoord;
-import com.onthegomap.planetiler.render.RenderedFeature;
-import java.util.Arrays;
+import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import org.junit.jupiter.api.Test;
 
 class LayerAttrStatsTest {
@@ -17,109 +12,50 @@ class LayerAttrStatsTest {
 
   @Test
   void testEmptyLayerStats() {
-    assertEquals(Arrays.asList(new LayerAttrStats.VectorLayer[]{}), layerStats.getTileStats());
+    assertEquals(List.of(), layerStats.getTileStats());
   }
 
   @Test
   void testEmptyLayerStatsOneLayer() {
-    layerStats.accept(new RenderedFeature(
-      TileCoord.ofXYZ(1, 2, 3),
-      new VectorTile.Feature(
-        "layer1",
-        1,
-        VectorTile.encodeGeometry(GeoUtils.point(1, 2)),
-        Map.of("a", 1, "b", "string", "c", true)
-      ),
-      1,
-      Optional.empty()
-    ));
-    assertEquals(Arrays.asList(new LayerAttrStats.VectorLayer[]{
-      new LayerAttrStats.VectorLayer("layer1", Map.of(
-        "a", LayerAttrStats.FieldType.NUMBER,
-        "b", LayerAttrStats.FieldType.STRING,
-        "c", LayerAttrStats.FieldType.BOOLEAN
-      ), 3, 3)
-    }), layerStats.getTileStats());
+    layerStats.accept("layer1", 3, "a", 1);
+    layerStats.accept("layer1", 3, "b", "string");
+    layerStats.accept("layer1", 3, "c", true);
+    assertEquals(List.of(new LayerAttrStats.VectorLayer("layer1", Map.of(
+      "a", LayerAttrStats.FieldType.NUMBER,
+      "b", LayerAttrStats.FieldType.STRING,
+      "c", LayerAttrStats.FieldType.BOOLEAN
+    ), 3, 3)), layerStats.getTileStats());
   }
 
   @Test
   void testEmptyLayerStatsTwoLayers() {
-    layerStats.accept(new RenderedFeature(
-      TileCoord.ofXYZ(1, 2, 3),
-      new VectorTile.Feature(
-        "layer1",
-        1,
-        VectorTile.encodeGeometry(GeoUtils.point(1, 2)),
-        Map.of()
-      ),
-      1,
-      Optional.empty()
-    ));
-    layerStats.accept(new RenderedFeature(
-      TileCoord.ofXYZ(1, 2, 4),
-      new VectorTile.Feature(
-        "layer2",
-        1,
-        VectorTile.encodeGeometry(GeoUtils.point(1, 2)),
-        Map.of("a", 1, "b", true, "c", true)
-      ),
-      1,
-      Optional.empty()
-    ));
-    layerStats.accept(new RenderedFeature(
-      TileCoord.ofXYZ(1, 2, 1),
-      new VectorTile.Feature(
-        "layer2",
-        1,
-        VectorTile.encodeGeometry(GeoUtils.point(1, 2)),
-        Map.of("a", 1, "b", true, "c", 1)
-      ),
-      1,
-      Optional.empty()
-    ));
-    assertEquals(Arrays.asList(new LayerAttrStats.VectorLayer[]{
-      new LayerAttrStats.VectorLayer("layer1", Map.of(
-      ), 3, 3),
+    layerStats.handlerForThread().forZoom(3).forLayer("layer1");
+    layerStats.accept("layer2", 4, "a", 1);
+    layerStats.accept("layer2", 4, "b", true);
+    layerStats.accept("layer2", 4, "c", true);
+    layerStats.accept("layer2", 1, "a", 1);
+    layerStats.accept("layer2", 1, "b", true);
+    layerStats.accept("layer2", 1, "c", 1);
+    assertEquals(List.of(new LayerAttrStats.VectorLayer("layer1", Map.of(
+    ), 3, 3),
       new LayerAttrStats.VectorLayer("layer2", Map.of(
         "a", LayerAttrStats.FieldType.NUMBER,
         "b", LayerAttrStats.FieldType.BOOLEAN,
         "c", LayerAttrStats.FieldType.STRING
-      ), 1, 4)
-    }), layerStats.getTileStats());
+      ), 1, 4)), layerStats.getTileStats());
   }
 
   @Test
   void testMergeFromMultipleThreads() throws InterruptedException {
-    Thread t1 = new Thread(() -> layerStats.accept(new RenderedFeature(
-      TileCoord.ofXYZ(1, 2, 3),
-      new VectorTile.Feature(
-        "layer1",
-        1,
-        VectorTile.encodeGeometry(GeoUtils.point(1, 2)),
-        Map.of("a", 1)
-      ),
-      1,
-      Optional.empty()
-    )));
+    layerStats.accept("layer1", 3, "a", true);
+    Thread t1 = new Thread(() -> layerStats.accept("layer1", 3, "a", 1));
     t1.start();
-    Thread t2 = new Thread(() -> layerStats.accept(new RenderedFeature(
-      TileCoord.ofXYZ(1, 2, 4),
-      new VectorTile.Feature(
-        "layer1",
-        1,
-        VectorTile.encodeGeometry(GeoUtils.point(1, 2)),
-        Map.of("a", true)
-      ),
-      1,
-      Optional.empty()
-    )));
+    Thread t2 = new Thread(() -> layerStats.accept("layer1", 4, "a", true));
     t2.start();
     t1.join();
     t2.join();
-    assertEquals(Arrays.asList(new LayerAttrStats.VectorLayer[]{
-      new LayerAttrStats.VectorLayer("layer1", Map.of(
-        "a", LayerAttrStats.FieldType.STRING
-      ), 3, 4)
-    }), layerStats.getTileStats());
+    assertEquals(List.of(new LayerAttrStats.VectorLayer("layer1", Map.of(
+      "a", LayerAttrStats.FieldType.STRING
+    ), 3, 4)), layerStats.getTileStats());
   }
 }
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/LogUtilTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/LogUtilTest.java
new file mode 100644
index 00000000..0fc3b22a
--- /dev/null
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/LogUtilTest.java
@@ -0,0 +1,20 @@
+package com.onthegomap.planetiler.util;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import org.junit.jupiter.api.Test;
+
+class LogUtilTest {
+  @Test
+  void testStageHandling() {
+    LogUtil.clearStage();
+    assertNull(LogUtil.getStage());
+    LogUtil.setStage("test");
+    assertEquals("test", LogUtil.getStage());
+    LogUtil.setStage(LogUtil.getStage(), "child");
+    assertEquals("test:child", LogUtil.getStage());
+    LogUtil.clearStage();
+    assertNull(LogUtil.getStage());
+  }
+}
diff --git a/planetiler-core/src/test/resources/issue_700/exception_1.wkb b/planetiler-core/src/test/resources/issue_700/exception_1.wkb
new file mode 100644
index 00000000..4b2108d4
Binary files /dev/null and b/planetiler-core/src/test/resources/issue_700/exception_1.wkb differ
diff --git a/planetiler-core/src/test/resources/issue_700/exception_2.wkb b/planetiler-core/src/test/resources/issue_700/exception_2.wkb
new file mode 100644
index 00000000..7f4c774f
Binary files /dev/null and b/planetiler-core/src/test/resources/issue_700/exception_2.wkb differ
diff --git a/planetiler-core/src/test/resources/issue_700/exception_3.wkb b/planetiler-core/src/test/resources/issue_700/exception_3.wkb
new file mode 100644
index 00000000..e75c9684
Binary files /dev/null and b/planetiler-core/src/test/resources/issue_700/exception_3.wkb differ
diff --git a/planetiler-core/src/test/resources/issue_700/exception_4.wkb b/planetiler-core/src/test/resources/issue_700/exception_4.wkb
new file mode 100644
index 00000000..f16cc015
Binary files /dev/null and b/planetiler-core/src/test/resources/issue_700/exception_4.wkb differ
diff --git a/planetiler-core/src/test/resources/issue_700/exception_5.wkb b/planetiler-core/src/test/resources/issue_700/exception_5.wkb
new file mode 100644
index 00000000..e259e5f9
Binary files /dev/null and b/planetiler-core/src/test/resources/issue_700/exception_5.wkb differ
diff --git a/planetiler-core/src/test/resources/issue_700/exception_6.wkb b/planetiler-core/src/test/resources/issue_700/exception_6.wkb
new file mode 100644
index 00000000..9e14a471
Binary files /dev/null and b/planetiler-core/src/test/resources/issue_700/exception_6.wkb differ
diff --git a/planetiler-core/src/test/resources/issue_700/exception_7.wkb b/planetiler-core/src/test/resources/issue_700/exception_7.wkb
new file mode 100644
index 00000000..ecbbe0b9
Binary files /dev/null and b/planetiler-core/src/test/resources/issue_700/exception_7.wkb differ
diff --git a/planetiler-core/src/test/resources/issue_700/exception_8.wkb b/planetiler-core/src/test/resources/issue_700/exception_8.wkb
new file mode 100644
index 00000000..fcf0537b
Binary files /dev/null and b/planetiler-core/src/test/resources/issue_700/exception_8.wkb differ
diff --git a/planetiler-core/src/test/resources/issue_700/exception_9.wkb b/planetiler-core/src/test/resources/issue_700/exception_9.wkb
new file mode 100644
index 00000000..b66fca19
Binary files /dev/null and b/planetiler-core/src/test/resources/issue_700/exception_9.wkb differ
diff --git a/planetiler-core/src/test/resources/log4j2-test.properties b/planetiler-core/src/test/resources/log4j2-test.properties
index f5350b6c..3140f703 100644
--- a/planetiler-core/src/test/resources/log4j2-test.properties
+++ b/planetiler-core/src/test/resources/log4j2-test.properties
@@ -2,7 +2,7 @@ appenders=console
 appender.console.type=Console
 appender.console.name=STDOUT
 appender.console.layout.type=PatternLayout
-appender.console.layout.pattern=$${uptime:now} %level{length=3} %notEmpty{[%X{stage}] }- %msg%n%throwable
+appender.console.layout.pattern=$${uptime:now} %level{length=3} %X{stage}- %msg%n%throwable
 packages=com.onthegomap.planetiler.util.log4j
 rootLogger.level=warn
 rootLogger.appenderRefs=stdout
diff --git a/planetiler-custommap/README.md b/planetiler-custommap/README.md
index 170b7ed0..539140d8 100644
--- a/planetiler-custommap/README.md
+++ b/planetiler-custommap/README.md
@@ -292,15 +292,18 @@ Specific tile post processing operations for merging features may be defined:
 
 - `merge_line_strings` - Combines linestrings with the same set of attributes into a multilinestring where segments with
   touching endpoints are merged.
-- `merge_polygons` - Combines polygons with the same set of attributes into a multipolygon where overlapping/touching polygons
+- `merge_polygons` - Combines polygons with the same set of attributes into a multipolygon where overlapping/touching
+  polygons
   are combined into fewer polygons covering the same area.
 
 The follow attributes for `merge_line_strings` may be set:
+
 - `min_length` - Minimum tile pixel length of features to emit, or 0 to emit all merged linestrings.
 - `tolerance` - After merging, simplify linestrings using this pixel tolerance, or -1 to skip simplification step.
 - `buffer` - Number of pixels outside the visible tile area to include detail for, or -1 to skip clipping step.
 
 The follow attribute for `merge_polygons` may be set:
+
 - `min_area` - Minimum area in square tile pixels of polygons to emit.
 
 For example:
@@ -482,6 +485,11 @@ nested, so each child context can also access the variables from its parent.
 >> - `feature.id` - numeric ID of the input feature
 >> - `feature.source` - string source ID this feature came from
 >> - `feature.source_layer` - optional layer within the source the feature came from
+>> - `feature.osm_changeset` - optional OSM changeset ID for this feature
+>> - `feature.osm_version` - optional OSM element version for this feature
+>> - `feature.osm_timestamp` - optional OSM last modified timestamp for this feature
+>> - `feature.osm_user_id` - optional ID of the OSM user that last modified this feature
+>> - `feature.osm_user_name` - optional name of the OSM user that last modified this feature
 >>
 >>> ##### post-match context
 >>>
@@ -534,7 +542,7 @@ in [PlanetilerStdLib](src/main/java/com/onthegomap/planetiler/custommap/expressi
   - `.replace(from, to, limit)` returns the input string with the first N occurrences of from replaced by to
   - `.replaceRegex(pattern, value)` replaces every occurrence of regular expression with value from the string
     it was called on using java's
-    built-in [replaceAll]()
+    built-in [replaceAll]()
     behavior
   - `.split(separator)` returns a list of strings split from the input by a separator
   - `.split(separator, limit)` splits the list into up to N parts
diff --git a/planetiler-custommap/pom.xml b/planetiler-custommap/pom.xml
index 0336802f..40548cc7 100644
--- a/planetiler-custommap/pom.xml
+++ b/planetiler-custommap/pom.xml
@@ -45,7 +45,7 @@
       
         org.projectnessie.cel
         cel-bom
-        0.3.21
+        0.4.3
         pom
         import
       
diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfigExpressionParser.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfigExpressionParser.java
index beaf5bca..1137bfff 100644
--- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfigExpressionParser.java
+++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfigExpressionParser.java
@@ -86,7 +86,7 @@ public class ConfigExpressionParser {
         return cast(signature(output), child, dataType);
       } else {
         var keys = map.keySet();
-        if (keys.equals(Set.of("coalesce")) && map.get("coalesce")instanceof Collection cases) {
+        if (keys.equals(Set.of("coalesce")) && map.get("coalesce") instanceof Collection cases) {
           return coalesce(cases.stream().map(item -> parse(item, output)).toList());
         } else if (keys.equals(Set.of("match"))) {
           return parseMatch(map.get("match"), true, output);
diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java
index 57ca78c9..bd90ac8b 100644
--- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java
+++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java
@@ -13,6 +13,8 @@ import com.onthegomap.planetiler.expression.DataType;
 import com.onthegomap.planetiler.reader.SourceFeature;
 import com.onthegomap.planetiler.reader.WithGeometryType;
 import com.onthegomap.planetiler.reader.WithTags;
+import com.onthegomap.planetiler.reader.osm.OsmElement;
+import com.onthegomap.planetiler.reader.osm.OsmSourceFeature;
 import com.onthegomap.planetiler.util.Try;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
@@ -340,6 +342,11 @@ public class Contexts {
     private static final String FEATURE_ID = "feature.id";
     private static final String FEATURE_SOURCE = "feature.source";
     private static final String FEATURE_SOURCE_LAYER = "feature.source_layer";
+    private static final String FEATURE_OSM_CHANGESET = "feature.osm_changeset";
+    private static final String FEATURE_OSM_VERSION = "feature.osm_version";
+    private static final String FEATURE_OSM_TIMESTAMP = "feature.osm_timestamp";
+    private static final String FEATURE_OSM_USER_ID = "feature.osm_user_id";
+    private static final String FEATURE_OSM_USER_NAME = "feature.osm_user_name";
 
     public static ScriptEnvironment description(Root root) {
       return root.description()
@@ -348,7 +355,12 @@ public class Contexts {
           Decls.newVar(FEATURE_TAGS, Decls.newMapType(Decls.String, Decls.Any)),
           Decls.newVar(FEATURE_ID, Decls.Int),
           Decls.newVar(FEATURE_SOURCE, Decls.String),
-          Decls.newVar(FEATURE_SOURCE_LAYER, Decls.String)
+          Decls.newVar(FEATURE_SOURCE_LAYER, Decls.String),
+          Decls.newVar(FEATURE_OSM_CHANGESET, Decls.Int),
+          Decls.newVar(FEATURE_OSM_VERSION, Decls.Int),
+          Decls.newVar(FEATURE_OSM_TIMESTAMP, Decls.Int),
+          Decls.newVar(FEATURE_OSM_USER_ID, Decls.Int),
+          Decls.newVar(FEATURE_OSM_USER_NAME, Decls.String)
         );
     }
 
@@ -360,7 +372,17 @@ public class Contexts {
           case FEATURE_ID -> feature.id();
           case FEATURE_SOURCE -> feature.getSource();
           case FEATURE_SOURCE_LAYER -> wrapNullable(feature.getSourceLayer());
-          default -> null;
+          default -> {
+            OsmElement.Info info = feature instanceof OsmSourceFeature osm ? osm.originalElement().info() : null;
+            yield info == null ? null : switch (key) {
+              case FEATURE_OSM_CHANGESET -> info.changeset();
+              case FEATURE_OSM_VERSION -> info.version();
+              case FEATURE_OSM_TIMESTAMP -> info.timestamp();
+              case FEATURE_OSM_USER_ID -> info.userId();
+              case FEATURE_OSM_USER_NAME -> wrapNullable(info.user());
+              default -> null;
+            };
+          }
         };
       } else {
         return null;
@@ -410,7 +432,7 @@ public class Contexts {
     }
 
     public String matchKey() {
-      return matchKeys().isEmpty() ? null : matchKeys().get(0);
+      return matchKeys().isEmpty() ? null : matchKeys().getFirst();
     }
 
     public Object matchValue() {
diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/TypeConversion.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/TypeConversion.java
index 124a4cfd..0b9b45ee 100644
--- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/TypeConversion.java
+++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/TypeConversion.java
@@ -65,7 +65,7 @@ public class TypeConversion {
     return d % 1 == 0 ? Long.toString(d.longValue()) : d.toString();
   }
 
-  private record Converter (Class in, Class out, Function fn) implements Function {
+  private record Converter(Class in, Class out, Function fn) implements Function {
     @Override
     public O apply(Object in) {
       @SuppressWarnings("unchecked") I converted = (I) in;
diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/BooleanExpressionScript.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/BooleanExpressionScript.java
index fbc2b9ed..a8c5c566 100644
--- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/BooleanExpressionScript.java
+++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/BooleanExpressionScript.java
@@ -15,7 +15,7 @@ import java.util.Objects;
  *
  * @param             Type of the expression context
  */
-public record BooleanExpressionScript (
+public record BooleanExpressionScript(
   String expressionText,
   ConfigExpressionScript expression,
   Class inputClass
diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ConfigExpression.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ConfigExpression.java
index 1ccf4994..a6be2619 100644
--- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ConfigExpression.java
+++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ConfigExpression.java
@@ -70,7 +70,7 @@ public interface ConfigExpression
   }
 
   /** An expression that always returns {@code value}. */
-  record Const (O value) implements ConfigExpression {
+  record Const(O value) implements ConfigExpression {
 
     @Override
     public O apply(I i) {
@@ -79,7 +79,7 @@ public interface ConfigExpression
   }
 
   /** An expression that returns the value associated with the first matching boolean expression. */
-  record Match (
+  record Match(
     Signature signature,
     MultiExpression> multiExpression,
     ConfigExpression fallback,
@@ -146,7 +146,7 @@ public interface ConfigExpression
   }
 
   /** An expression that returns the first non-null result of evaluating each child expression. */
-  record Coalesce (List> children)
+  record Coalesce(List> children)
     implements ConfigExpression {
 
     @Override
@@ -164,7 +164,7 @@ public interface ConfigExpression
     public ConfigExpression simplifyOnce() {
       return switch (children.size()) {
         case 0 -> constOf(null);
-        case 1 -> children.get(0);
+        case 1 -> children.getFirst();
         default -> {
           var result = children.stream()
             .flatMap(
@@ -184,7 +184,7 @@ public interface ConfigExpression
   }
 
   /** An expression that returns the value associated a given variable name at runtime. */
-  record Variable (
+  record Variable(
     Signature signature,
     String name
   ) implements ConfigExpression {
@@ -202,7 +202,7 @@ public interface ConfigExpression
   }
 
   /** An expression that returns the value associated a given tag of the input feature at runtime. */
-  record GetTag (
+  record GetTag(
     Signature signature,
     ConfigExpression tag
   ) implements ConfigExpression {
@@ -219,7 +219,7 @@ public interface ConfigExpression
   }
 
   /** An expression that returns the value associated a given argument at runtime. */
-  record GetArg (
+  record GetArg(
     Signature signature,
     ConfigExpression arg
   ) implements ConfigExpression {
@@ -242,7 +242,7 @@ public interface ConfigExpression
   }
 
   /** An expression that converts the input to a desired output {@link DataType} at runtime. */
-  record Cast (
+  record Cast(
     Signature signature,
     ConfigExpression input,
     DataType output
@@ -268,7 +268,7 @@ public interface ConfigExpression
     }
   }
 
-  record Signature (ScriptEnvironment in, Class out) {
+  record Signature(ScriptEnvironment in, Class out) {
 
     public  Signature withOutput(Class newOut) {
       return new Signature<>(in, newOut);
diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ScriptEnvironment.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ScriptEnvironment.java
index 244a4122..db56016d 100644
--- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ScriptEnvironment.java
+++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ScriptEnvironment.java
@@ -12,7 +12,7 @@ import java.util.stream.Stream;
  * @param clazz        Class of the input context type
  * @param           The runtime expression context type
  */
-public record ScriptEnvironment (List declarations, Class clazz, Contexts.Root root) {
+public record ScriptEnvironment(List declarations, Class clazz, Contexts.Root root) {
   private static  List concat(List a, List b) {
     return Stream.concat(a.stream(), b.stream()).toList();
   }
diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/validator/SchemaValidator.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/validator/SchemaValidator.java
index 431bda0c..b112a189 100644
--- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/validator/SchemaValidator.java
+++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/validator/SchemaValidator.java
@@ -29,9 +29,9 @@ import java.util.Set;
 import java.util.TreeSet;
 import java.util.stream.Stream;
 import org.apache.commons.lang3.exception.ExceptionUtils;
-import org.geotools.geometry.jts.WKTReader2;
 import org.locationtech.jts.geom.Geometry;
 import org.locationtech.jts.io.ParseException;
+import org.locationtech.jts.io.WKTReader;
 import org.snakeyaml.engine.v2.exceptions.YamlEngineException;
 
 /** Verifies that a profile maps input elements map to expected output vector tile features. */
@@ -164,7 +164,7 @@ public class SchemaValidator {
       default -> geometry;
     };
     try {
-      return new WKTReader2().read(wkt);
+      return new WKTReader().read(wkt);
     } catch (ParseException e) {
       throw new IllegalArgumentException("""
         Bad geometry: "%s", must be "point" "line" "polygon" or a valid WKT string.
diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java
index 848d8339..f17d253f 100644
--- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java
+++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java
@@ -22,6 +22,7 @@ import com.onthegomap.planetiler.geo.GeoUtils;
 import com.onthegomap.planetiler.geo.GeometryException;
 import com.onthegomap.planetiler.reader.SimpleFeature;
 import com.onthegomap.planetiler.reader.SourceFeature;
+import com.onthegomap.planetiler.reader.osm.OsmElement;
 import com.onthegomap.planetiler.stats.Stats;
 import java.nio.file.Path;
 import java.util.List;
@@ -40,6 +41,7 @@ class ConfiguredFeatureTest {
   private static final Function TEST_RESOURCE = TestConfigurableUtils::pathToTestResource;
   private static final Function SAMPLE_RESOURCE = TestConfigurableUtils::pathToSample;
   private static final Function TEST_INVALID_RESOURCE = TestConfigurableUtils::pathToTestInvalidResource;
+  private static final OsmElement.Info OSM_INFO = new OsmElement.Info(2, 3, 4, 5, "user");
 
   private static final Map waterTags = Map.of(
     "natural", "water",
@@ -130,14 +132,15 @@ class ConfiguredFeatureTest {
   private void testPolygon(String config, Map tags,
     Consumer test, int expectedMatchCount) {
     var sf =
-      SimpleFeature.createFakeOsmFeature(newPolygon(0, 0, 1, 0, 1, 1, 0, 0), tags, "osm", null, 1, emptyList());
+      SimpleFeature.createFakeOsmFeature(newPolygon(0, 0, 1, 0, 1, 1, 0, 0), tags, "osm", null, 1, emptyList(),
+        OSM_INFO);
     testFeature(config, sf, test, expectedMatchCount);
   }
 
   private void testPoint(String config, Map tags,
     Consumer test, int expectedMatchCount) {
     var sf =
-      SimpleFeature.createFakeOsmFeature(newPoint(0, 0), tags, "osm", null, 1, emptyList());
+      SimpleFeature.createFakeOsmFeature(newPoint(0, 0), tags, "osm", null, 1, emptyList(), OSM_INFO);
     testFeature(config, sf, test, expectedMatchCount);
   }
 
@@ -145,21 +148,22 @@ class ConfiguredFeatureTest {
   private void testLinestring(String config,
     Map tags, Consumer test, int expectedMatchCount) {
     var sf =
-      SimpleFeature.createFakeOsmFeature(newLineString(0, 0, 1, 0, 1, 1), tags, "osm", null, 1, emptyList());
+      SimpleFeature.createFakeOsmFeature(newLineString(0, 0, 1, 0, 1, 1), tags, "osm", null, 1, emptyList(), OSM_INFO);
     testFeature(config, sf, test, expectedMatchCount);
   }
 
   private void testPolygon(Function pathFunction, String schemaFilename, Map tags,
     Consumer test, int expectedMatchCount) {
     var sf =
-      SimpleFeature.createFakeOsmFeature(newPolygon(0, 0, 1, 0, 1, 1, 0, 0), tags, "osm", null, 1, emptyList());
+      SimpleFeature.createFakeOsmFeature(newPolygon(0, 0, 1, 0, 1, 1, 0, 0), tags, "osm", null, 1, emptyList(),
+        OSM_INFO);
     testFeature(pathFunction, schemaFilename, sf, test, expectedMatchCount);
   }
 
   private void testLinestring(Function pathFunction, String schemaFilename,
     Map tags, Consumer test, int expectedMatchCount) {
     var sf =
-      SimpleFeature.createFakeOsmFeature(newLineString(0, 0, 1, 0, 1, 1), tags, "osm", null, 1, emptyList());
+      SimpleFeature.createFakeOsmFeature(newLineString(0, 0, 1, 0, 1, 1), tags, "osm", null, 1, emptyList(), OSM_INFO);
     testFeature(pathFunction, schemaFilename, sf, test, expectedMatchCount);
   }
 
@@ -547,6 +551,11 @@ class ConfiguredFeatureTest {
     "\\\\${feature.id}|\\${feature.id}",
     "${feature.source}|osm",
     "${feature.source_layer}|null",
+    "${feature.osm_changeset}|2",
+    "${feature.osm_timestamp}|3",
+    "${feature.osm_user_id}|4",
+    "${feature.osm_version}|5",
+    "${feature.osm_user_name}|user",
     "${coalesce(feature.source_layer, 'missing')}|missing",
     "{match: {test: {natural: water}}}|test",
     "{match: {test: {natural: not_water}}}|null",
diff --git a/planetiler-dist/pom.xml b/planetiler-dist/pom.xml
index e147dd7f..a5ab8839 100644
--- a/planetiler-dist/pom.xml
+++ b/planetiler-dist/pom.xml
@@ -14,6 +14,12 @@
 
   
     com.onthegomap.planetiler.Main
+    
+    17
+    17
     ${project.version}
     ghcr.io/onthegomap/planetiler:${image.version}
     package
@@ -54,7 +60,7 @@
           false
           
             
-              eclipse-temurin:17-jre
+              eclipse-temurin:21-jre
             
             
               
@@ -73,6 +79,8 @@
               
             
             ${mainClass}
+            ${maven.build.timestamp}
+            ${maven.build.timestamp}
           
         
       
diff --git a/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java b/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java
index 3b6217fe..0d9a1316 100644
--- a/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java
+++ b/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java
@@ -26,6 +26,15 @@ import org.openmaptiles.util.VerifyMonaco;
  * public static void main(String[] args)} methods of runnable classes.
  */
 public class Main {
+  static {
+    int version = Runtime.version().feature();
+    if (version < 21) {
+      System.err.println(
+        "You are using Java " + version +
+          " but Planetiler requires 21 or later, for more details on upgrading see: https://github.com/onthegomap/planetiler/blob/main/CONTRIBUTING.md");
+      System.exit(1);
+    }
+  }
 
   private static final EntryPoint DEFAULT_TASK = OpenMapTilesMain::main;
   private static final Map ENTRY_POINTS = Map.ofEntries(
diff --git a/planetiler-examples/README.md b/planetiler-examples/README.md
index c6e39225..e7faea39 100644
--- a/planetiler-examples/README.md
+++ b/planetiler-examples/README.md
@@ -4,7 +4,7 @@ This is a minimal example project that shows how to create custom maps with Plan
 
 Requirements:
 
-- Java 17+ (see [CONTIRBUTING.md](../CONTRIBUTING.md))
+- Java 21+ (see [CONTIRBUTING.md](../CONTRIBUTING.md))
   - on mac: `brew install --cask temurin`
 - [Maven](https://maven.apache.org/install.html)
   - on mac: `brew install maven`
diff --git a/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/BikeRouteOverlay.java b/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/BikeRouteOverlay.java
index 719b5629..46abb3b1 100644
--- a/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/BikeRouteOverlay.java
+++ b/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/BikeRouteOverlay.java
@@ -64,11 +64,11 @@ public class BikeRouteOverlay implements Profile {
           relation.getString("route"),
           // except map network abbreviation to a human-readable value
           switch (relation.getString("network", "")) {
-          case "icn" -> "international";
-          case "ncn" -> "national";
-          case "rcn" -> "regional";
-          case "lcn" -> "local";
-          default -> "other";
+            case "icn" -> "international";
+            case "ncn" -> "national";
+            case "rcn" -> "regional";
+            case "lcn" -> "local";
+            default -> "other";
           }
         ));
       }
diff --git a/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/OsmQaTiles.java b/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/OsmQaTiles.java
index 4b149034..72fe948e 100644
--- a/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/OsmQaTiles.java
+++ b/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/OsmQaTiles.java
@@ -76,10 +76,12 @@ public class OsmQaTiles implements Profile {
         }
         feature
           .setAttr("@id", sourceFeature.id())
-          .setAttr("@type", element instanceof OsmElement.Node ? "node" :
-            element instanceof OsmElement.Way ? "way" :
-            element instanceof OsmElement.Relation ? "relation" : null
-          );
+          .setAttr("@type", switch (element) {
+            case OsmElement.Node ignored -> "node";
+            case OsmElement.Way ignored -> "way";
+            case OsmElement.Relation ignored -> "relation";
+            default -> null;
+          });
         var info = element.info();
         if (info != null) {
           feature
diff --git a/planetiler-examples/standalone.pom.xml b/planetiler-examples/standalone.pom.xml
index d36771ad..49eb051e 100644
--- a/planetiler-examples/standalone.pom.xml
+++ b/planetiler-examples/standalone.pom.xml
@@ -9,10 +9,10 @@
 
   
     UTF-8
-    17
-    17
+    21
+    21
     0.7-SNAPSHOT
-    5.10.0
+    5.10.1
     
     com.onthegomap.planetiler.examples.BikeRouteOverlay
   
@@ -78,12 +78,12 @@
       
         org.apache.maven.plugins
         maven-surefire-plugin
-        3.1.2
+        3.2.3
       
       
         org.apache.maven.plugins
         maven-failsafe-plugin
-        3.1.2
+        3.2.3
       
 
       
diff --git a/planetiler-openmaptiles b/planetiler-openmaptiles
index 613d0a8f..7dbbc508 160000
--- a/planetiler-openmaptiles
+++ b/planetiler-openmaptiles
@@ -1 +1 @@
-Subproject commit 613d0a8f7eba34ef75ac202538843f78b0e28284
+Subproject commit 7dbbc5089e8ca5607d9c3ecf1991d35b1de5c6cf
diff --git a/pom.xml b/pom.xml
index 840d61ca..50da0dbc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -17,11 +17,11 @@
 
   
     UTF-8
-    17
-    17
+    21
+    21
     true
-    2.15.2
-    5.10.0
+    2.16.0
+    5.10.1
     https://sonarcloud.io
     onthegomap
     onthegomap_planetiler
@@ -125,7 +125,7 @@
       
         org.mockito
         mockito-core
-        5.6.0
+        5.8.0
         test
       
     
@@ -172,13 +172,19 @@
       
         com.diffplug.spotless
         spotless-maven-plugin
-        2.40.0
+        2.41.1
         
           
+            
+              *.java
+            
+            
+              planetiler-openmaptiles/**/*.java
+            
             
             
             
-              4.21.0
+              4.29
               
               ${maven.multiModuleProjectDirectory}/eclipse-formatter.xml
             
@@ -203,7 +209,7 @@
           
         
       
-      
+      
       
         org.apache.maven.plugins
         maven-enforcer-plugin
@@ -217,7 +223,7 @@
             
               
                 
-                  17
+                  21
                 
               
             
@@ -227,7 +233,7 @@
       
         org.apache.maven.plugins
         maven-surefire-plugin
-        3.1.2
+        3.2.3
         
         
           
@@ -238,13 +244,23 @@
       
         org.apache.maven.plugins
         maven-failsafe-plugin
-        3.1.2
+        3.2.2
       
       
         org.apache.maven.plugins
         maven-deploy-plugin
         3.1.1
       
+      
+        org.apache.maven.plugins
+        maven-compiler-plugin
+        3.11.0
+        
+          
+            -proc:full
+          
+        
+      
     
 
     
@@ -276,7 +292,7 @@
         
           org.apache.maven.plugins
           maven-javadoc-plugin
-          3.6.0
+          3.6.3
           
             true
             -missing
@@ -316,7 +332,7 @@
         
           org.jacoco
           jacoco-maven-plugin
-          0.8.10
+          0.8.11