/* **************************************************************** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. ****************************************************************/ package com.onthegomap.planetiler; import com.carrotsearch.hppc.IntArrayList; import com.google.common.primitives.Ints; import com.google.protobuf.InvalidProtocolBufferException; import com.onthegomap.planetiler.collection.FeatureGroup; import com.onthegomap.planetiler.geo.GeoUtils; import com.onthegomap.planetiler.geo.GeometryException; import com.onthegomap.planetiler.geo.GeometryType; import com.onthegomap.planetiler.geo.MutableCoordinateSequence; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.concurrent.NotThreadSafe; import org.locationtech.jts.algorithm.Orientation; import org.locationtech.jts.geom.CoordinateSequence; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.geom.LineString; import org.locationtech.jts.geom.LinearRing; import org.locationtech.jts.geom.MultiLineString; import org.locationtech.jts.geom.MultiPoint; import org.locationtech.jts.geom.MultiPolygon; import org.locationtech.jts.geom.Point; import org.locationtech.jts.geom.Polygon; import org.locationtech.jts.geom.Puntal; import org.locationtech.jts.geom.impl.CoordinateArraySequence; import org.locationtech.jts.geom.impl.PackedCoordinateSequence; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import vector_tile.VectorTileProto; /** * Encodes a single output tile containing JTS {@link Geometry} features into the compact binary Mapbox Vector Tile * format. *

* This class is copied from VectorTileEncoder.java * and VectorTileDecoder.java * and modified to decouple geometry encoding from vector tile encoding so that encoded commands can be stored in the * sorted feature map prior to encoding vector tiles. The internals are also refactored to improve performance by using * hppc primitive collections. * * @see Mapbox Vector Tile Specification */ @NotThreadSafe public class VectorTile { private static final Logger LOGGER = LoggerFactory.getLogger(VectorTile.class); // TODO make these configurable private static final int EXTENT = 4096; private static final double SIZE = 256d; private final Map layers = new LinkedHashMap<>(); private static int[] getCommands(Geometry input, int scale) { var encoder = new CommandEncoder(scale); encoder.accept(input); return encoder.result.toArray(); } /** * Scales a geometry down by a factor of {@code 2^scale} without materializing an intermediate JTS geometry and * returns the encoded result. */ private static int[] unscale(int[] commands, int scale, GeometryType geomType) { IntArrayList result = new IntArrayList(); int geometryCount = commands.length; int length = 0; int command = 0; int i = 0; int inX = 0, inY = 0; int outX = 0, outY = 0; int startX = 0, startY = 0; double scaleFactor = Math.pow(2, -scale); int lengthIdx = 0; int moveToIdx = 0; int pointsInShape = 0; boolean first = true; while (i < geometryCount) { if (length <= 0) { length = commands[i++]; lengthIdx = result.size(); result.add(length); command = length & ((1 << 3) - 1); length = length >> 3; } if (length > 0) { if (command == Command.MOVE_TO.value) { // degenerate geometry, remove it from output entirely if (!first && pointsInShape < geomType.minPoints()) { int prevCommand = result.get(lengthIdx); result.elementsCount = moveToIdx; result.add(prevCommand); // reset deltas outX = startX; outY = startY; } // keep track of size of next shape... pointsInShape = 0; startX = outX; startY = outY; moveToIdx = result.size() - 1; } first = false; if (command == Command.CLOSE_PATH.value) { pointsInShape++; length--; continue; } int dx = commands[i++]; int dy = commands[i++]; length--; dx = zigZagDecode(dx); dy = zigZagDecode(dy); inX = inX + dx; inY = inY + dy; int nextX = (int) Math.round(inX * scaleFactor); int nextY = (int) Math.round(inY * scaleFactor); if (nextX == outX && nextY == outY && command == Command.LINE_TO.value) { int commandLength = result.get(lengthIdx) - 8; if (commandLength < 8) { // get rid of lineto section if empty result.elementsCount = lengthIdx; } else { result.set(lengthIdx, commandLength); } } else { pointsInShape++; int dxOut = nextX - outX; int dyOut = nextY - outY; result.add( zigZagEncode(dxOut), zigZagEncode(dyOut) ); outX = nextX; outY = nextY; } } } // degenerate geometry, remove it from output entirely if (pointsInShape < geomType.minPoints()) { result.elementsCount = moveToIdx; } return result.toArray(); } private static int zigZagEncode(int n) { // https://developers.google.com/protocol-buffers/docs/encoding#types return (n << 1) ^ (n >> 31); } private static int zigZagDecode(int n) { // https://developers.google.com/protocol-buffers/docs/encoding#types return ((n >> 1) ^ (-(n & 1))); } private static Geometry decodeCommands(GeometryType geomType, int[] commands, int scale) throws GeometryException { try { GeometryFactory gf = GeoUtils.JTS_FACTORY; double SCALE = (EXTENT << scale) / SIZE; int x = 0; int y = 0; List allCoordSeqs = new ArrayList<>(); MutableCoordinateSequence currentCoordSeq = null; int geometryCount = commands.length; int length = 0; int command = 0; int i = 0; while (i < geometryCount) { if (length <= 0) { length = commands[i++]; command = length & ((1 << 3) - 1); length = length >> 3; } if (length > 0) { if (command == Command.MOVE_TO.value) { currentCoordSeq = new MutableCoordinateSequence(); allCoordSeqs.add(currentCoordSeq); } else { assert currentCoordSeq != null; } if (command == Command.CLOSE_PATH.value) { if (geomType != GeometryType.POINT && !currentCoordSeq.isEmpty()) { currentCoordSeq.closeRing(); } length--; continue; } int dx = commands[i++]; int dy = commands[i++]; length--; dx = zigZagDecode(dx); dy = zigZagDecode(dy); x = x + dx; y = y + dy; currentCoordSeq.forceAddPoint(x / SCALE, y / SCALE); } } Geometry geometry = null; boolean outerCCW = false; switch (geomType) { case LINE -> { List lineStrings = new ArrayList<>(allCoordSeqs.size()); for (MutableCoordinateSequence coordSeq : allCoordSeqs) { if (coordSeq.size() <= 1) { continue; } lineStrings.add(gf.createLineString(coordSeq)); } if (lineStrings.size() == 1) { geometry = lineStrings.get(0); } else if (lineStrings.size() > 1) { geometry = gf.createMultiLineString(lineStrings.toArray(new LineString[0])); } } case POINT -> { CoordinateSequence cs = new PackedCoordinateSequence.Double(allCoordSeqs.size(), 2, 0); for (int j = 0; j < allCoordSeqs.size(); j++) { MutableCoordinateSequence coordSeq = allCoordSeqs.get(j); cs.setOrdinate(j, 0, coordSeq.getX(0)); cs.setOrdinate(j, 1, coordSeq.getY(0)); } if (cs.size() == 1) { geometry = gf.createPoint(cs); } else if (cs.size() > 1) { geometry = gf.createMultiPoint(cs); } } case POLYGON -> { List> polygonRings = new ArrayList<>(); List ringsForCurrentPolygon = new ArrayList<>(); boolean first = true; for (MutableCoordinateSequence coordSeq : allCoordSeqs) { // skip hole with too few coordinates if (ringsForCurrentPolygon.size() > 0 && coordSeq.size() < 2) { continue; } LinearRing ring = gf.createLinearRing(coordSeq); boolean ccw = Orientation.isCCW(coordSeq); if (first) { first = false; outerCCW = ccw; assert outerCCW : "outer ring is not counter-clockwise"; } if (ccw == outerCCW) { ringsForCurrentPolygon = new ArrayList<>(); polygonRings.add(ringsForCurrentPolygon); } ringsForCurrentPolygon.add(ring); } List polygons = new ArrayList<>(); for (List rings : polygonRings) { LinearRing shell = rings.get(0); LinearRing[] holes = rings.subList(1, rings.size()).toArray(new LinearRing[rings.size() - 1]); polygons.add(gf.createPolygon(shell, holes)); } if (polygons.size() == 1) { geometry = polygons.get(0); } if (polygons.size() > 1) { geometry = gf.createMultiPolygon(GeometryFactory.toPolygonArray(polygons)); } } default -> { } } if (geometry == null) { geometry = GeoUtils.EMPTY_GEOMETRY; } return geometry; } catch (IllegalArgumentException e) { throw new GeometryException("decode_vector_tile", "Unable to decode geometry", e); } } /** * Parses a binary-encoded vector tile protobuf into a list of features. *

* Does not decode geometries, but clients can call {@link VectorGeometry#decode()} to decode a JTS {@link Geometry} * if needed. *

* If {@code encoded} is compressed, clients must decompress it first. * * @param encoded encoded vector tile protobuf * @return list of features on that tile * @throws IllegalStateException if decoding fails * @throws IndexOutOfBoundsException if a tag's key or value refers to an index that does not exist in the keys/values * array for a layer */ public static List decode(byte[] encoded) { try { VectorTileProto.Tile tile = VectorTileProto.Tile.parseFrom(encoded); List features = new ArrayList<>(); for (VectorTileProto.Tile.Layer layer : tile.getLayersList()) { String layerName = layer.getName(); assert layer.getExtent() == 4096; List keys = layer.getKeysList(); List values = new ArrayList<>(); for (VectorTileProto.Tile.Value value : layer.getValuesList()) { if (value.hasBoolValue()) { values.add(value.getBoolValue()); } else if (value.hasDoubleValue()) { values.add(value.getDoubleValue()); } else if (value.hasFloatValue()) { values.add(value.getFloatValue()); } else if (value.hasIntValue()) { values.add(value.getIntValue()); } else if (value.hasSintValue()) { values.add(value.getSintValue()); } else if (value.hasUintValue()) { values.add(value.getUintValue()); } else if (value.hasStringValue()) { values.add(value.getStringValue()); } else { values.add(null); } } for (VectorTileProto.Tile.Feature feature : layer.getFeaturesList()) { int tagsCount = feature.getTagsCount(); Map attrs = new HashMap<>(tagsCount / 2); int tagIdx = 0; while (tagIdx < feature.getTagsCount()) { String key = keys.get(feature.getTags(tagIdx++)); Object value = values.get(feature.getTags(tagIdx++)); attrs.put(key, value); } features.add(new Feature( layerName, feature.getId(), new VectorGeometry(Ints.toArray(feature.getGeometryList()), GeometryType.valueOf(feature.getType()), 0), attrs )); } } return features; } catch (InvalidProtocolBufferException e) { throw new IllegalStateException(e); } } /** * Encodes a JTS geometry according to * Geometry Encoding * Specification. * * @param geometry the JTS geometry to encoded * @return the geometry type and command array for the encoded geometry */ public static VectorGeometry encodeGeometry(Geometry geometry) { return encodeGeometry(geometry, 0); } public static VectorGeometry encodeGeometry(Geometry geometry, int scale) { return new VectorGeometry(getCommands(geometry, scale), GeometryType.valueOf(geometry), scale); } /** * Adds features in a layer to this tile. * * @param layerName name of the layer in this tile to add the features to * @param features features to add to the tile * @return this encoder for chaining */ public VectorTile addLayerFeatures(String layerName, List features) { if (features.isEmpty()) { return this; } Layer layer = layers.get(layerName); if (layer == null) { layer = new Layer(); layers.put(layerName, layer); } for (Feature inFeature : features) { if (inFeature != null && inFeature.geometry().commands().length > 0) { EncodedFeature outFeature = new EncodedFeature(inFeature); for (Map.Entry e : inFeature.attrs().entrySet()) { // skip attribute without value if (e.getValue() != null) { outFeature.tags.add(layer.key(e.getKey())); outFeature.tags.add(layer.value(e.getValue())); } } layer.encodedFeatures.add(outFeature); } } return this; } /** * Creates a vector tile protobuf with all features in this tile and serializes it as a byte array. *

* Does not compress the result. */ public byte[] encode() { VectorTileProto.Tile.Builder tile = VectorTileProto.Tile.newBuilder(); for (Map.Entry e : layers.entrySet()) { String layerName = e.getKey(); Layer layer = e.getValue(); VectorTileProto.Tile.Layer.Builder tileLayer = VectorTileProto.Tile.Layer.newBuilder() .setVersion(2) .setName(layerName) .setExtent(EXTENT) .addAllKeys(layer.keys()); for (Object value : layer.values()) { VectorTileProto.Tile.Value.Builder tileValue = VectorTileProto.Tile.Value.newBuilder(); if (value instanceof String stringValue) { tileValue.setStringValue(stringValue); } else if (value instanceof Integer intValue) { tileValue.setSintValue(intValue); } else if (value instanceof Long longValue) { tileValue.setSintValue(longValue); } else if (value instanceof Float floatValue) { tileValue.setFloatValue(floatValue); } else if (value instanceof Double doubleValue) { tileValue.setDoubleValue(doubleValue); } else if (value instanceof Boolean booleanValue) { tileValue.setBoolValue(booleanValue); } else { tileValue.setStringValue(value.toString()); } tileLayer.addValues(tileValue.build()); } for (EncodedFeature feature : layer.encodedFeatures) { VectorTileProto.Tile.Feature.Builder featureBuilder = VectorTileProto.Tile.Feature.newBuilder() .addAllTags(Ints.asList(feature.tags.toArray())) .setType(feature.geometry().geomType().asProtobufType()) .addAllGeometry(Ints.asList(feature.geometry().commands())); if (feature.id >= 0) { featureBuilder.setId(feature.id); } tileLayer.addFeatures(featureBuilder.build()); } tile.addLayers(tileLayer.build()); } return tile.build().toByteArray(); } private enum Command { MOVE_TO(1), LINE_TO(2), CLOSE_PATH(7); final int value; Command(int value) { this.value = value; } } /** * A vector tile encoded as a list of commands according to the * vector tile * specification. *

* To encode extra precision in intermediate feature geometries, the geometry contained in {@code commands} is scaled * to a tile extent of {@code EXTENT * 2^scale}, so when the {@code scale == 0} the extent is {@link #EXTENT} and when * {@code scale == 2} the extent is 4x{@link #EXTENT}. Geometries must be scaled back to 0 using {@link #unscale()} * before outputting to mbtiles. */ public record VectorGeometry(int[] commands, GeometryType geomType, int scale) { public VectorGeometry { if (scale < 0) { throw new IllegalArgumentException("scale can not be less than 0, got: " + scale); } } /** Converts an encoded geometry back to a JTS geometry. */ public Geometry decode() throws GeometryException { return decodeCommands(geomType, commands, scale); } /** Returns this encoded geometry, scaled back to 0, so it is safe to emit to mbtiles output. */ public VectorGeometry unscale() { return scale == 0 ? this : new VectorGeometry(VectorTile.unscale(commands, scale, geomType), geomType, 0); } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } VectorGeometry that = (VectorGeometry) o; if (geomType != that.geomType) { return false; } return Arrays.equals(commands, that.commands); } @Override public int hashCode() { int result = Arrays.hashCode(commands); result = 31 * result + geomType.hashCode(); return result; } @Override public String toString() { return "VectorGeometry[" + "commands=int[" + commands.length + "], geomType=" + geomType + " (" + geomType.asByte() + ")]"; } } /** * A feature in a vector tile. * * @param layer the layer the feature was in * @param id the feature ID * @param geometry the encoded feature geometry (decode using {@link VectorGeometry#decode()}) * @param attrs tags for the feature to output * @param group grouping key used to limit point density or {@link #NO_GROUP} if not in a group. NOTE: this is only * populated when this feature was deserialized from {@link FeatureGroup}, not when parsed from a tile * since vector tile schema does not encode group. */ public record Feature( String layer, long id, VectorGeometry geometry, Map attrs, long group ) { public static final long NO_GROUP = Long.MIN_VALUE; public Feature( String layer, long id, VectorGeometry geometry, Map attrs ) { this(layer, id, geometry, attrs, NO_GROUP); } public boolean hasGroup() { return group != NO_GROUP; } /** * Encodes {@code newGeometry} and returns a copy of this feature with {@code geometry} replaced with the encoded * new geometry. */ public Feature copyWithNewGeometry(Geometry newGeometry) { return copyWithNewGeometry(encodeGeometry(newGeometry)); } /** * Returns a copy of this feature with {@code geometry} replaced with {@code newGeometry}. */ public Feature copyWithNewGeometry(VectorGeometry newGeometry) { return new Feature( layer, id, newGeometry, attrs, group ); } /** Returns a copy of this feature with {@code extraAttrs} added to {@code attrs}. */ public Feature copyWithExtraAttrs(Map extraAttrs) { return new Feature( layer, id, geometry, Stream.concat(attrs.entrySet().stream(), extraAttrs.entrySet().stream()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)), group ); } } /** * Encodes a geometry as a sequence of integers according to the * Geometry * Encoding * Specification. */ private static class CommandEncoder { final IntArrayList result = new IntArrayList(); private final double SCALE; // Initial points use absolute locations, then subsequent points in a geometry use offsets so // need to keep track of previous x/y location during the encoding. int x = 0, y = 0; CommandEncoder(int scale) { this.SCALE = (EXTENT << scale) / SIZE; } static boolean shouldClosePath(Geometry geometry) { return (geometry instanceof Polygon) || (geometry instanceof LinearRing); } static int commandAndLength(Command command, int repeat) { return repeat << 3 | command.value; } void accept(Geometry geometry) { if (geometry instanceof MultiLineString multiLineString) { for (int i = 0; i < multiLineString.getNumGeometries(); i++) { encode(((LineString) multiLineString.getGeometryN(i)).getCoordinateSequence(), false, GeometryType.LINE); } } else if (geometry instanceof Polygon polygon) { LineString exteriorRing = polygon.getExteriorRing(); encode(exteriorRing.getCoordinateSequence(), true, GeometryType.POLYGON); for (int i = 0; i < polygon.getNumInteriorRing(); i++) { LineString interiorRing = polygon.getInteriorRingN(i); encode(interiorRing.getCoordinateSequence(), true, GeometryType.LINE); } } else if (geometry instanceof MultiPolygon multiPolygon) { for (int i = 0; i < multiPolygon.getNumGeometries(); i++) { accept(multiPolygon.getGeometryN(i)); } } else if (geometry instanceof LineString lineString) { encode(lineString.getCoordinateSequence(), shouldClosePath(geometry), GeometryType.LINE); } else if (geometry instanceof Point point) { encode(point.getCoordinateSequence(), false, GeometryType.POINT); } else if (geometry instanceof Puntal) { encode(new CoordinateArraySequence(geometry.getCoordinates()), shouldClosePath(geometry), geometry instanceof MultiPoint, GeometryType.POINT); } else { LOGGER.warn("Unrecognized geometry type: " + geometry.getGeometryType()); } } void encode(CoordinateSequence cs, boolean closePathAtEnd, GeometryType geomType) { encode(cs, closePathAtEnd, false, geomType); } void encode(CoordinateSequence cs, boolean closePathAtEnd, boolean multiPoint, GeometryType geomType) { if (cs.size() == 0) { throw new IllegalArgumentException("empty geometry"); } int startIdx = result.size(); int numPoints = 0; int lineToIndex = 0; int lineToLength = 0; int startX = x; int startY = y; for (int i = 0; i < cs.size(); i++) { double cx = cs.getX(i); double cy = cs.getY(i); if (i == 0) { result.add(commandAndLength(Command.MOVE_TO, multiPoint ? cs.size() : 1)); } int _x = (int) Math.round(cx * SCALE); int _y = (int) Math.round(cy * SCALE); // prevent point equal to the previous if (i > 0 && _x == x && _y == y && !multiPoint) { lineToLength--; continue; } // prevent double closing if (closePathAtEnd && cs.size() > 1 && i == (cs.size() - 1) && cs.getX(0) == cx && cs.getY(0) == cy) { lineToLength--; continue; } // delta, then zigzag result.add(zigZagEncode(_x - x)); result.add(zigZagEncode(_y - y)); numPoints++; x = _x; y = _y; if (i == 0 && cs.size() > 1 && !multiPoint) { // can length be too long? lineToIndex = result.size(); lineToLength = cs.size() - 1; result.add(commandAndLength(Command.LINE_TO, lineToLength)); } } // update LineTo length if (lineToIndex > 0) { if (lineToLength == 0) { // remove empty LineTo result.remove(lineToIndex); } else { // update LineTo with new length result.set(lineToIndex, commandAndLength(Command.LINE_TO, lineToLength)); } } if (closePathAtEnd) { result.add(commandAndLength(Command.CLOSE_PATH, 1)); numPoints++; } // degenerate geometry, skip emitting if (numPoints < geomType.minPoints()) { result.elementsCount = startIdx; // reset deltas x = startX; y = startY; } } } private record EncodedFeature(IntArrayList tags, long id, VectorGeometry geometry) { EncodedFeature(Feature in) { this(new IntArrayList(), in.id(), in.geometry()); } } /** * Holds all features in an output layer of this tile, along with the index of each tag key/value so that features can * store each key/value as a pair of integers. */ private static final class Layer { final List encodedFeatures = new ArrayList<>(); final Map keys = new LinkedHashMap<>(); final Map values = new LinkedHashMap<>(); List keys() { return new ArrayList<>(keys.keySet()); } List values() { return new ArrayList<>(values.keySet()); } /** Returns the ID associated with {@code key} or adds a new one if not present. */ Integer key(String key) { Integer i = keys.get(key); if (i == null) { i = keys.size(); keys.put(key, i); } return i; } /** Returns the ID associated with {@code value} or adds a new one if not present. */ Integer value(Object value) { Integer i = values.get(value); if (i == null) { i = values.size(); values.put(value, i); } return i; } @Override public String toString() { return "Layer{" + encodedFeatures.size() + "}"; } } }