vector tile encoder

scripts/generate-protobuf.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -ex
+
+protoc --java_out=src/main/java/ src/main/resources/vector_tile.proto

src/main/java/com/onthegomap/flatmap/Arguments.java

@@ -1,5 +1,6 @@
 package com.onthegomap.flatmap;
 
+import com.onthegomap.flatmap.geo.GeoUtils;
 import com.onthegomap.flatmap.monitoring.Stats;
 import java.io.File;
 import java.time.Duration;

src/main/java/com/onthegomap/flatmap/LayerFeature.java

@@ -0,0 +1,15 @@
+package com.onthegomap.flatmap;
+
+import java.util.Map;
+
+public record LayerFeature(
+  boolean hasGroup,
+  long group,
+  int zorder,
+  Map<String, Object> attrs,
+  byte geomType,
+  int[] commands,
+  long id
+) {
+
+}

src/main/java/com/onthegomap/flatmap/MbtilesWriter.java

@@ -1,11 +1,8 @@
 package com.onthegomap.flatmap;
 
-import static com.onthegomap.flatmap.GeoUtils.x;
-import static com.onthegomap.flatmap.GeoUtils.y;
-import static com.onthegomap.flatmap.GeoUtils.z;
-
 import com.onthegomap.flatmap.collections.MergeSortFeatureMap;
 import com.onthegomap.flatmap.collections.MergeSortFeatureMap.TileFeatures;
+import com.onthegomap.flatmap.geo.TileCoord;
 import com.onthegomap.flatmap.monitoring.ProgressLoggers;
 import com.onthegomap.flatmap.monitoring.Stats;
 import com.onthegomap.flatmap.worker.Topology;
@@ -32,7 +29,7 @@ public class MbtilesWriter {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(MbtilesWriter.class);
 
-  private static record RenderedTile(int tile, byte[] contents) {
+  private static record RenderedTile(TileCoord tile, byte[] contents) {
 
   }
 
@@ -65,25 +62,23 @@ public class MbtilesWriter {
     while ((tileFeatures = prev.get()) != null) {
       featuresProcessed.addAndGet(tileFeatures.getNumFeatures());
       byte[] bytes, encoded;
-      int zoom = z(tileFeatures.getTileId());
       if (tileFeatures.hasSameContents(last)) {
         bytes = lastBytes;
         encoded = lastEncoded;
         memoizedTiles.incrementAndGet();
       } else {
-        VectorTile en = tileFeatures.getTile();
+        VectorTileEncoder en = tileFeatures.getTile();
         encoded = en.encode();
         bytes = gzipCompress(encoded);
         last = tileFeatures;
         lastEncoded = encoded;
         lastBytes = bytes;
         if (encoded.length > 1_000_000) {
-          LOGGER.warn("Tile " + zoom + "/" + x(tileFeatures.getTileId()) + "/" + y(tileFeatures.getTileId()) + " "
-            + encoded.length / 1024 + "kb uncompressed");
+          LOGGER.warn(tileFeatures.coord() + " " + encoded.length / 1024 + "kb uncompressed");
         }
       }
-      stats.encodedTile(zoom, encoded.length);
-      next.accept(new RenderedTile(tileFeatures.getTileId(), bytes));
+      stats.encodedTile(tileFeatures.coord().z(), encoded.length);
+      next.accept(new RenderedTile(tileFeatures.coord(), bytes));
     }
   }
 

src/main/java/com/onthegomap/flatmap/RenderedFeature.java

@@ -1,5 +1,5 @@
 package com.onthegomap.flatmap;
 
-public class RenderedFeature {
+public record RenderedFeature(long sort, byte[] value) {
 
 }

src/main/java/com/onthegomap/flatmap/VectorTile.java

@@ -1,8 +0,0 @@
-package com.onthegomap.flatmap;
-
-public class VectorTile {
-
-  public byte[] encode() {
-    return new byte[]{};
-  }
-}

src/main/java/com/onthegomap/flatmap/VectorTileEncoder.java

@@ -0,0 +1,538 @@
+/*****************************************************************
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing,
+ *  software distributed under the License is distributed on an
+ *  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  KIND, either express or implied.  See the License for the
+ *  specific language governing permissions and limitations
+ *  under the License.
+ ****************************************************************/
+package com.onthegomap.flatmap;
+
+import com.carrotsearch.hppc.DoubleArrayList;
+import com.carrotsearch.hppc.IntArrayList;
+import com.google.common.primitives.Ints;
+import com.onthegomap.flatmap.geo.GeoUtils;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import org.locationtech.jts.algorithm.Orientation;
+import org.locationtech.jts.geom.CoordinateSequence;
+import org.locationtech.jts.geom.Geometry;
+import org.locationtech.jts.geom.GeometryFactory;
+import org.locationtech.jts.geom.LineString;
+import org.locationtech.jts.geom.LinearRing;
+import org.locationtech.jts.geom.MultiLineString;
+import org.locationtech.jts.geom.MultiPoint;
+import org.locationtech.jts.geom.MultiPolygon;
+import org.locationtech.jts.geom.Point;
+import org.locationtech.jts.geom.Polygon;
+import org.locationtech.jts.geom.impl.CoordinateArraySequence;
+import org.locationtech.jts.geom.impl.PackedCoordinateSequence;
+import vector_tile.VectorTile;
+import vector_tile.VectorTile.Tile.GeomType;
+
+/**
+ * This class is copied from https://github.com/ElectronicChartCentre/java-vector-tile/blob/master/src/main/java/no/ecc/vectortile/VectorTileEncoder.java
+ * and https://github.com/ElectronicChartCentre/java-vector-tile/blob/master/src/main/java/no/ecc/vectortile/VectorTileDecoder.java
+ * and modified.
+ * <p>
+ * The modifications decouple geometry encoding from vector tile encoding so that encoded commands can be stored in the
+ * sorted feature map prior to encoding vector tiles.  The internals are also refactored to improve performance by using
+ * hppc primitive collections.
+ */
+public class VectorTileEncoder {
+
+  private static final int EXTENT = 4096;
+  private static final double SIZE = 256d;
+  private static final double SCALE = ((double) EXTENT) / SIZE;
+  private final Map<String, Layer> layers = new LinkedHashMap<>();
+
+  public static int[] getCommands(Geometry input) {
+    var encoder = new CommandEncoder();
+    encoder.accept(input);
+    return encoder.result.toArray();
+  }
+
+  public static VectorTile.Tile.GeomType toGeomType(Geometry geometry) {
+    if (geometry instanceof Point || geometry instanceof MultiPoint) {
+      return VectorTile.Tile.GeomType.POINT;
+    } else if (geometry instanceof LineString || geometry instanceof MultiLineString) {
+      return VectorTile.Tile.GeomType.LINESTRING;
+    } else if (geometry instanceof Polygon || geometry instanceof MultiPolygon) {
+      return VectorTile.Tile.GeomType.POLYGON;
+    }
+    return VectorTile.Tile.GeomType.UNKNOWN;
+  }
+
+  private static CoordinateSequence toCs(DoubleArrayList seq) {
+    return new PackedCoordinateSequence.Double(seq.toArray(), 2, 0);
+  }
+
+  private static int zigZagEncode(int n) {
+    // https://developers.google.com/protocol-buffers/docs/encoding#types
+    return (n << 1) ^ (n >> 31);
+  }
+
+  private static int zigZagDecode(int n) {
+    // https://developers.google.com/protocol-buffers/docs/encoding#types
+    return ((n >> 1) ^ (-(n & 1)));
+  }
+
+  public static Geometry decode(byte geomTypeByte, int[] commands) {
+    VectorTile.Tile.GeomType geomType = Objects.requireNonNull(VectorTile.Tile.GeomType.forNumber(geomTypeByte));
+    GeometryFactory gf = GeoUtils.gf;
+    int x = 0;
+    int y = 0;
+
+    List<DoubleArrayList> coordsList = new ArrayList<>();
+    DoubleArrayList coords = null;
+
+    int geometryCount = commands.length;
+    int length = 0;
+    int command = 0;
+    int i = 0;
+    while (i < geometryCount) {
+
+      if (length <= 0) {
+        length = commands[i++];
+        command = length & ((1 << 3) - 1);
+        length = length >> 3;
+      }
+
+      if (length > 0) {
+
+        if (command == Command.MOVE_TO.value) {
+          coords = new DoubleArrayList();
+          coordsList.add(coords);
+        } else {
+          Objects.requireNonNull(coords);
+        }
+
+        if (command == Command.CLOSE_PATH.value) {
+          if (geomType != VectorTile.Tile.GeomType.POINT && !coords.isEmpty()) {
+            coords.add(coords.get(0), coords.get(1));
+          }
+          length--;
+          continue;
+        }
+
+        int dx = commands[i++];
+        int dy = commands[i++];
+
+        length--;
+
+        dx = zigZagDecode(dx);
+        dy = zigZagDecode(dy);
+
+        x = x + dx;
+        y = y + dy;
+
+        coords.add(x / SCALE, y / SCALE);
+      }
+
+    }
+
+    Geometry geometry = null;
+    boolean outerCCW = false;
+
+    switch (geomType) {
+      case LINESTRING:
+        List<LineString> lineStrings = new ArrayList<>(coordsList.size());
+        for (DoubleArrayList cs : coordsList) {
+          if (cs.size() <= 2) {
+            continue;
+          }
+          lineStrings.add(gf.createLineString(toCs(cs)));
+        }
+        if (lineStrings.size() == 1) {
+          geometry = lineStrings.get(0);
+        } else if (lineStrings.size() > 1) {
+          geometry = gf.createMultiLineString(lineStrings.toArray(new LineString[0]));
+        }
+        break;
+      case POINT:
+        CoordinateSequence cs = new PackedCoordinateSequence.Double(coordsList.size(), 2, 0);
+        for (int j = 0; j < coordsList.size(); j++) {
+          cs.setOrdinate(j, 0, coordsList.get(j).get(0));
+          cs.setOrdinate(j, 1, coordsList.get(j).get(1));
+        }
+        if (cs.size() == 1) {
+          geometry = gf.createPoint(cs);
+        } else if (cs.size() > 1) {
+          geometry = gf.createMultiPoint(cs);
+        }
+        break;
+      case POLYGON:
+        List<List<LinearRing>> polygonRings = new ArrayList<>();
+        List<LinearRing> ringsForCurrentPolygon = new ArrayList<>();
+        boolean first = true;
+        for (DoubleArrayList clist : coordsList) {
+          // skip hole with too few coordinates
+          if (ringsForCurrentPolygon.size() > 0 && clist.size() < 4) {
+            continue;
+          }
+          LinearRing ring = gf.createLinearRing(toCs(clist));
+          boolean ccw = Orientation.isCCW(ring.getCoordinates());
+          if (first) {
+            first = false;
+            outerCCW = ccw;
+          }
+          if (ccw == outerCCW) {
+            ringsForCurrentPolygon = new ArrayList<>();
+            polygonRings.add(ringsForCurrentPolygon);
+          }
+          ringsForCurrentPolygon.add(ring);
+        }
+        List<Polygon> polygons = new ArrayList<>();
+        for (List<LinearRing> rings : polygonRings) {
+          LinearRing shell = rings.get(0);
+          LinearRing[] holes = rings.subList(1, rings.size()).toArray(new LinearRing[rings.size() - 1]);
+          polygons.add(gf.createPolygon(shell, holes));
+        }
+        if (polygons.size() == 1) {
+          geometry = polygons.get(0);
+        }
+        if (polygons.size() > 1) {
+          geometry = gf.createMultiPolygon(GeometryFactory.toPolygonArray(polygons));
+        }
+        break;
+      default:
+        break;
+    }
+
+    if (geometry == null) {
+      geometry = gf.createGeometryCollection(new Geometry[0]);
+    }
+
+    return geometry;
+  }
+
+  public static List<DecodedFeature> decode(byte[] encoded) throws IOException {
+    VectorTile.Tile tile = VectorTile.Tile.parseFrom(encoded);
+    List<DecodedFeature> features = new ArrayList<>();
+    for (VectorTile.Tile.Layer layer : tile.getLayersList()) {
+      String layerName = layer.getName();
+      int extent = layer.getExtent();
+      List<String> keys = layer.getKeysList();
+      List<Object> values = new ArrayList<>();
+
+      for (VectorTile.Tile.Value value : layer.getValuesList()) {
+        if (value.hasBoolValue()) {
+          values.add(value.getBoolValue());
+        } else if (value.hasDoubleValue()) {
+          values.add(value.getDoubleValue());
+        } else if (value.hasFloatValue()) {
+          values.add(value.getFloatValue());
+        } else if (value.hasIntValue()) {
+          values.add(value.getIntValue());
+        } else if (value.hasSintValue()) {
+          values.add(value.getSintValue());
+        } else if (value.hasUintValue()) {
+          values.add(value.getUintValue());
+        } else if (value.hasStringValue()) {
+          values.add(value.getStringValue());
+        } else {
+          values.add(null);
+        }
+      }
+
+      for (VectorTile.Tile.Feature feature : layer.getFeaturesList()) {
+        int tagsCount = feature.getTagsCount();
+        Map<String, Object> attrs = new HashMap<>(tagsCount / 2);
+        int tagIdx = 0;
+        while (tagIdx < feature.getTagsCount()) {
+          String key = keys.get(feature.getTags(tagIdx++));
+          Object value = values.get(feature.getTags(tagIdx++));
+          attrs.put(key, value);
+        }
+        Geometry geometry = decode(feature.getType(), feature.getGeometryList());
+        features.add(new DecodedFeature(
+          layerName,
+          extent,
+          geometry,
+          attrs,
+          feature.getId()
+        ));
+      }
+    }
+    return features;
+  }
+
+  private static Geometry decode(GeomType type, List<Integer> geometryList) {
+    return decode((byte) type.getNumber(), geometryList.stream().mapToInt(i -> i).toArray());
+  }
+
+  public VectorTileEncoder addLayerFeatures(String layerName, List<LayerFeature> features) {
+    if (features.isEmpty()) {
+      return this;
+    }
+
+    Layer layer = layers.get(layerName);
+    if (layer == null) {
+      layer = new Layer();
+      layers.put(layerName, layer);
+    }
+
+    for (LayerFeature inFeature : features) {
+      if (inFeature.commands().length > 0) {
+        EncodedFeature outFeature = new EncodedFeature(inFeature);
+
+        for (Map.Entry<String, ?> e : inFeature.attrs().entrySet()) {
+          // skip attribute without value
+          if (e.getValue() == null) {
+            continue;
+          }
+          outFeature.tags.add(layer.key(e.getKey()));
+          outFeature.tags.add(layer.value(e.getValue()));
+        }
+
+        layer.encodedFeatures.add(outFeature);
+      }
+    }
+    return this;
+  }
+
+  public byte[] encode() {
+    VectorTile.Tile.Builder tile = VectorTile.Tile.newBuilder();
+    for (Map.Entry<String, Layer> e : layers.entrySet()) {
+      String layerName = e.getKey();
+      Layer layer = e.getValue();
+
+      VectorTile.Tile.Layer.Builder tileLayer = VectorTile.Tile.Layer.newBuilder();
+
+      tileLayer.setVersion(2);
+      tileLayer.setName(layerName);
+
+      tileLayer.addAllKeys(layer.keys());
+
+      for (Object value : layer.values()) {
+        VectorTile.Tile.Value.Builder tileValue = VectorTile.Tile.Value.newBuilder();
+        if (value instanceof String stringValue) {
+          tileValue.setStringValue(stringValue);
+        } else if (value instanceof Integer intValue) {
+          tileValue.setSintValue(intValue);
+        } else if (value instanceof Long longValue) {
+          tileValue.setSintValue(longValue);
+        } else if (value instanceof Float floatValue) {
+          tileValue.setFloatValue(floatValue);
+        } else if (value instanceof Double doubleValue) {
+          tileValue.setDoubleValue(doubleValue);
+        } else if (value instanceof Boolean booleanValue) {
+          tileValue.setBoolValue(booleanValue);
+        } else {
+          tileValue.setStringValue(value.toString());
+        }
+        tileLayer.addValues(tileValue.build());
+      }
+
+      tileLayer.setExtent(EXTENT);
+
+      for (EncodedFeature feature : layer.encodedFeatures) {
+
+        VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder();
+
+        featureBuilder.addAllTags(Ints.asList(feature.tags.toArray()));
+        if (feature.id >= 0) {
+          featureBuilder.setId(feature.id);
+        }
+
+        featureBuilder.setType(VectorTile.Tile.GeomType.forNumber(feature.geometryType));
+        featureBuilder.addAllGeometry(Ints.asList(feature.geometry));
+        tileLayer.addFeatures(featureBuilder.build());
+      }
+
+      tile.addLayers(tileLayer.build());
+    }
+    return tile.build().toByteArray();
+  }
+
+  private enum Command {
+    MOVE_TO(1),
+    LINE_TO(2),
+    CLOSE_PATH(7);
+    final int value;
+
+    Command(int value) {
+      this.value = value;
+    }
+  }
+
+  private static class CommandEncoder {
+
+    private final IntArrayList result = new IntArrayList();
+    private int x = 0, y = 0;
+
+    private static boolean shouldClosePath(Geometry geometry) {
+      return (geometry instanceof Polygon) || (geometry instanceof LinearRing);
+    }
+
+    private static int commandAndLength(Command command, int repeat) {
+      return repeat << 3 | command.value;
+    }
+
+    private void accept(Geometry geometry) {
+      if (geometry instanceof MultiLineString multiLineString) {
+        for (int i = 0; i < multiLineString.getNumGeometries(); i++) {
+          encode(((LineString) multiLineString.getGeometryN(i)).getCoordinateSequence(), false);
+        }
+      } else if (geometry instanceof Polygon polygon) {
+        LineString exteriorRing = polygon.getExteriorRing();
+        encode(exteriorRing.getCoordinateSequence(), true);
+
+        for (int i = 0; i < polygon.getNumInteriorRing(); i++) {
+          LineString interiorRing = polygon.getInteriorRingN(i);
+          encode(interiorRing.getCoordinateSequence(), true);
+        }
+      } else if (geometry instanceof MultiPolygon multiPolygon) {
+        for (int i = 0; i < multiPolygon.getNumGeometries(); i++) {
+          accept(multiPolygon.getGeometryN(i));
+        }
+      } else if (geometry instanceof LineString lineString) {
+        encode(lineString.getCoordinateSequence(), shouldClosePath(geometry));
+      } else if (geometry instanceof Point point) {
+        encode(point.getCoordinateSequence(), false);
+      } else {
+        encode(new CoordinateArraySequence(geometry.getCoordinates()), shouldClosePath(geometry),
+          geometry instanceof MultiPoint);
+      }
+    }
+
+    private void encode(CoordinateSequence cs, boolean closePathAtEnd) {
+      encode(cs, closePathAtEnd, false);
+    }
+
+    private void encode(CoordinateSequence cs, boolean closePathAtEnd, boolean multiPoint) {
+
+      if (cs.size() == 0) {
+        throw new IllegalArgumentException("empty geometry");
+      }
+
+      int lineToIndex = 0;
+      int lineToLength = 0;
+
+      for (int i = 0; i < cs.size(); i++) {
+
+        double cx = cs.getX(i);
+        double cy = cs.getY(i);
+
+        if (i == 0) {
+          result.add(commandAndLength(Command.MOVE_TO, multiPoint ? cs.size() : 1));
+        }
+
+        int _x = (int) Math.round(cx * SCALE);
+        int _y = (int) Math.round(cy * SCALE);
+
+        // prevent point equal to the previous
+        if (i > 0 && _x == x && _y == y) {
+          lineToLength--;
+          continue;
+        }
+
+        // prevent double closing
+        if (closePathAtEnd && cs.size() > 1 && i == (cs.size() - 1) && cs.getX(0) == cx && cs.getY(0) == cy) {
+          lineToLength--;
+          continue;
+        }
+
+        // delta, then zigzag
+        result.add(zigZagEncode(_x - x));
+        result.add(zigZagEncode(_y - y));
+
+        x = _x;
+        y = _y;
+
+        if (i == 0 && cs.size() > 1 && !multiPoint) {
+          // can length be too long?
+          lineToIndex = result.size();
+          lineToLength = cs.size() - 1;
+          result.add(commandAndLength(Command.LINE_TO, lineToLength));
+        }
+
+      }
+
+      // update LineTo length
+      if (lineToIndex > 0) {
+        if (lineToLength == 0) {
+          // remove empty LineTo
+          result.remove(lineToIndex);
+        } else {
+          // update LineTo with new length
+          result.set(lineToIndex, commandAndLength(Command.LINE_TO, lineToLength));
+        }
+      }
+
+      if (closePathAtEnd) {
+        result.add(commandAndLength(Command.CLOSE_PATH, 1));
+      }
+    }
+  }
+
+  private static final record EncodedFeature(IntArrayList tags, long id, byte geometryType, int[] geometry) {
+
+    EncodedFeature(LayerFeature in) {
+      this(new IntArrayList(), in.id(), in.geomType(), in.commands());
+    }
+  }
+
+  public static final record DecodedFeature(
+    String layerName,
+    int extent,
+    Geometry geometry,
+    Map<String, Object> attributes,
+    long id
+  ) {
+
+  }
+
+  private static final class Layer {
+
+    private final List<EncodedFeature> encodedFeatures = new ArrayList<>();
+    private final Map<String, Integer> keys = new LinkedHashMap<>();
+    private final Map<Object, Integer> values = new LinkedHashMap<>();
+
+    public Integer key(String key) {
+      Integer i = keys.get(key);
+      if (i == null) {
+        i = keys.size();
+        keys.put(key, i);
+      }
+      return i;
+    }
+
+    public List<String> keys() {
+      return new ArrayList<>(keys.keySet());
+    }
+
+    public List<Object> values() {
+      return new ArrayList<>(values.keySet());
+    }
+
+    public Integer value(Object value) {
+      Integer i = values.get(value);
+      if (i == null) {
+        i = values.size();
+        values.put(value, i);
+      }
+      return i;
+    }
+
+    @Override
+    public String toString() {
+      return "Layer{" + encodedFeatures.size() + "}";
+    }
+  }
+}

src/main/java/com/onthegomap/flatmap/collections/MergeSortFeatureMap.java

@@ -1,24 +1,33 @@
 package com.onthegomap.flatmap.collections;
 
+import com.carrotsearch.hppc.LongArrayList;
 import com.onthegomap.flatmap.RenderedFeature;
-import com.onthegomap.flatmap.VectorTile;
+import com.onthegomap.flatmap.VectorTileEncoder;
+import com.onthegomap.flatmap.geo.TileCoord;
 import com.onthegomap.flatmap.monitoring.Stats;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.List;
 import java.util.function.Consumer;
 
 public class MergeSortFeatureMap implements Consumer<RenderedFeature> {
 
+  private volatile boolean prepared = false;
+
   public MergeSortFeatureMap(Path featureDb, Stats stats) {
 
   }
 
   public void sort() {
+    prepared = true;
   }
 
   @Override
   public void accept(RenderedFeature renderedFeature) {
-
+    if (prepared) {
+      throw new IllegalStateException("Attempting to add feature but already prepared");
+    }
   }
 
   public long getStorageSize() {
@@ -26,25 +35,55 @@ public class MergeSortFeatureMap implements Consumer<RenderedFeature> {
   }
 
   public Iterator<TileFeatures> getAll() {
-    return null;
+    if (!prepared) {
+      throw new IllegalStateException("Attempting to iterate over features but not prepared yet");
+    }
+    return new Iterator<>() {
+      @Override
+      public boolean hasNext() {
+        return false;
+      }
+
+      @Override
+      public TileFeatures next() {
+        return null;
+      }
+    };
   }
 
   public static class TileFeatures {
 
+    private final TileCoord tile;
+    private final LongArrayList sortKeys = new LongArrayList();
+    private final List<byte[]> entries = new ArrayList<>();
+
+    public TileFeatures(int tile) {
+      this.tile = TileCoord.decode(tile);
+    }
+
     public long getNumFeatures() {
       return 0;
     }
 
-    public int getTileId() {
-      return 0;
+    public TileCoord coord() {
+      return tile;
     }
 
     public boolean hasSameContents(TileFeatures other) {
       return false;
     }
 
-    public VectorTile getTile() {
+    public VectorTileEncoder getTile() {
       return null;
     }
+
+    @Override
+    public String toString() {
+      return "TileFeatures{" +
+        "tile=" + tile +
+        ", sortKeys=" + sortKeys +
+        ", entries=" + entries +
+        '}';
+    }
   }
 }

src/main/java/com/onthegomap/flatmap/geo/GeoUtils.java

@@ -1,4 +1,4 @@
-package com.onthegomap.flatmap;
+package com.onthegomap.flatmap.geo;
 
 import org.locationtech.jts.geom.CoordinateSequence;
 import org.locationtech.jts.geom.Geometry;
@@ -88,17 +88,4 @@ public class GeoUtils {
     long y = (long) (worldY * QUANTIZED_WORLD_SIZE);
     return (x << 32) | y;
   }
-
-  public static int z(int key) {
-    int result = key >> 28;
-    return result < 0 ? 16 + result : result;
-  }
-
-  public static int x(int key) {
-    return (key >> 14) & ((1 << 14) - 1);
-  }
-
-  public static int y(int key) {
-    return (key) & ((1 << 14) - 1);
-  }
 }

src/main/java/com/onthegomap/flatmap/geo/TileCoord.java

@@ -0,0 +1,97 @@
+package com.onthegomap.flatmap.geo;
+
+public class TileCoord {
+
+  private final int encoded;
+  private final int x;
+  private final int y;
+  private final int z;
+
+  private TileCoord(int encoded, int x, int y, int z) {
+    this.encoded = encoded;
+    this.x = x;
+    this.y = y;
+    this.z = z;
+  }
+
+  public static TileCoord of(int x, int y, int z) {
+    return new TileCoord(encode(x, y, z), x, y, z);
+  }
+
+  public static TileCoord decode(int encoded) {
+    return new TileCoord(encoded, decodeX(encoded), decodeY(encoded), decodeZ(encoded));
+  }
+
+  public int encoded() {
+    return encoded;
+  }
+
+  public int x() {
+    return x;
+  }
+
+  public int y() {
+    return y;
+  }
+
+  public int z() {
+    return z;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+
+    TileCoord tileCoord = (TileCoord) o;
+
+    return encoded == tileCoord.encoded;
+  }
+
+  @Override
+  public int hashCode() {
+    return encoded;
+  }
+
+  public static int decodeZ(int key) {
+    int result = key >> 28;
+    return result < 0 ? 16 + result : result;
+  }
+
+  public static int decodeX(int key) {
+    return (key >> 14) & ((1 << 14) - 1);
+  }
+
+  public static int decodeY(int key) {
+    return (key) & ((1 << 14) - 1);
+  }
+
+  private static int encode(int x, int y, int z) {
+    int max = 1 << z;
+    if (x >= max) {
+      x %= max;
+    }
+    if (x < 0) {
+      x += max;
+    }
+    if (y < 0) {
+      y = 0;
+    }
+    if (y >= max) {
+      y = max;
+    }
+    return (z << 28) | (x << 14) | y;
+  }
+
+  @Override
+  public String toString() {
+    return "TileCoord{" +
+      z + "/" + x + "/" + y +
+      ", encoded=" + encoded +
+      '}';
+  }
+}

src/main/java/com/onthegomap/flatmap/reader/NaturalEarthReader.java

@@ -1,7 +1,7 @@
 package com.onthegomap.flatmap.reader;
 
-import com.onthegomap.flatmap.GeoUtils;
 import com.onthegomap.flatmap.SourceFeature;
+import com.onthegomap.flatmap.geo.GeoUtils;
 import com.onthegomap.flatmap.monitoring.Stats;
 import com.onthegomap.flatmap.worker.Topology.SourceStep;
 import java.io.File;

src/main/java/com/onthegomap/flatmap/reader/OpenStreetMapReader.java

@@ -10,7 +10,6 @@ import com.graphhopper.reader.ReaderRelation;
 import com.graphhopper.reader.ReaderWay;
 import com.onthegomap.flatmap.FeatureRenderer;
 import com.onthegomap.flatmap.FlatMapConfig;
-import com.onthegomap.flatmap.GeoUtils;
 import com.onthegomap.flatmap.OsmInputFile;
 import com.onthegomap.flatmap.Profile;
 import com.onthegomap.flatmap.RenderableFeature;
@@ -20,6 +19,7 @@ import com.onthegomap.flatmap.SourceFeature;
 import com.onthegomap.flatmap.collections.LongLongMap;
 import com.onthegomap.flatmap.collections.LongLongMultimap;
 import com.onthegomap.flatmap.collections.MergeSortFeatureMap;
+import com.onthegomap.flatmap.geo.GeoUtils;
 import com.onthegomap.flatmap.monitoring.ProgressLoggers;
 import com.onthegomap.flatmap.monitoring.Stats;
 import com.onthegomap.flatmap.worker.Topology;

src/main/resources/vector_tile.proto

@@ -0,0 +1,79 @@
+syntax = "proto2";
+package vector_tile;
+
+option optimize_for = SPEED;
+
+message Tile {
+
+  // GeomType is described in section 4.3.4 of the specification
+  enum GeomType {
+    UNKNOWN = 0;
+    POINT = 1;
+    LINESTRING = 2;
+    POLYGON = 3;
+  }
+
+  // Variant type encoding
+  // The use of values is described in section 4.1 of the specification
+  message Value {
+    // Exactly one of these values must be present in a valid message
+    optional string string_value = 1;
+    optional float float_value = 2;
+    optional double double_value = 3;
+    optional int64 int_value = 4;
+    optional uint64 uint_value = 5;
+    optional sint64 sint_value = 6;
+    optional bool bool_value = 7;
+
+    extensions 8 to max;
+  }
+
+  // Features are described in section 4.2 of the specification
+  message Feature {
+    optional uint64 id = 1 [default = 0];
+
+    // Tags of this feature are encoded as repeated pairs of
+    // integers.
+    // A detailed description of tags is located in sections
+    // 4.2 and 4.4 of the specification
+    repeated uint32 tags = 2 [packed = true];
+
+    // The type of geometry stored in this feature.
+    optional GeomType type = 3 [default = UNKNOWN];
+
+    // Contains a stream of commands and parameters (vertices).
+    // A detailed description on geometry encoding is located in
+    // section 4.3 of the specification.
+    repeated uint32 geometry = 4 [packed = true];
+  }
+
+  // Layers are described in section 4.1 of the specification
+  message Layer {
+    // Any compliant implementation must first read the version
+    // number encoded in this message and choose the correct
+    // implementation for this version number before proceeding to
+    // decode other parts of this message.
+    required uint32 version = 15 [default = 1];
+
+    required string name = 1;
+
+    // The actual features in this tile.
+    repeated Feature features = 2;
+
+    // Dictionary encoding for keys
+    repeated string keys = 3;
+
+    // Dictionary encoding for values
+    repeated Value values = 4;
+
+    // Although this is an "optional" field it is required by the specification.
+    // See https://github.com/mapbox/vector-tile-spec/issues/47
+    optional uint32 extent = 5 [default = 4096];
+
+    extensions 16 to max;
+  }
+
+  repeated Layer layers = 3;
+
+  extensions 16 to 8191;
+}

src/test/java/com/onthegomap/flatmap/VectorTileEncoderTest.java

@@ -0,0 +1,145 @@
+package com.onthegomap.flatmap;
+
+import static com.onthegomap.flatmap.geo.GeoUtils.gf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import com.onthegomap.flatmap.VectorTileEncoder.DecodedFeature;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import org.junit.jupiter.api.Test;
+import org.locationtech.jts.geom.Coordinate;
+import org.locationtech.jts.geom.CoordinateXY;
+import org.locationtech.jts.geom.Geometry;
+import org.locationtech.jts.geom.LinearRing;
+import org.locationtech.jts.geom.Point;
+import org.locationtech.jts.geom.Polygon;
+import vector_tile.VectorTile.Tile.GeomType;
+
+public class VectorTileEncoderTest {
+
+  @Test
+  public void testRoundTripPoint() throws IOException {
+    testRoundTripGeometry(gf.createPoint(new CoordinateXY(1, 2)));
+  }
+
+  @Test
+  public void testRoundTripMultipoint() throws IOException {
+    testRoundTripGeometry(gf.createMultiPointFromCoords(new Coordinate[]{
+      new CoordinateXY(1, 2),
+      new CoordinateXY(3, 4)
+    }));
+  }
+
+  @Test
+  public void testRoundTripLineString() throws IOException {
+    testRoundTripGeometry(gf.createLineString(new Coordinate[]{
+      new CoordinateXY(1, 2),
+      new CoordinateXY(3, 4)
+    }));
+  }
+
+  @Test
+  public void testRoundTripPolygon() throws IOException {
+    testRoundTripGeometry(gf.createPolygon(
+      gf.createLinearRing(new Coordinate[]{
+        new CoordinateXY(0, 0),
+        new CoordinateXY(4, 0),
+        new CoordinateXY(4, 4),
+        new CoordinateXY(0, 4),
+        new CoordinateXY(0, 0)
+      }),
+      new LinearRing[]{
+        gf.createLinearRing(new Coordinate[]{
+          new CoordinateXY(1, 1),
+          new CoordinateXY(1, 2),
+          new CoordinateXY(2, 2),
+          new CoordinateXY(2, 1),
+          new CoordinateXY(1, 1)
+        })
+      }
+    ));
+  }
+
+  @Test
+  public void testRoundTripMultiPolygon() throws IOException {
+    testRoundTripGeometry(gf.createMultiPolygon(new Polygon[]{
+      gf.createPolygon(new Coordinate[]{
+        new CoordinateXY(0, 0),
+        new CoordinateXY(1, 0),
+        new CoordinateXY(1, 1),
+        new CoordinateXY(0, 1),
+        new CoordinateXY(0, 0)
+      }),
+      gf.createPolygon(new Coordinate[]{
+        new CoordinateXY(3, 0),
+        new CoordinateXY(4, 0),
+        new CoordinateXY(4, 1),
+        new CoordinateXY(3, 1),
+        new CoordinateXY(3, 0)
+      })
+    }));
+  }
+
+  @Test
+  public void testRoundTripAttributes() throws IOException {
+    testRoundTripAttrs(Map.of(
+      "string", "string",
+      "long", 1L,
+      "double", 3.5d,
+      "true", true,
+      "false", false
+    ));
+  }
+
+  @Test
+  public void testMultipleFeaturesMultipleLayer() throws IOException {
+    Point point = gf.createPoint(new CoordinateXY(0, 0));
+    Map<String, Object> attrs1 = Map.of("a", 1L, "b", 2L);
+    Map<String, Object> attrs2 = Map.of("b", 3L, "c", 2L);
+    byte[] encoded = new VectorTileEncoder().addLayerFeatures("layer1", List.of(
+      new LayerFeature(false, 0, 0, attrs1,
+        (byte) GeomType.POINT.getNumber(), VectorTileEncoder.getCommands(point), 1L),
+      new LayerFeature(false, 0, 0, attrs2,
+        (byte) GeomType.POINT.getNumber(), VectorTileEncoder.getCommands(point), 2L)
+    )).addLayerFeatures("layer2", List.of(
+      new LayerFeature(false, 0, 0, attrs1,
+        (byte) GeomType.POINT.getNumber(), VectorTileEncoder.getCommands(point), 3L)
+    )).encode();
+
+    List<DecodedFeature> decoded = VectorTileEncoder.decode(encoded);
+    assertEquals(attrs1, decoded.get(0).attributes());
+    assertEquals("layer1", decoded.get(0).layerName());
+
+    assertEquals(attrs2, decoded.get(1).attributes());
+    assertEquals("layer1", decoded.get(1).layerName());
+
+    assertEquals(attrs1, decoded.get(2).attributes());
+    assertEquals("layer2", decoded.get(2).layerName());
+  }
+
+  private void testRoundTripAttrs(Map<String, Object> attrs) throws IOException {
+    testRoundTrip(gf.createPoint(new CoordinateXY(0, 0)), "layer", attrs, 1);
+  }
+
+  private void testRoundTripGeometry(Geometry input) throws IOException {
+    testRoundTrip(input, "layer", Map.of(), 1);
+  }
+
+  private void testRoundTrip(Geometry input, String layer, Map<String, Object> attrs, long id) throws IOException {
+    int[] commands = VectorTileEncoder.getCommands(input);
+    byte geomType = (byte) VectorTileEncoder.toGeomType(input).ordinal();
+    Geometry output = VectorTileEncoder.decode(geomType, commands);
+    assertTrue(input.equalsExact(output), "\n" + input + "\n!=\n" + output);
+
+    byte[] encoded = new VectorTileEncoder().addLayerFeatures(layer, List.of(
+      new LayerFeature(false, 0, 0, attrs,
+        (byte) VectorTileEncoder.toGeomType(input).getNumber(), VectorTileEncoder.getCommands(input), id)
+    )).encode();
+
+    List<DecodedFeature> decoded = VectorTileEncoder.decode(encoded);
+    DecodedFeature expected = new DecodedFeature(layer, 4096, input, attrs, id);
+    assertEquals(List.of(expected), decoded);
+  }
+}

src/test/java/com/onthegomap/flatmap/collections/MergeSortFeatureMapTest.java

@@ -0,0 +1,32 @@
+package com.onthegomap.flatmap.collections;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import com.onthegomap.flatmap.RenderedFeature;
+import com.onthegomap.flatmap.monitoring.Stats.InMemory;
+import java.nio.file.Path;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+public class MergeSortFeatureMapTest {
+
+  @TempDir
+  Path tmpDir;
+
+  @Test
+  public void testEmpty() {
+    var features = new MergeSortFeatureMap(tmpDir, new InMemory());
+    features.sort();
+    assertFalse(features.getAll().hasNext());
+  }
+
+  @Test
+  public void testThrowsWhenPreparedOutOfOrder() {
+    var features = new MergeSortFeatureMap(tmpDir, new InMemory());
+    features.accept(new RenderedFeature(1, new byte[]{}));
+    assertThrows(IllegalStateException.class, features::getAll);
+    features.sort();
+    assertThrows(IllegalStateException.class, () -> features.accept(new RenderedFeature(1, new byte[]{})));
+  }
+}

src/test/java/com/onthegomap/flatmap/reader/NaturalEarthReaderTest.java

@@ -4,7 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
-import com.onthegomap.flatmap.GeoUtils;
+import com.onthegomap.flatmap.geo.GeoUtils;
 import com.onthegomap.flatmap.monitoring.Stats.InMemory;
 import com.onthegomap.flatmap.worker.Topology;
 import java.io.File;

src/test/java/com/onthegomap/flatmap/reader/ShapefileReaderTest.java

@@ -3,7 +3,7 @@ package com.onthegomap.flatmap.reader;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
-import com.onthegomap.flatmap.GeoUtils;
+import com.onthegomap.flatmap.geo.GeoUtils;
 import com.onthegomap.flatmap.monitoring.Stats.InMemory;
 import com.onthegomap.flatmap.worker.Topology;
 import java.io.File;