Support yaml merge operator (#1040)

planetiler-core/src/main/java/com/onthegomap/planetiler/util/YAML.java

@@ -1,6 +1,7 @@
 package com.onthegomap.planetiler.util;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.onthegomap.planetiler.reader.FileFormatException;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -8,6 +9,12 @@ import java.io.UncheckedIOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Collections;
+import java.util.IdentityHashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import org.snakeyaml.engine.v2.api.Load;
 import org.snakeyaml.engine.v2.api.LoadSettings;
 
@@ -33,12 +40,64 @@ public class YAML {
   public static <T> T load(InputStream stream, Class<T> clazz) {
     try (stream) {
       Object parsed = snakeYaml.loadFromInputStream(stream);
+      handleMergeOperator(parsed);
       return convertValue(parsed, clazz);
     } catch (IOException e) {
       throw new UncheckedIOException(e);
     }
   }
 
+  private static void handleMergeOperator(Object parsed) {
+    handleMergeOperator(parsed, Collections.newSetFromMap(new IdentityHashMap<>()));
+  }
+
+  /**
+   * SnakeYaml doesn't handle the <a href="https://yaml.org/type/merge.html">merge operator</a> so manually post-process
+   * the parsed yaml object to merge referenced objects into the parent one.
+   */
+  private static void handleMergeOperator(Object parsed, Set<Object> parentNodes) {
+    if (!parentNodes.add(parsed)) {
+      throw new FileFormatException("Illegal recursive reference in yaml file");
+    }
+    if (parsed instanceof Map<?, ?> map) {
+      Object toMerge = map.remove("<<");
+      if (toMerge != null) {
+        var orig = new LinkedHashMap<>(map);
+        // to preserve the map key order we insert the merged operator objects first, then the original ones
+        map.clear();
+        mergeInto(map, toMerge, false, parentNodes);
+        mergeInto(map, orig, true, parentNodes);
+      }
+      for (var value : map.values()) {
+        handleMergeOperator(value, parentNodes);
+      }
+    } else if (parsed instanceof List<?> list) {
+      for (var item : list) {
+        handleMergeOperator(item, parentNodes);
+      }
+    }
+    parentNodes.remove(parsed);
+  }
+
+  @SuppressWarnings("rawtypes")
+  private static void mergeInto(Map dest, Object source, boolean replace, Set<Object> parentNodes) {
+    if (!parentNodes.add(source)) {
+      throw new FileFormatException("Illegal recursive reference in yaml file");
+    }
+    if (source instanceof Map<?, ?> map) {
+      if (replace) {
+        dest.putAll(map);
+      } else {
+        map.forEach(dest::putIfAbsent);
+      }
+    } else if (source instanceof List<?> nesteds) {
+      for (var nested : nesteds) {
+        mergeInto(dest, nested, replace, parentNodes);
+      }
+    }
+    parentNodes.remove(source);
+  }
+
   public static <T> T load(String config, Class<T> clazz) {
     try (var stream = new ByteArrayInputStream(config.getBytes(StandardCharsets.UTF_8))) {
       return load(stream, clazz);

planetiler-core/src/test/java/com/onthegomap/planetiler/util/YamlTest.java

@@ -1,7 +1,9 @@
 package com.onthegomap.planetiler.util;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 
+import com.onthegomap.planetiler.reader.FileFormatException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -45,4 +47,258 @@ class YamlTest {
     }
     assertEquals(expected, YAML.load(builder.toString(), Map.class));
   }
+
+  @Test
+  void testMergeOperator() {
+    assertSameYaml("""
+      source: &label
+        a: 1
+      dest:
+        <<: *label
+        b: 2
+      """, """
+      source:
+        a: 1
+      dest:
+        a: 1
+        b: 2
+      """);
+  }
+
+  @Test
+  void testMergeOperatorNested() {
+    assertSameYaml("""
+      source: &label
+        a: 1
+      dest:
+        l1:
+          l2:
+            l3:
+              <<: *label
+              b: 2
+      """, """
+      source:
+        a: 1
+      dest:
+        l1:
+          l2:
+            l3:
+              a: 1
+              b: 2
+      """);
+  }
+
+  @Test
+  void testMergeOperatorOverride() {
+    assertSameYaml("""
+      source: &label
+        a: 1
+      dest:
+        <<: *label
+        a: 2
+      """, """
+      source:
+        a: 1
+      dest:
+        a: 2
+      """);
+  }
+
+  @Test
+  void testMergeOperatorMultiple() {
+    assertSameYaml("""
+      source: &label1
+        a: 1
+        z: 1
+      source2: &label2
+        a: 2
+        b: 3
+      dest:
+        <<: [*label1, *label2]
+        b: 4
+        c: 5
+      """, """
+      source:
+        a: 1
+        z: 1
+      source2:
+        a: 2
+        b: 3
+      dest:
+        a: 1 # from label1 since it came first
+        b: 4
+        c: 5
+        z: 1
+      """);
+  }
+
+  @Test
+  void testMergeNotAnchor() {
+    assertSameYaml("""
+      <<:
+        a: 1
+        b: 2
+      b: 3
+      c: 4
+      """, """
+      a: 1
+      b: 3
+      c: 4
+      """);
+  }
+
+  @Test
+  void testMergeOperatorSecond() {
+    assertSameYaml("""
+      source: &label
+        a: 1
+      dest:
+        c: 3
+        <<: *label
+        b: 2
+      """, """
+      source:
+        a: 1
+      dest:
+        a: 1
+        b: 2
+        c: 3
+      """);
+  }
+
+  @Test
+  void testMergeOperatorFromDraft1() {
+    assertSameYaml("""
+      - { x: 1, y: 2 }
+      - { x: 0, y: 2 }
+      - { r: 10 }
+      - { r: 1 }
+      - # Explicit keys
+        x: 1
+        y: 2
+        r: 10
+        label: center/big
+      """, """
+      - &CENTER { x: 1, y: 2 }
+      - &LEFT { x: 0, y: 2 }
+      - &BIG { r: 10 }
+      - &SMALL { r: 1 }
+      - # Merge one map
+       << : *CENTER
+       r: 10
+       label: center/big
+      """);
+  }
+
+  @Test
+  void testMergeOperatorFromDraft2() {
+    assertSameYaml("""
+      - { x: 1, y: 2 }
+      - { x: 0, y: 2 }
+      - { r: 10 }
+      - { r: 1 }
+      - # Explicit keys
+        x: 1
+        y: 2
+        r: 10
+        label: center/big
+      """, """
+      - &CENTER { x: 1, y: 2 }
+      - &LEFT { x: 0, y: 2 }
+      - &BIG { r: 10 }
+      - &SMALL { r: 1 }
+      - # Merge multiple maps
+        << : [ *CENTER, *BIG ]
+        label: center/big
+      """);
+  }
+
+  @Test
+  void testMergeOperatorFromDraft3() {
+    assertSameYaml("""
+      - { x: 1, y: 2 }
+      - { x: 0, y: 2 }
+      - { r: 10 }
+      - { r: 1 }
+      - # Explicit keys
+        x: 1
+        y: 2
+        r: 10
+        label: center/big
+      """, """
+      - &CENTER { x: 1, y: 2 }
+      - &LEFT { x: 0, y: 2 }
+      - &BIG { r: 10 }
+      - &SMALL { r: 1 }
+      - # Override
+        << : [ *BIG, *LEFT, *SMALL ]
+        x: 1
+        label: center/big
+      """);
+  }
+
+  @Test
+  void testAnchorAndAliasMap() {
+    assertSameYaml("""
+      source: &label
+        a: 1
+      dest: *label
+      """, """
+      source:
+        a: 1
+      dest:
+        a: 1
+      """);
+  }
+
+  @Test
+  void testAnchorAndAliasList() {
+    assertSameYaml("""
+      source: &label
+        - 1
+      dest: *label
+      """, """
+      source: [1]
+      dest: [1]
+      """);
+  }
+
+  @Test
+  void testAllowRefInMergeDoc() {
+    assertSameYaml("""
+      source: &label
+        a: &label1
+          c: 1
+        b: *label1
+        d:
+          <<: *label1
+      dest: *label
+      """, """
+      source: {a: {c: 1}, b: {c: 1}, d: {c: 1}}
+      dest: {a: {c: 1}, b: {c: 1}, d: {c: 1}}
+      """);
+  }
+
+  @Test
+  void testFailsOnRecursiveRefs() {
+    assertThrows(FileFormatException.class, () -> YAML.load("""
+      source: &label
+        - *label
+      """, Object.class));
+    assertThrows(FileFormatException.class, () -> YAML.load("""
+      source: &label
+        <<: *label
+      """, Object.class));
+    assertThrows(FileFormatException.class, () -> YAML.load("""
+      source: &label
+        a: *label
+      """, Object.class));
+  }
+
+  private static void assertSameYaml(String a, String b) {
+    assertEquals(
+      YAML.load(b, Object.class),
+      YAML.load(a, Object.class)
+    );
+  }
 }