Fast tests script (#1102)

planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java

@@ -124,7 +124,7 @@ public record PlanetilerConfig(
     int renderMaxzoom =
       arguments.getInteger("render_maxzoom", "maximum rendering zoom level up to " + MAX_MAXZOOM,
         Math.max(maxzoom, DEFAULT_MAXZOOM));
-    Path tmpDir = arguments.file("tmpdir", "temp directory", Path.of("data", "tmp"));
+    Path tmpDir = arguments.file("tmpdir|tmp", "temp directory", Path.of("data", "tmp"));
 
     return new PlanetilerConfig(
       arguments,

planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java

@@ -62,8 +62,8 @@ public class Wikidata {
 
   private static final ObjectMapper objectMapper = new ObjectMapper();
   private static final Logger LOGGER = LoggerFactory.getLogger(Wikidata.class);
-  private static final Pattern wikidataIRIMatcher = Pattern.compile("http://www.wikidata.org/entity/Q([0-9]+)");
-  private static final Pattern qidPattern = Pattern.compile("Q([0-9]+)");
+  private static final Pattern wikidataIRIMatcher = Pattern.compile("http://www.wikidata.org/entity/Q(\\d+)");
+  private static final Pattern qidPattern = Pattern.compile("Q(\\d+)");
   private final Counter.Readable blocks = Counter.newMultiThreadCounter();
   private final Counter.Readable nodes = Counter.newMultiThreadCounter();
   private final Counter.Readable ways = Counter.newMultiThreadCounter();
@@ -128,7 +128,7 @@ public class Wikidata {
 
     var timer = stats.startStage("wikidata");
     int processThreads = Math.max(1, config.threads() - 1);
-    LOGGER.info("Starting with " + processThreads + " process threads");
+    LOGGER.info("Starting with {} process threads", processThreads);
 
     WikidataTranslations oldMappings = load(outfile, maxAge, updateLimit);
     try (
@@ -166,7 +166,7 @@ public class Wikidata {
         .addPipelineStats(pipeline);
 
       pipeline.awaitAndLog(loggers, config.logInterval());
-      LOGGER.info("DONE fetched:" + fetcher.wikidatas.get());
+      LOGGER.info("DONE fetched: {}", fetcher.wikidatas.get());
     } catch (IOException e) {
       throw new UncheckedIOException(e);
     }
@@ -192,10 +192,14 @@ public class Wikidata {
       try (BufferedReader fis = Files.newBufferedReader(path)) {
         WikidataTranslations result = load(fis, maxAge, updateLimit, Clock.systemUTC());
         LOGGER.info(
-          "loaded from " + result.getAll().size() + " mappings from " + path.toAbsolutePath() + " in " + timer.stop());
+          "loaded from {} mappings from {} in {}",
+          result.getAll().size(),
+          path.toAbsolutePath(),
+          timer.stop()
+        );
         return result;
       } catch (IOException e) {
-        LOGGER.info("error loading " + path.toAbsolutePath() + ": " + e);
+        LOGGER.info("error loading {}: {}", path.toAbsolutePath(), e);
         return new WikidataTranslations();
       }
     }
@@ -247,7 +251,7 @@ public class Wikidata {
       String idText = matcher.group(1);
       return Long.parseLong(idText);
     } else {
-      throw new RuntimeException("Unexpected response IRI: " + iri);
+      throw new IllegalStateException("Unexpected response IRI: " + iri);
     }
   }
 
@@ -277,14 +281,13 @@ public class Wikidata {
           blockRelations++;
         }
         Object wikidata = elem.getString("wikidata");
-        if (wikidata instanceof String wikidataString) {
-          if (profile.caresAboutWikidataTranslation(elem)) {
-            long qid = parseQid(wikidataString);
-            if (qid > 0) {
-              next.accept(qid);
-            }
+        if (wikidata instanceof String wikidataString && profile.caresAboutWikidataTranslation(elem)) {
+          long qid = parseQid(wikidataString);
+          if (qid > 0) {
+            next.accept(qid);
           }
         }
+
       }
       blocks.inc();
       nodes.incBy(blockNodes);
@@ -356,12 +359,12 @@ public class Wikidata {
       } catch (IOException e) {
         boolean lastTry = i == config.httpRetries();
         if (!lastTry) {
-          LOGGER.warn("sparql query failed, retrying: " + e);
+          LOGGER.warn("sparql query failed, retrying: {}", e.toString());
         } else {
-          LOGGER.error("sparql query failed, exhausted retries: " + e);
+          LOGGER.error("sparql query failed, exhausted retries: {}", e.toString());
           throw e;
         }
-        Thread.sleep(config.httpRetryWait());
+        sleep(config.httpRetryWait());
       }
     }
 
@@ -372,11 +375,15 @@ public class Wikidata {
     }
   }
 
+  protected void sleep(Duration duration) throws InterruptedException {
+    Thread.sleep(duration.toMillis());
+  }
+
   void loadExisting(WikidataTranslations oldMappings) throws IOException {
     LongObjectMap<Map<String, String>> alreadyHave = oldMappings.getAll();
     LongObjectMap<Instant> alreadyHaveUpdateTimes = oldMappings.getUpdateTimes();
     if (!alreadyHave.isEmpty()) {
-      LOGGER.info("skipping " + alreadyHave.size() + " mappings we already have");
+      LOGGER.info("skipping {} mappings we already have", alreadyHave.size());
       writeTranslations(alreadyHave, alreadyHaveUpdateTimes);
       for (LongObjectCursor<Map<String, String>> cursor : alreadyHave) {
         visited.add(cursor.key);
@@ -405,12 +412,12 @@ public class Wikidata {
   interface Client {
 
     static Client wrap(HttpClient client) {
-      return (req) -> {
+      return req -> {
         var response = client.send(req, BodyHandlers.ofInputStream());
         if (response.statusCode() >= 400) {
           String body;
-          try {
-            body = new String(response.body().readAllBytes(), StandardCharsets.UTF_8);
+          try (var responseBody = response.body()) {
+            body = new String(responseBody.readAllBytes(), StandardCharsets.UTF_8);
           } catch (IOException e) {
             body = "Error reading body: " + e;
           }
@@ -435,8 +442,6 @@ public class Wikidata {
     private final LongObjectMap<Map<String, String>> data = Hppc.newLongObjectHashMap();
     private final LongObjectMap<Instant> updateTimes = Hppc.newLongObjectHashMap();
 
-    public WikidataTranslations() {}
-
     /** Returns a map from language code to translated name for {@code qid}. */
     public Map<String, String> get(long qid) {
       return data.get(qid);

planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureMergeTest.java

@@ -638,6 +638,7 @@ class FeatureMergeTest {
     );
   }
 
+  @Slow
   @ParameterizedTest
   @CsvSource({
     "bostonbuildings.mbtiles, 2477, 3028, 13, 1141",

planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java

@@ -72,6 +72,7 @@ import org.slf4j.LoggerFactory;
 /**
  * In-memory tests with fake data and profiles to ensure all features work end-to-end.
  */
+@Slow
 class PlanetilerTests {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(PlanetilerTests.class);

planetiler-core/src/test/java/com/onthegomap/planetiler/Slow.java

@@ -0,0 +1,14 @@
+package com.onthegomap.planetiler;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import org.junit.jupiter.api.Tag;
+
+/** Add to any junit test classes or methods to exclude when run with {@code -Pfast} maven argument. */
+@Target({ElementType.TYPE, ElementType.METHOD})
+@Tag("slow")
+@Retention(RetentionPolicy.RUNTIME)
+public @interface Slow {
+}

planetiler-core/src/test/java/com/onthegomap/planetiler/collection/AppendStoreTest.java

@@ -3,22 +3,29 @@ package com.onthegomap.planetiler.collection;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
+import java.io.IOException;
 import java.nio.file.Path;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 
-class AppendStoreTest {
+abstract class AppendStoreTest {
 
-  static abstract class IntsTest {
+  abstract static class IntsTest {
 
     protected AppendStore.Ints store;
 
+    @AfterEach
+    void close() throws IOException {
+      store.close();
+    }
+
     @ParameterizedTest
     @ValueSource(ints = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
-    public void writeThenRead(int num) {
+    void writeThenRead(int num) {
       for (int i = 0; i < num; i++) {
         store.appendInt(i + 1);
       }
@@ -30,7 +37,7 @@ class AppendStoreTest {
     }
 
     @Test
-    public void readBig() {
+    void readBig() {
       store.appendInt(Integer.MAX_VALUE);
       store.appendInt(Integer.MAX_VALUE - 1);
       store.appendInt(Integer.MAX_VALUE - 2);
@@ -40,13 +47,18 @@ class AppendStoreTest {
     }
   }
 
-  static abstract class LongsTest {
+  abstract static class LongsTest {
 
     protected AppendStore.Longs store;
 
+    @AfterEach
+    void close() throws IOException {
+      store.close();
+    }
+
     @ParameterizedTest
     @ValueSource(ints = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
-    public void writeThenRead(int num) {
+    void writeThenRead(int num) {
       for (int i = 0; i < num; i++) {
         store.appendLong(i + 1);
       }
@@ -57,87 +69,88 @@ class AppendStoreTest {
       assertThrows(IndexOutOfBoundsException.class, () -> store.getLong(num + 1));
     }
 
-    private static final long maxInt = Integer.MAX_VALUE;
+    private static final long MAX_INT = Integer.MAX_VALUE;
 
     @ParameterizedTest
-    @ValueSource(longs = {maxInt - 1, maxInt, maxInt + 1, 2 * maxInt - 1, 2 * maxInt, 5 * maxInt - 1, 5 * maxInt + 1})
-    public void readBig(long value) {
+    @ValueSource(longs = {MAX_INT - 1,
+      MAX_INT, MAX_INT + 1, 2 * MAX_INT - 1, 2 * MAX_INT, 5 * MAX_INT - 1, 5 * MAX_INT + 1})
+    void readBig(long value) {
       store.appendLong(value);
       assertEquals(value, store.getLong(0));
     }
 
   }
 
-  static class RamInt extends IntsTest {
+  static class RamIntTest extends IntsTest {
 
     @BeforeEach
-    public void setup() {
+    void setup() {
       this.store = new AppendStoreRam.Ints(false, 4 << 2);
     }
   }
 
-  static class MMapInt extends IntsTest {
+  static class MMapIntTest extends IntsTest {
 
     @BeforeEach
-    public void setup(@TempDir Path path) {
+    void setup(@TempDir Path path) {
       this.store = new AppendStoreMmap.Ints(path.resolve("ints"), 4 << 2, true);
     }
   }
 
-  static class DirectInt extends IntsTest {
+  static class DirectIntTest extends IntsTest {
 
     @BeforeEach
-    public void setup() {
+    void setup() {
       this.store = new AppendStoreRam.Ints(true, 4 << 2);
     }
   }
 
-  static class RamLong extends LongsTest {
+  static class RamLongTest extends LongsTest {
 
     @BeforeEach
-    public void setup() {
+    void setup() {
       this.store = new AppendStoreRam.Longs(false, 4 << 2);
     }
   }
 
-  static class MMapLong extends LongsTest {
+  static class MMapLongTest extends LongsTest {
 
     @BeforeEach
-    public void setup(@TempDir Path path) {
+    void setup(@TempDir Path path) {
       this.store = new AppendStoreMmap.Longs(path.resolve("longs"), 4 << 2, true);
     }
   }
 
-  static class DirectLong extends LongsTest {
+  static class DirectLongTest extends LongsTest {
 
     @BeforeEach
-    public void setup() {
+    void setup() {
       this.store = new AppendStoreRam.Longs(true, 4 << 2);
     }
   }
 
-  static class MMapSmallLong extends LongsTest {
+  static class MMapSmallLongTest extends LongsTest {
 
     @BeforeEach
-    public void setup(@TempDir Path path) {
+    void setup(@TempDir Path path) {
       this.store = new AppendStore.SmallLongs(
-        (i) -> new AppendStoreMmap.Ints(path.resolve("smalllongs" + i), 4 << 2, true));
+        i -> new AppendStoreMmap.Ints(path.resolve("smalllongs" + i), 4 << 2, true));
     }
   }
 
-  static class RamSmallLong extends LongsTest {
+  static class RamSmallLongTest extends LongsTest {
 
     @BeforeEach
-    public void setup() {
-      this.store = new AppendStore.SmallLongs((i) -> new AppendStoreRam.Ints(false, 4 << 2));
+    void setup() {
+      this.store = new AppendStore.SmallLongs(i -> new AppendStoreRam.Ints(false, 4 << 2));
     }
   }
 
-  static class DirectSmallLong extends LongsTest {
+  static class DirectSmallLongTest extends LongsTest {
 
     @BeforeEach
-    public void setup() {
-      this.store = new AppendStore.SmallLongs((i) -> new AppendStoreRam.Ints(true, 4 << 2));
+    void setup() {
+      this.store = new AppendStore.SmallLongs(i -> new AppendStoreRam.Ints(true, 4 << 2));
     }
   }
 }

planetiler-core/src/test/java/com/onthegomap/planetiler/collection/LongLongMultimapTest.java

@@ -11,18 +11,18 @@ import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
 
-public abstract class LongLongMultimapTest {
+abstract class LongLongMultimapTest {
 
   protected LongLongMultimap map;
   protected boolean retainInputOrder = false;
 
   @Test
-  public void missingValue() {
+  void missingValue() {
     assertTrue(map.get(0).isEmpty());
   }
 
   @Test
-  public void oneValue() {
+  void oneValue() {
     put(1, 1);
     assertResultLists(LongArrayList.from(), map.get(0));
     assertResultLists(LongArrayList.from(1), map.get(1));
@@ -50,7 +50,7 @@ public abstract class LongLongMultimapTest {
   }
 
   @Test
-  public void twoConsecutiveValues() {
+  void twoConsecutiveValues() {
     put(1, 1);
     put(2, 2);
     assertResultLists(LongArrayList.from(), map.get(0));
@@ -60,7 +60,7 @@ public abstract class LongLongMultimapTest {
   }
 
   @Test
-  public void twoNonconsecutiveValues() {
+  void twoNonconsecutiveValues() {
     put(1, 1);
     put(3, 3);
     assertResultLists(LongArrayList.from(), map.get(0));
@@ -71,7 +71,7 @@ public abstract class LongLongMultimapTest {
   }
 
   @Test
-  public void returnToFirstKey() {
+  void returnToFirstKey() {
     if (retainInputOrder) {
       return;
     }
@@ -91,7 +91,7 @@ public abstract class LongLongMultimapTest {
   }
 
   @Test
-  public void manyInsertsOrdered() {
+  void manyInsertsOrdered() {
     long[] toInsert = new long[10];
     for (int i = 0; i < 100; i++) {
       for (int j = 0; j < 10; j++) {
@@ -128,7 +128,7 @@ public abstract class LongLongMultimapTest {
   }
 
   @Test
-  public void manyInsertsUnordered() {
+  void manyInsertsUnordered() {
     for (long i = 99; i >= 0; i--) {
       putAll(i, LongArrayList.from(
         i * 10 + 10,
@@ -160,7 +160,7 @@ public abstract class LongLongMultimapTest {
   }
 
   @Test
-  public void multiInsert() {
+  void multiInsert() {
     putAll(1, LongArrayList.from(1, 2, 3));
     put(0, 3);
     assertResultLists(LongArrayList.from(3), map.get(0));
@@ -168,35 +168,35 @@ public abstract class LongLongMultimapTest {
     assertResultLists(LongArrayList.from(), map.get(2));
   }
 
-  public static class SparseUnorderedTest extends LongLongMultimapTest {
+  static class SparseUnorderedTest extends LongLongMultimapTest {
 
     @BeforeEach
-    public void setup() {
+    void setup() {
       this.map = LongLongMultimap.newAppendableMultimap();
     }
   }
 
-  public static class DenseOrderedTest extends LongLongMultimapTest {
+  static class DenseOrderedTest extends LongLongMultimapTest {
 
     @BeforeEach
-    public void setup() {
+    void setup() {
       retainInputOrder = true;
       this.map =
         LongLongMultimap.newInMemoryReplaceableMultimap();
     }
   }
 
-  public static class DenseOrderedMmapTest extends LongLongMultimapTest {
+  static class DenseOrderedMmapTest extends LongLongMultimapTest {
 
     @BeforeEach
-    public void setup(@TempDir Path dir) {
+    void setup(@TempDir Path dir) {
       retainInputOrder = true;
       this.map =
         LongLongMultimap.newReplaceableMultimap(Storage.MMAP, new Storage.Params(dir.resolve("multimap"), true));
     }
 
     @AfterEach
-    public void teardown() {
+    void teardown() {
       this.map.close();
     }
   }

planetiler-core/src/test/java/com/onthegomap/planetiler/util/WikidataTest.java

@@ -111,7 +111,7 @@ class WikidataTest {
   List<DynamicTest> testFetchWikidata() throws IOException, InterruptedException {
     StringWriter writer = new StringWriter();
     Wikidata.Client client = Mockito.mock(Wikidata.Client.class, Mockito.RETURNS_SMART_NULLS);
-    Wikidata fixture = new Wikidata(writer, client, 2, profile, config);
+    Wikidata fixture = createFixture(writer, client, 2);
     fixture.fetch(1L);
     Mockito.verifyNoInteractions(client);
     Mockito.when(client.send(Mockito.any()))
@@ -142,7 +142,7 @@ class WikidataTest {
       dynamicTest("do not re-request on subsequent loads", () -> {
         StringWriter writer2 = new StringWriter();
         Wikidata.Client client2 = Mockito.mock(Wikidata.Client.class, Mockito.RETURNS_SMART_NULLS);
-        Wikidata fixture2 = new Wikidata(writer2, client2, 2, profile, config);
+        Wikidata fixture2 = createFixture(writer2, client2, 2);
         fixture2.loadExisting(Wikidata.load(new BufferedReader(new StringReader(writer.toString()))));
         fixture2.fetch(1L);
         fixture2.fetch(2L);
@@ -157,7 +157,7 @@ class WikidataTest {
   void testRetryFailedRequestOnce() throws IOException, InterruptedException {
     StringWriter writer = new StringWriter();
     Wikidata.Client client = Mockito.mock(Wikidata.Client.class, Mockito.RETURNS_SMART_NULLS);
-    Wikidata fixture = new Wikidata(writer, client, 1, profile, config);
+    Wikidata fixture = createFixture(writer, client, 1);
     Mockito.when(client.send(Mockito.any()))
       // fail once then succeed
       .thenThrow(IOException.class)
@@ -236,4 +236,13 @@ class WikidataTest {
     }));
     return stringSubscriber.getBody().toCompletableFuture().join();
   }
+
+  private Wikidata createFixture(StringWriter writer, Wikidata.Client client, int batchSize) {
+    return new Wikidata(writer, client, batchSize, profile, config) {
+      @Override
+      protected void sleep(Duration duration) {
+        // don't sleep in tests
+      }
+    };
+  }
 }

planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/BikeRouteOverlayTest.java

@@ -96,7 +96,7 @@ class BikeRouteOverlayTest {
       // Override input source locations
       "osm_path", TestUtils.pathToResource("monaco-latest.osm.pbf"),
       // Override temp dir location
-      "tmp", tmpDir.toString(),
+      "tmpdir", tmpDir.toString(),
       // Override output location
       "output", dbPath.toString()
     ));

planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/OsmQaTilesTest.java

@@ -91,7 +91,7 @@ class OsmQaTilesTest {
       // Override input source locations
       "osm_path", TestUtils.pathToResource("monaco-latest.osm.pbf"),
       // Override temp dir location
-      "tmp", tmpDir.toString(),
+      "tmpdir", tmpDir.toString(),
       // Override output location
       "output", dbPath.toString()
     ));

planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/ToiletsProfileTest.java

@@ -56,7 +56,7 @@ class ToiletsProfileTest {
       // Override input source locations
       "osm_path", TestUtils.pathToResource("monaco-latest.osm.pbf"),
       // Override temp dir location
-      "tmp", tmpDir.toString(),
+      "tmpdir", tmpDir.toString(),
       // Override output location
       "output", dbPath.toString()
     ));

pom.xml

@@ -236,11 +236,15 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
         <version>3.5.2</version>
-        <!-- by default surefire excludes tests on nested classes https://github.com/junit-team/junit5/issues/1377 -->
         <configuration>
+          <!-- by default surefire excludes tests on nested classes https://github.com/junit-team/junit5/issues/1377 -->
           <excludes>
             <exclude/>
           </excludes>
+          <!-- by default surefire only includes tests matching Test*.java *Test.java or *TestCase.java -->
+          <includes>
+            <include>**/*.java</include>
+          </includes>
         </configuration>
       </plugin>
       <plugin>
@@ -441,5 +445,21 @@
         </plugins>
       </build>
     </profile>
+    <!-- run with -Pfast to skip slow tests -->
+    <profile>
+      <id>fast</id>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <configuration>
+              <groups>!slow</groups>
+              <forkCount>1C</forkCount>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
   </profiles>
 </project>

scripts/fasttests.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+./mvnw -T 1C -Pfast clean test