package com.onthegomap.planetiler.stats; import static io.prometheus.client.Collector.NANOSECONDS_PER_SECOND; import com.onthegomap.planetiler.util.FileUtils; import com.onthegomap.planetiler.util.Format; import com.onthegomap.planetiler.util.LogUtil; import com.onthegomap.planetiler.util.MemoryEstimator; import java.nio.file.Path; import java.time.Duration; import java.util.Map; import java.util.concurrent.ConcurrentSkipListMap; import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * A utility that collects and reports more detailed statistics about the JVM and running tasks than logs can convey. *
* {@link #inMemory()} stores basic stats in-memory to report at the end of the job and * {@link #prometheusPushGateway(String, String, Duration)} pushes stats at a regular interval to a * prometheus push gateway. */ public interface Stats extends AutoCloseable { /** Returns a new stat collector that stores basic stats in-memory to report through {@link #printSummary()}. */ static Stats inMemory() { return new InMemory(); } /** * Returns a new stat collector pushes stats at a regular interval to a * prometheus push gateway at {@code destination}. */ static Stats prometheusPushGateway(String destination, String job, Duration interval) { return PrometheusStats.createAndStartPushing(destination, job, interval); } /** * Logs top-level stats at the end of a job like the amount of user and CPU time that each task has taken, and size of * each monitored file. */ default void printSummary() { Format format = Format.defaultInstance(); Logger LOGGER = LoggerFactory.getLogger(getClass()); LOGGER.info(""); LOGGER.info("-".repeat(40)); timers().printSummary(); LOGGER.info("-".repeat(40)); for (var entry : monitoredFiles().entrySet()) { long size = FileUtils.size(entry.getValue()); if (size > 0) { LOGGER.info("\t" + entry.getKey() + "\t" + format.storage(size, false) + "B"); } } } /** * Records that a long-running task with {@code name} has started and returns a handle to call when finished. *
* Also sets the "stage" prefix that shows up in the logs to {@code name}.
*/
default Timers.Finishable startStage(String name) {
LogUtil.setStage(name);
var timer = timers().startTimer(name);
return () -> {
timer.stop();
LogUtil.clearStage();
};
}
/**
* Records that {@code numFeatures} features have been rendered to an output {@code layer} while processing a data
* source.
*/
void emittedFeatures(int z, String layer, int numFeatures);
/** Records that an input element was processed and emitted some output features in {@code layer}. */
void processedElement(String elemType, String layer);
/** Records that a tile has been written to the mbtiles output where compressed size is {@code bytes}. */
void wroteTile(int zoom, int bytes);
/** Returns the timers for all stages started with {@link #startStage(String)}. */
Timers timers();
/** Returns all the files being monitored. */
Map