2021-12-23 10:42:24 +00:00
|
|
|
package com.onthegomap.planetiler.reader;
|
2021-04-10 09:25:42 +00:00
|
|
|
|
2022-04-23 09:58:49 +00:00
|
|
|
import static com.onthegomap.planetiler.util.Exceptions.throwFatalException;
|
|
|
|
|
2021-12-23 10:42:24 +00:00
|
|
|
import com.onthegomap.planetiler.Profile;
|
|
|
|
import com.onthegomap.planetiler.collection.FeatureGroup;
|
|
|
|
import com.onthegomap.planetiler.config.PlanetilerConfig;
|
|
|
|
import com.onthegomap.planetiler.geo.GeoUtils;
|
|
|
|
import com.onthegomap.planetiler.stats.Stats;
|
|
|
|
import com.onthegomap.planetiler.util.FileUtils;
|
|
|
|
import com.onthegomap.planetiler.util.LogUtil;
|
2021-04-23 11:26:02 +00:00
|
|
|
import java.io.IOException;
|
2023-03-19 18:01:17 +00:00
|
|
|
import java.net.URLEncoder;
|
|
|
|
import java.nio.charset.StandardCharsets;
|
2021-05-01 20:40:44 +00:00
|
|
|
import java.nio.file.FileSystems;
|
2021-04-23 11:26:02 +00:00
|
|
|
import java.nio.file.Files;
|
|
|
|
import java.nio.file.Path;
|
2021-04-23 21:01:00 +00:00
|
|
|
import java.nio.file.StandardCopyOption;
|
2021-04-23 11:26:02 +00:00
|
|
|
import java.sql.Connection;
|
|
|
|
import java.sql.DriverManager;
|
|
|
|
import java.sql.ResultSet;
|
|
|
|
import java.sql.SQLException;
|
|
|
|
import java.sql.Statement;
|
|
|
|
import java.util.ArrayList;
|
2021-09-10 00:46:20 +00:00
|
|
|
import java.util.HashMap;
|
2021-04-23 11:26:02 +00:00
|
|
|
import java.util.List;
|
2022-12-15 19:19:22 +00:00
|
|
|
import java.util.function.Consumer;
|
2022-04-23 09:58:49 +00:00
|
|
|
import java.util.regex.Pattern;
|
2021-04-23 11:26:02 +00:00
|
|
|
import org.locationtech.jts.geom.Geometry;
|
|
|
|
import org.slf4j.Logger;
|
|
|
|
import org.slf4j.LoggerFactory;
|
2021-04-10 09:25:42 +00:00
|
|
|
|
2021-09-10 00:46:20 +00:00
|
|
|
/**
|
|
|
|
* Utility that reads {@link SourceFeature SourceFeatures} from the geometries contained in a Natural Earth sqlite
|
|
|
|
* distribution.
|
|
|
|
*
|
|
|
|
* @see <a href="https://www.naturalearthdata.com/">Natural Earth</a>
|
|
|
|
*/
|
2022-12-15 19:19:22 +00:00
|
|
|
public class NaturalEarthReader extends SimpleReader<SimpleFeature> {
|
2021-04-10 09:25:42 +00:00
|
|
|
|
2022-04-23 09:58:49 +00:00
|
|
|
private static final Pattern VALID_TABLE_NAME = Pattern.compile("ne_[a-z0-9_]+", Pattern.CASE_INSENSITIVE);
|
2021-04-23 11:26:02 +00:00
|
|
|
private static final Logger LOGGER = LoggerFactory.getLogger(NaturalEarthReader.class);
|
2022-12-15 19:19:22 +00:00
|
|
|
|
2021-04-23 11:26:02 +00:00
|
|
|
private final Connection conn;
|
2023-03-19 18:01:17 +00:00
|
|
|
private final boolean keepUnzipped;
|
2021-04-23 11:26:02 +00:00
|
|
|
private Path extracted;
|
|
|
|
|
2021-05-05 10:29:12 +00:00
|
|
|
static {
|
2021-09-10 00:46:20 +00:00
|
|
|
// make sure sqlite driver loaded
|
2021-05-05 10:29:12 +00:00
|
|
|
try {
|
|
|
|
Class.forName("org.sqlite.JDBC");
|
|
|
|
} catch (ClassNotFoundException e) {
|
2021-09-10 00:46:20 +00:00
|
|
|
throw new IllegalStateException("sqlite JDBC driver not found");
|
2021-05-05 10:29:12 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-03-19 18:01:17 +00:00
|
|
|
NaturalEarthReader(String sourceName, Path input, Path tmpDir, boolean keepUnzipped) {
|
2022-12-15 19:19:22 +00:00
|
|
|
super(sourceName);
|
2023-03-19 18:01:17 +00:00
|
|
|
this.keepUnzipped = keepUnzipped;
|
2022-12-15 19:19:22 +00:00
|
|
|
|
2021-08-10 10:55:30 +00:00
|
|
|
LogUtil.setStage(sourceName);
|
2021-04-23 11:26:02 +00:00
|
|
|
try {
|
|
|
|
conn = open(input, tmpDir);
|
|
|
|
} catch (IOException | SQLException e) {
|
2021-09-10 00:46:20 +00:00
|
|
|
throw new IllegalArgumentException(e);
|
2021-04-23 11:26:02 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-09-10 00:46:20 +00:00
|
|
|
/**
|
|
|
|
* Renders map features for all elements from a Natural Earth sqlite file, or zip file containing a sqlite file, based
|
|
|
|
* on the mapping logic defined in {@code profile}.
|
|
|
|
*
|
2023-03-19 18:01:17 +00:00
|
|
|
* @param sourceName string ID for this reader to use in logs and stats
|
|
|
|
* @param sourcePath path to the sqlite or zip file
|
|
|
|
* @param tmpDir directory to extract the sqlite file into (if input is a zip file).
|
|
|
|
* @param writer consumer for rendered features
|
|
|
|
* @param config user-defined parameters controlling number of threads and log interval
|
|
|
|
* @param profile logic that defines what map features to emit for each source feature
|
|
|
|
* @param stats to keep track of counters and timings
|
|
|
|
* @param keepUnzipped to keep unzipped files around after running (speeds up subsequent runs, but uses more disk)
|
2021-09-10 00:46:20 +00:00
|
|
|
* @throws IllegalArgumentException if a problem occurs reading the input file
|
|
|
|
*/
|
2022-12-15 19:19:22 +00:00
|
|
|
public static void process(String sourceName, Path sourcePath, Path tmpDir, FeatureGroup writer,
|
2023-03-19 18:01:17 +00:00
|
|
|
PlanetilerConfig config, Profile profile, Stats stats, boolean keepUnzipped) {
|
2022-12-15 19:19:22 +00:00
|
|
|
SourceFeatureProcessor.processFiles(
|
|
|
|
sourceName,
|
|
|
|
List.of(sourcePath),
|
2023-03-19 18:01:17 +00:00
|
|
|
path -> new NaturalEarthReader(sourceName, path, tmpDir, keepUnzipped),
|
2022-12-15 19:19:22 +00:00
|
|
|
writer, config, profile, stats
|
|
|
|
);
|
2021-05-08 10:53:37 +00:00
|
|
|
}
|
|
|
|
|
2021-09-10 00:46:20 +00:00
|
|
|
/** Returns a JDBC connection to the sqlite file. Input can be the sqlite file itself or a zip file containing it. */
|
2023-03-19 18:01:17 +00:00
|
|
|
private Connection open(Path path, Path unzippedDir) throws IOException, SQLException {
|
2021-05-01 20:08:20 +00:00
|
|
|
String uri = "jdbc:sqlite:" + path.toAbsolutePath();
|
2021-05-01 20:40:44 +00:00
|
|
|
if (FileUtils.hasExtension(path, "zip")) {
|
|
|
|
try (var zipFs = FileSystems.newFileSystem(path)) {
|
|
|
|
var zipEntry = FileUtils.walkFileSystem(zipFs)
|
2021-08-17 01:51:49 +00:00
|
|
|
.filter(Files::isRegularFile)
|
2021-05-01 20:40:44 +00:00
|
|
|
.filter(entry -> FileUtils.hasExtension(entry, "sqlite"))
|
2021-04-23 21:01:00 +00:00
|
|
|
.findFirst()
|
2021-05-01 20:40:44 +00:00
|
|
|
.orElseThrow(() -> new IllegalArgumentException("No .sqlite file found inside " + path));
|
2023-03-19 18:01:17 +00:00
|
|
|
extracted = unzippedDir.resolve(URLEncoder.encode(zipEntry.toString(), StandardCharsets.UTF_8));
|
|
|
|
FileUtils.createParentDirectories(extracted);
|
|
|
|
if (!keepUnzipped || FileUtils.isNewer(path, extracted)) {
|
2023-03-20 20:41:18 +00:00
|
|
|
LOGGER.info("unzipping {} to {}", path.toAbsolutePath(), extracted);
|
2023-03-19 18:01:17 +00:00
|
|
|
Files.copy(Files.newInputStream(zipEntry), extracted, StandardCopyOption.REPLACE_EXISTING);
|
|
|
|
}
|
|
|
|
if (!keepUnzipped) {
|
|
|
|
extracted.toFile().deleteOnExit();
|
|
|
|
}
|
2021-04-23 11:26:02 +00:00
|
|
|
}
|
2023-03-19 18:01:17 +00:00
|
|
|
uri = "jdbc:sqlite:" + extracted.toAbsolutePath();
|
2021-04-23 11:26:02 +00:00
|
|
|
}
|
2021-05-01 20:08:20 +00:00
|
|
|
return DriverManager.getConnection(uri);
|
2021-04-23 11:26:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
private List<String> tableNames() {
|
|
|
|
List<String> result = new ArrayList<>();
|
|
|
|
try (ResultSet rs = conn.getMetaData().getTables(null, null, null, null)) {
|
|
|
|
while (rs.next()) {
|
|
|
|
String table = rs.getString("TABLE_NAME");
|
2022-04-23 09:58:49 +00:00
|
|
|
if (VALID_TABLE_NAME.matcher(table).matches()) {
|
|
|
|
result.add(table);
|
|
|
|
}
|
2021-04-23 11:26:02 +00:00
|
|
|
}
|
|
|
|
} catch (SQLException e) {
|
2022-04-23 09:58:49 +00:00
|
|
|
throwFatalException(e);
|
2021-04-23 11:26:02 +00:00
|
|
|
}
|
|
|
|
return result;
|
2021-04-10 09:25:42 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2022-12-15 19:19:22 +00:00
|
|
|
public long getFeatureCount() {
|
|
|
|
long numFeatures = 0;
|
2021-04-23 11:26:02 +00:00
|
|
|
for (String table : tableNames()) {
|
|
|
|
try (
|
|
|
|
var stmt = conn.createStatement();
|
2022-04-23 09:58:49 +00:00
|
|
|
@SuppressWarnings("java:S2077") // table name checked against a regex
|
|
|
|
var result = stmt.executeQuery("SELECT COUNT(*) FROM %S WHERE GEOMETRY IS NOT NULL;".formatted(table))
|
2021-04-23 11:26:02 +00:00
|
|
|
) {
|
2022-12-15 19:19:22 +00:00
|
|
|
numFeatures += result.getLong(1);
|
2021-04-23 11:26:02 +00:00
|
|
|
} catch (SQLException e) {
|
|
|
|
// maybe no GEOMETRY column?
|
|
|
|
}
|
|
|
|
}
|
2022-12-15 19:19:22 +00:00
|
|
|
return numFeatures;
|
2021-04-11 20:10:28 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2022-12-15 19:19:22 +00:00
|
|
|
public void readFeatures(Consumer<SimpleFeature> next) throws Exception {
|
|
|
|
long id = 0;
|
|
|
|
// pass every element in every table through the profile
|
|
|
|
var tables = tableNames();
|
|
|
|
for (int i = 0; i < tables.size(); i++) {
|
|
|
|
String table = tables.get(i);
|
|
|
|
LOGGER.trace("Naturalearth loading {}/{}: {}", i, tables.size(), table);
|
|
|
|
|
|
|
|
try (Statement statement = conn.createStatement()) {
|
|
|
|
@SuppressWarnings("java:S2077") // table name checked against a regex
|
|
|
|
ResultSet rs = statement.executeQuery("SELECT * FROM %s;".formatted(table));
|
|
|
|
String[] column = new String[rs.getMetaData().getColumnCount()];
|
|
|
|
int geometryColumn = -1;
|
|
|
|
for (int c = 0; c < column.length; c++) {
|
|
|
|
String name = rs.getMetaData().getColumnName(c + 1);
|
|
|
|
column[c] = name;
|
|
|
|
if ("GEOMETRY".equals(name)) {
|
|
|
|
geometryColumn = c;
|
2021-04-23 11:26:02 +00:00
|
|
|
}
|
2022-12-15 19:19:22 +00:00
|
|
|
}
|
|
|
|
if (geometryColumn >= 0) {
|
|
|
|
while (rs.next()) {
|
|
|
|
byte[] geometry = rs.getBytes(geometryColumn + 1);
|
|
|
|
if (geometry == null) {
|
|
|
|
continue;
|
|
|
|
}
|
2021-09-10 00:46:20 +00:00
|
|
|
|
2022-12-15 19:19:22 +00:00
|
|
|
// create the feature and pass to next stage
|
|
|
|
Geometry latLonGeometry = GeoUtils.WKB_READER.read(geometry);
|
|
|
|
SimpleFeature readerGeometry = SimpleFeature.create(latLonGeometry, new HashMap<>(column.length - 1),
|
|
|
|
sourceName, table, ++id);
|
|
|
|
for (int c = 0; c < column.length; c++) {
|
|
|
|
if (c != geometryColumn) {
|
|
|
|
Object value = rs.getObject(c + 1);
|
|
|
|
String key = column[c];
|
|
|
|
readerGeometry.setTag(key, value);
|
2021-04-23 11:26:02 +00:00
|
|
|
}
|
|
|
|
}
|
2022-12-15 19:19:22 +00:00
|
|
|
next.accept(readerGeometry);
|
2021-04-23 11:26:02 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2022-12-15 19:19:22 +00:00
|
|
|
}
|
2021-04-10 09:25:42 +00:00
|
|
|
}
|
2021-04-22 21:19:58 +00:00
|
|
|
|
|
|
|
@Override
|
|
|
|
public void close() {
|
2021-04-23 11:26:02 +00:00
|
|
|
try {
|
|
|
|
conn.close();
|
|
|
|
} catch (SQLException e) {
|
|
|
|
LOGGER.error("Error closing sqlite file", e);
|
|
|
|
}
|
2023-03-19 18:01:17 +00:00
|
|
|
if (!keepUnzipped && extracted != null) {
|
2021-09-10 00:46:20 +00:00
|
|
|
FileUtils.deleteFile(extracted);
|
2021-04-23 11:26:02 +00:00
|
|
|
}
|
2021-04-22 21:19:58 +00:00
|
|
|
}
|
2021-04-10 09:25:42 +00:00
|
|
|
}
|