From 5588fca3b216839e4fbd929e48cfd2bc25bdc0dd Mon Sep 17 00:00:00 2001 From: Harel M Date: Fri, 17 Jan 2025 12:43:20 +0200 Subject: [PATCH] Add support for geojson (#1147) --- planetiler-core/pom.xml | 10 +++ .../com/onthegomap/planetiler/Planetiler.java | 25 ++++++ .../planetiler/reader/GeoJsonReader.java | 86 +++++++++++++++++++ .../planetiler/PlanetilerTests.java | 2 + .../planetiler/reader/GeoJsonReaderTest.java | 46 ++++++++++ .../src/test/resources/geojson.geojson | 37 ++++++++ planetiler-custommap/planetiler.schema.json | 3 +- .../custommap/ConfiguredMapMain.java | 1 + .../configschema/DataSourceType.java | 4 +- 9 files changed, 212 insertions(+), 2 deletions(-) create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/reader/GeoJsonReader.java create mode 100644 planetiler-core/src/test/java/com/onthegomap/planetiler/reader/GeoJsonReaderTest.java create mode 100644 planetiler-core/src/test/resources/geojson.geojson diff --git a/planetiler-core/pom.xml b/planetiler-core/pom.xml index f3fda136be..d23941f903 100644 --- a/planetiler-core/pom.xml +++ b/planetiler-core/pom.xml @@ -65,6 +65,16 @@ gt-epsg-hsql ${geotools.version} + + org.geotools + gt-geojson + ${geotools.version} + + + org.geotools + gt-geojson-store + ${geotools.version} + org.xerial sqlite-jdbc diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java index 47b250879c..2969ddd123 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java @@ -10,6 +10,7 @@ import com.onthegomap.planetiler.collection.LongLongMultimap; import com.onthegomap.planetiler.config.Arguments; import com.onthegomap.planetiler.config.PlanetilerConfig; +import com.onthegomap.planetiler.reader.GeoJsonReader; import com.onthegomap.planetiler.reader.GeoPackageReader; import com.onthegomap.planetiler.reader.NaturalEarthReader; import com.onthegomap.planetiler.reader.ShapefileReader; @@ -435,6 +436,30 @@ public Planetiler addGeoPackageSource(String name, Path defaultPath, String defa return addGeoPackageSource(null, name, defaultPath, defaultUrl); } + /** + * Adds a new GeoJSON source that will be processed when {@link #run()} is called. + *

+ * If the file does not exist and {@code download=true} argument is set, then the file will first be downloaded from + * {@code defaultUrl}. + *

+ * To override the location of the {@code geojson} file, set {@code name_path=newpath.geojson} in the arguments and to + * override the download URL set {@code name_url=http://url/of/file.geojson}. + * + * @param name string to use in stats and logs to identify this stage + * @param defaultPath path to the input file to use if {@code name_path} key is not set through arguments + * @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and {@code + * name_url} argument is not set + * @return this runner instance for chaining + * @see GeoJsonReader + * @see Downloader + */ + public Planetiler addGeoJsonSource(String name, Path defaultPath, String defaultUrl) { + Path path = getPath(name, "geojson", defaultPath, defaultUrl); + return addStage(name, "Process features in " + path, + ifSourceUsed(name, + () -> GeoJsonReader.process(name, List.of(path), featureGroup, config, profile, stats))); + } + /** * Adds a new Natural Earth sqlite file source that will be processed when {@link #run()} is called. *

diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/GeoJsonReader.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/GeoJsonReader.java new file mode 100644 index 0000000000..a7e974ce27 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/GeoJsonReader.java @@ -0,0 +1,86 @@ +package com.onthegomap.planetiler.reader; + +import com.onthegomap.planetiler.Profile; +import com.onthegomap.planetiler.collection.FeatureGroup; +import com.onthegomap.planetiler.config.PlanetilerConfig; +import com.onthegomap.planetiler.stats.Stats; +import java.io.IOException; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.function.Consumer; +import org.geotools.data.geojson.store.GeoJSONDataStore; +import org.geotools.data.simple.SimpleFeatureCollection; +import org.locationtech.jts.geom.Geometry; + +/** + * Utility that reads {@link SourceFeature SourceFeatures} from the vector geometries contained in a GeoJSON file. + */ +public class GeoJsonReader extends SimpleReader { + + private final GeoJSONDataStore store; + private final String layer; + + GeoJsonReader(String sourceName, Path input) { + super(sourceName); + store = new GeoJSONDataStore(input.toFile()); + layer = input.getFileName().toString().replaceFirst("\\.[^.]+$", ""); // remove file extention. + } + + /** + * Renders map features for all elements from an GeoJSON on the mapping logic defined in {@code + * profile}. + * + * @param sourceName string ID for this reader to use in logs and stats + * @param sourcePaths paths to the {@code .geojson} files on disk + * @param writer consumer for rendered features + * @param config user-defined parameters controlling number of threads and log interval + * @param profile logic that defines what map features to emit for each source feature + * @param stats to keep track of counters and timings + * @throws IllegalArgumentException if a problem occurs reading the input file + */ + public static void process(String sourceName, List sourcePaths, FeatureGroup writer, PlanetilerConfig config, + Profile profile, Stats stats) { + SourceFeatureProcessor.processFiles( + sourceName, + sourcePaths, + path -> new GeoJsonReader(sourceName, path), + writer, config, profile, stats + ); + } + + @Override + public void close() throws IOException { + store.dispose(); + } + + @Override + public long getFeatureCount() { + String typeName; + try { + typeName = store.getTypeNames()[0]; + SimpleFeatureCollection features = store.getFeatureSource(typeName).getFeatures(); + return Long.valueOf(features.size()); + } catch (IOException e) { + return 0; + } + } + + @Override + public void readFeatures(Consumer next) throws Exception { + long id = 0; + String typeName = store.getTypeNames()[0]; + SimpleFeatureCollection features = store.getFeatureSource(typeName).getFeatures(); + + try (var iter = features.features()) { + while (iter.hasNext()) { + var feature = iter.next(); + var properties = feature.getProperties(); + SimpleFeature simpleFeature = SimpleFeature.create((Geometry) feature.getDefaultGeometry(), HashMap.newHashMap(properties.size()), + sourceName, layer, id++); + properties.forEach(property -> simpleFeature.setTag(property.getName().toString(), property.getValue())); + next.accept(simpleFeature); + } + } + } +} diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java index 305b472a84..dfce6a7f17 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.dataformat.csv.CsvMapper; import com.fasterxml.jackson.dataformat.csv.CsvSchema; +import com.onthegomap.planetiler.TestUtils.OsmXml; import com.onthegomap.planetiler.archive.ReadableTileArchive; import com.onthegomap.planetiler.archive.TileArchiveConfig; import com.onthegomap.planetiler.archive.TileArchiveMetadata; @@ -2194,6 +2195,7 @@ public void processFeature(SourceFeature source, FeatureCollector features) { .addNaturalEarthSource("ne", TestUtils.pathToResource("natural_earth_vector.sqlite")) .addShapefileSource("shapefile", TestUtils.pathToResource("shapefile.zip")) .addGeoPackageSource("geopackage", TestUtils.pathToResource("geopackage.gpkg.zip"), null) + .addGeoJsonSource("geojson", TestUtils.pathToResource("geojson.geojson"), null) .setOutput(outputUri) .run(); diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/GeoJsonReaderTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/GeoJsonReaderTest.java new file mode 100644 index 0000000000..94a0ce27ec --- /dev/null +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/GeoJsonReaderTest.java @@ -0,0 +1,46 @@ +package com.onthegomap.planetiler.reader; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.onthegomap.planetiler.TestUtils; +import com.onthegomap.planetiler.geo.GeoUtils; +import com.onthegomap.planetiler.stats.Stats; +import com.onthegomap.planetiler.worker.WorkerPipeline; +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import org.junit.jupiter.api.Test; +import org.locationtech.jts.geom.Geometry; + +public class GeoJsonReaderTest { + @Test + void testReadGeoJson() throws IOException { + Path path = TestUtils.pathToResource("geojson.geojson"); + try (var reader = new GeoJsonReader("test", path)) { + assertEquals(3, reader.getFeatureCount()); + List points = new CopyOnWriteArrayList<>(); + List names = new CopyOnWriteArrayList<>(); + WorkerPipeline.start("test", Stats.inMemory()) + .fromGenerator("source", reader::readFeatures) + .addBuffer("reader_queue", 100, 1) + .sinkToConsumer("counter", 1, elem -> { + assertTrue(elem.getTag("name") instanceof String); + assertEquals("test", elem.getSource()); + assertEquals("geojson", elem.getSourceLayer()); + points.add(elem.latLonGeometry()); + names.add(elem.getTag("name").toString()); + }).await(); + assertEquals(3, points.size()); + assertTrue(names.contains("line")); + assertTrue(names.contains("point")); + assertTrue(names.contains("polygon")); + var gc = GeoUtils.JTS_FACTORY.createGeometryCollection(points.toArray(new Geometry[0])); + var centroid = gc.getCentroid(); + assertEquals(100.5, centroid.getX(), 1e-5); + assertEquals(0.5, centroid.getY(), 1e-5); + } + } +} + diff --git a/planetiler-core/src/test/resources/geojson.geojson b/planetiler-core/src/test/resources/geojson.geojson new file mode 100644 index 0000000000..e01b98d2a2 --- /dev/null +++ b/planetiler-core/src/test/resources/geojson.geojson @@ -0,0 +1,37 @@ +{ + "type": "FeatureCollection", + "features": [{ + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [102.0, 0.5] + }, + "properties": { + "name": "point" + } + }, { + "type": "Feature", + "geometry": { + "type": "LineString", + "coordinates": [[102.0, 0.0], [103.0, 1.0], [104.0, 0.0], [105.0, 1.0]] + }, + "properties": { + "name": "line", + "prop1": 0.0 + } + }, { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [100.0, 0.0], [101.0, 0.0], [101.0, 1.0], + [100.0, 1.0], [100.0, 0.0] + ] + ] + }, + "properties": { + "name": "polygon" + } + }] +} \ No newline at end of file diff --git a/planetiler-custommap/planetiler.schema.json b/planetiler-custommap/planetiler.schema.json index e4863ea1e9..6de6b8668a 100644 --- a/planetiler-custommap/planetiler.schema.json +++ b/planetiler-custommap/planetiler.schema.json @@ -41,7 +41,8 @@ "enum": [ "osm", "shapefile", - "geopackage" + "geopackage", + "geojson" ] }, "url": { diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredMapMain.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredMapMain.java index c4a11437aa..ac5aa6f333 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredMapMain.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredMapMain.java @@ -74,6 +74,7 @@ private static void configureSource(Planetiler planetiler, Path sourcesDir, Sour case OSM -> planetiler.addOsmSource(source.id(), localPath, source.url()); case SHAPEFILE -> planetiler.addShapefileSource(projection, source.id(), localPath, source.url()); case GEOPACKAGE -> planetiler.addGeoPackageSource(projection, source.id(), localPath, source.url()); + case GEOJSON -> planetiler.addGeoJsonSource(source.id(), localPath, source.url()); default -> throw new IllegalArgumentException("Unhandled source type for " + source.id() + ": " + sourceType); } } diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/DataSourceType.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/DataSourceType.java index 39e12c3fe0..ddda1293b9 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/DataSourceType.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/DataSourceType.java @@ -8,5 +8,7 @@ public enum DataSourceType { @JsonProperty("shapefile") SHAPEFILE, @JsonProperty("geopackage") - GEOPACKAGE + GEOPACKAGE, + @JsonProperty("geojson") + GEOJSON }