decouple celliterator from grid implementation

CHANGELOG.md

@@ -7,6 +7,10 @@ Changelog
 
 * `OSMEntitySnapshot` now also returns the `lastContributionTimestamp` for each snapshot ([#495])
 
+### other changes
+
+* `CellIterator` is now decoupled from implementation of the "Grid" ([#495])
+
 [#495]: https://github.com/GIScience/oshdb/pull/495
 
 

oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/Kernels.java

@@ -12,6 +12,7 @@
 import org.heigit.ohsome.oshdb.api.object.OSMEntitySnapshotImpl;
 import org.heigit.ohsome.oshdb.grid.GridOSHEntity;
 import org.heigit.ohsome.oshdb.util.celliterator.CellIterator;
+import org.heigit.ohsome.oshdb.util.celliterator.OSHEntitySource;
 import org.heigit.ohsome.oshdb.util.function.SerializableBiFunction;
 import org.heigit.ohsome.oshdb.util.function.SerializableFunction;
 import org.heigit.ohsome.oshdb.util.function.SerializableSupplier;
@@ -60,7 +61,7 @@ static <R, S> CellProcessor<S> getOSMContributionCellReducer(
     return (oshEntityCell, cellIterator) -> {
       // iterate over the history of all OSM objects in the current cell
       AtomicReference<S> accInternal = new AtomicReference<>(identitySupplier.get());
-      cellIterator.iterateByContribution(oshEntityCell)
+      cellIterator.iterateByContribution(OSHEntitySource.fromGridOSHEntity(oshEntityCell))
           .takeWhile(process::isActive)
           .forEach(contribution -> {
             OSMContribution osmContribution = new OSMContributionImpl(contribution);
@@ -90,7 +91,7 @@ static <R, S> CellProcessor<S> getOSMContributionGroupingCellReducer(
       AtomicReference<S> accInternal = new AtomicReference<>(identitySupplier.get());
       // iterate over the history of all OSM objects in the current cell
       List<OSMContribution> contributions = new ArrayList<>();
-      cellIterator.iterateByContribution(oshEntityCell)
+      cellIterator.iterateByContribution(OSHEntitySource.fromGridOSHEntity(oshEntityCell))
           .takeWhile(process::isActive)
           .forEach(contribution -> {
             OSMContribution thisContribution = new OSMContributionImpl(contribution);
@@ -134,7 +135,7 @@ static <R, S> CellProcessor<S> getOSMEntitySnapshotCellReducer(
     return (oshEntityCell, cellIterator) -> {
       // iterate over the history of all OSM objects in the current cell
       AtomicReference<S> accInternal = new AtomicReference<>(identitySupplier.get());
-      cellIterator.iterateByTimestamps(oshEntityCell)
+      cellIterator.iterateByTimestamps(OSHEntitySource.fromGridOSHEntity(oshEntityCell))
           .takeWhile(process::isActive)
           .forEach(data -> {
             OSMEntitySnapshot snapshot = new OSMEntitySnapshotImpl(data);
@@ -165,7 +166,7 @@ static <R, S> CellProcessor<S> getOSMEntitySnapshotGroupingCellReducer(
       // iterate over the history of all OSM objects in the current cell
       AtomicReference<S> accInternal = new AtomicReference<>(identitySupplier.get());
       List<OSMEntitySnapshot> osmEntitySnapshots = new ArrayList<>();
-      cellIterator.iterateByTimestamps(oshEntityCell)
+      cellIterator.iterateByTimestamps(OSHEntitySource.fromGridOSHEntity(oshEntityCell))
           .takeWhile(process::isActive)
           .forEach(data -> {
             OSMEntitySnapshot thisSnapshot = new OSMEntitySnapshotImpl(data);
@@ -206,7 +207,7 @@ static <S> CellProcessor<Stream<S>> getOSMContributionCellStreamer(
   ) {
     return (oshEntityCell, cellIterator) -> {
       // iterate over the history of all OSM objects in the current cell
-      return cellIterator.iterateByContribution(oshEntityCell)
+      return cellIterator.iterateByContribution(OSHEntitySource.fromGridOSHEntity(oshEntityCell))
           .takeWhile(process::isActive)
           .map(OSMContributionImpl::new)
           .map(mapper);
@@ -229,7 +230,7 @@ static <S> CellProcessor<Stream<S>> getOSMContributionGroupingCellStreamer(
       // iterate over the history of all OSM objects in the current cell
       List<OSMContribution> contributions = new ArrayList<>();
       List<S> result = new LinkedList<>();
-      cellIterator.iterateByContribution(oshEntityCell)
+      cellIterator.iterateByContribution(OSHEntitySource.fromGridOSHEntity(oshEntityCell))
           .takeWhile(process::isActive)
           .map(OSMContributionImpl::new)
           .forEach(contribution -> {
@@ -263,7 +264,7 @@ static <S> CellProcessor<Stream<S>> getOSMEntitySnapshotCellStreamer(
   ) {
     return (oshEntityCell, cellIterator) -> {
       // iterate over the history of all OSM objects in the current cell
-      return cellIterator.iterateByTimestamps(oshEntityCell)
+      return cellIterator.iterateByTimestamps(OSHEntitySource.fromGridOSHEntity(oshEntityCell))
           .takeWhile(process::isActive)
           .map(OSMEntitySnapshotImpl::new)
           .map(mapper);
@@ -286,7 +287,7 @@ static <S> CellProcessor<Stream<S>> getOSMEntitySnapshotGroupingCellStreamer(
       // iterate over the history of all OSM objects in the current cell
       List<OSMEntitySnapshot> snapshots = new ArrayList<>();
       List<S> result = new LinkedList<>();
-      cellIterator.iterateByTimestamps(oshEntityCell)
+      cellIterator.iterateByTimestamps(OSHEntitySource.fromGridOSHEntity(oshEntityCell))
           .takeWhile(process::isActive)
           .map(OSMEntitySnapshotImpl::new)
           .forEach(contribution -> {

oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/celliterator/CellIterator.java

@@ -229,28 +229,28 @@ public record IterateByTimestampEntry(
   ) {}
 
   /**
-   * Helper method to easily iterate over all entities in a cell that match a given condition/filter
+   * Helper method to easily iterate over all entities that match a given condition/filter
    * as they existed at the given timestamps.
    *
-   * @param cell the data cell
+   * @param source a provider of a stream of OSHEntity objects and a corresponding bounding box
    *
    * @return a stream of matching filtered OSMEntities with their clipped Geometries at each
    *         timestamp. If an object has not been modified between timestamps, the output may
    *         contain the *same* Geometry object in the output multiple times. This can be used to
    *         optimize away recalculating expensive geometry operations on unchanged feature
    *         geometries later on in the code.
    */
-  public Stream<IterateByTimestampEntry> iterateByTimestamps(GridOSHEntity cell) {
-    var cellBoundingBox = XYGrid.getBoundingBox(new CellId(cell.getLevel(), cell.getId()), true);
+  public Stream<IterateByTimestampEntry> iterateByTimestamps(OSHEntitySource source) {
+    var cellBoundingBox = source.getBoundingBox();
     final boolean allFullyInside = fullyInside(cellBoundingBox);
     if (!allFullyInside && isBoundByPolygon && bboxOutsidePolygon.test(cellBoundingBox)) {
       return Stream.empty();
     }
-    return iterateByTimestamps(Streams.stream(cell.getEntities()), allFullyInside);
+    return iterateByTimestamps(source.getData(), allFullyInside);
   }
 
   /**
-   * Helper method to easily iterate over all entities in a cell that match a given condition/filter
+   * Helper method to easily iterate over all entities that match a given condition/filter
    * as they existed at the given timestamps.
    *
    * @param oshData the entities to iterate through
@@ -494,25 +494,25 @@ public record IterateAllEntry(
   ) {}
 
   /**
-   * Helper method to easily iterate over all entity modifications in a cell that match a given
+   * Helper method to easily iterate over all entity modifications that match a given
    * condition/filter.
    *
-   * @param cell the data cell
+   * @param source a provider of a stream of OSHEntity objects and a corresponding bounding box
    *
    * @return a stream of matching filtered OSMEntities with their clipped Geometries and timestamp
    *         intervals.
    */
-  public Stream<IterateAllEntry> iterateByContribution(GridOSHEntity cell) {
-    var cellBoundingBox = XYGrid.getBoundingBox(new CellId(cell.getLevel(), cell.getId()), true);
+  public Stream<IterateAllEntry> iterateByContribution(OSHEntitySource source) {
+    var cellBoundingBox = source.getBoundingBox();
     final boolean allFullyInside = fullyInside(cellBoundingBox);
     if (!allFullyInside && isBoundByPolygon && bboxOutsidePolygon.test(cellBoundingBox)) {
       return Stream.empty();
     }
-    return iterateByContribution(Streams.stream(cell.getEntities()), allFullyInside);
+    return iterateByContribution(source.getData(), allFullyInside);
   }
 
   /**
-   * Helper method to easily iterate over all entity modifications in a cell that match a given
+   * Helper method to easily iterate over all entity modifications that match a given
    * condition/filter.
    *
    * @param oshData the entities to iterate through

oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/celliterator/OSHEntitySource.java

@@ -0,0 +1,52 @@
+package org.heigit.ohsome.oshdb.util.celliterator;
+
+import com.google.common.collect.Streams;
+import java.util.stream.Stream;
+import org.heigit.ohsome.oshdb.OSHDBBoundingBox;
+import org.heigit.ohsome.oshdb.grid.GridOSHEntity;
+import org.heigit.ohsome.oshdb.index.XYGrid;
+import org.heigit.ohsome.oshdb.osh.OSHEntity;
+import org.heigit.ohsome.oshdb.util.CellId;
+
+/**
+ * A source of OSH entities.
+ */
+public interface OSHEntitySource {
+
+  /**
+   * Returns a stream of OSH entities.
+   *
+   * @return a stream of OSH entities
+   */
+  Stream<? extends OSHEntity> getData();
+
+  /**
+   * Returns the bounding box of the entities returned by `getData()`.
+   *
+   * <p>By convention this bbox must contain the bboxes of all OSH entities returned by this source.
+   * It should be the minimal bbox encompassing all of the entities.</p>
+   *
+   * @return A bounding box enclosing all OSH entities of this source
+   */
+  OSHDBBoundingBox getBoundingBox();
+
+  /**
+   * A helper method which transforms a grid cell to an OSH entity source.
+   *
+   * @param cell A grid cell containing OSH entities
+   * @return A source object which will return the entities of the given grid cell and its bbox
+   */
+  static OSHEntitySource fromGridOSHEntity(GridOSHEntity cell) {
+    return new OSHEntitySource() {
+      @Override
+      public Stream<? extends OSHEntity> getData() {
+        return Streams.stream(cell.getEntities());
+      }
+
+      @Override
+      public OSHDBBoundingBox getBoundingBox() {
+        return XYGrid.getBoundingBox(new CellId(cell.getLevel(), cell.getId()), true);
+      }
+    };
+  }
+}

oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByContributionNodesTest.java

@@ -9,7 +9,6 @@
 import java.util.EnumSet;
 import java.util.List;
 import org.heigit.ohsome.oshdb.OSHDBBoundingBox;
-import org.heigit.ohsome.oshdb.grid.GridOSHEntity;
 import org.heigit.ohsome.oshdb.grid.GridOSHNodes;
 import org.heigit.ohsome.oshdb.index.XYGrid;
 import org.heigit.ohsome.oshdb.util.celliterator.CellIterator.IterateAllEntry;
@@ -26,7 +25,7 @@
 import org.locationtech.jts.geom.Polygon;
 
 /**
- * Tests the {@link CellIterator#iterateByContribution(GridOSHEntity)} method on nodes.
+ * Tests the {@link CellIterator#iterateByContribution(OSHEntitySource)} method on nodes.
  */
 class IterateByContributionNodesTest {
   private final GridOSHNodes oshdbDataGridCell;
@@ -58,7 +57,7 @@ void testGeometryChange() {
         osmEntity -> true,
         false
     )).iterateByContribution(
-        oshdbDataGridCell
+        OSHEntitySource.fromGridOSHEntity(oshdbDataGridCell)
     ).toList();
 
     assertEquals(3, result.size());
@@ -98,7 +97,7 @@ void testTagChange() {
         osmEntity -> true,
         false
     )).iterateByContribution(
-        oshdbDataGridCell
+        OSHEntitySource.fromGridOSHEntity(oshdbDataGridCell)
     ).toList();
 
     assertEquals(3, result.size());
@@ -134,7 +133,7 @@ void testVisibleChange() {
         osmEntity -> true,
         false
     )).iterateByContribution(
-        oshdbDataGridCell
+        OSHEntitySource.fromGridOSHEntity(oshdbDataGridCell)
     ).toList();
 
     assertEquals(5, result.size());
@@ -181,7 +180,7 @@ void testMultipleChanges() {
         osmEntity -> true,
         false
     )).iterateByContribution(
-        oshdbDataGridCell
+        OSHEntitySource.fromGridOSHEntity(oshdbDataGridCell)
     ).toList();
 
     assertEquals(6, result.size());
@@ -231,7 +230,7 @@ void testBboxMinAndMaxNotCorrect() {
         osmEntity -> true,
         false
     )).iterateByContribution(
-        oshdbDataGridCell
+        OSHEntitySource.fromGridOSHEntity(oshdbDataGridCell)
     ).toList();
     assertTrue(result.isEmpty());
   }
@@ -251,7 +250,7 @@ void testBboxMinExactlyAtDataMinMaxExcluded() {
         osmEntity -> true,
         false
     )).iterateByContribution(
-        oshdbDataGridCell
+        OSHEntitySource.fromGridOSHEntity(oshdbDataGridCell)
     ).toList();
     assertTrue(result.isEmpty());
   }
@@ -271,7 +270,7 @@ void testBboxMaxExactlyAtDataMaxMinExcluded() {
         osmEntity -> true,
         false
     )).iterateByContribution(
-        oshdbDataGridCell
+        OSHEntitySource.fromGridOSHEntity(oshdbDataGridCell)
     ).toList();
     assertTrue(result.isEmpty());
   }
@@ -291,7 +290,7 @@ void testBboxMinMaxExactlyAtDataMinMax() {
         osmEntity -> true,
         false
     )).iterateByContribution(
-        oshdbDataGridCell
+        OSHEntitySource.fromGridOSHEntity(oshdbDataGridCell)
     ).toList();
     assertEquals(3, result.size());
   }
@@ -311,7 +310,7 @@ void testTagChangeTagFilterWithSuccess() {
         osmEntity -> osmEntity.getTags().hasTagKey(osmXmlTestData.keys().get("shop")),
         false
     )).iterateByContribution(
-        oshdbDataGridCell
+        OSHEntitySource.fromGridOSHEntity(oshdbDataGridCell)
     ).toList();
 
     assertEquals(4, result.size());
@@ -347,7 +346,7 @@ void testTagChangeTagFilterDisused() {
         osmEntity -> osmEntity.getTags().hasTagKey(osmXmlTestData.keys().get("disused:shop")),
         false
     )).iterateByContribution(
-        oshdbDataGridCell
+        OSHEntitySource.fromGridOSHEntity(oshdbDataGridCell)
     ).toList();
 
     assertEquals(3, result.size());
@@ -379,7 +378,7 @@ void testMoreComplicatedFilter() {
         osmEntity -> osmEntity.getTags().hasTagKey(osmXmlTestData.keys().get("shop")),
         false
     )).iterateByContribution(
-        oshdbDataGridCell
+        OSHEntitySource.fromGridOSHEntity(oshdbDataGridCell)
     ).toList();
 
     assertEquals(4, result.size());
@@ -413,7 +412,7 @@ void testTagChangeTagFilterWithoutSuccess() {
             osmXmlTestData.keys().getOrDefault("amenity", -1)),
         false
     )).iterateByContribution(
-        oshdbDataGridCell
+        OSHEntitySource.fromGridOSHEntity(oshdbDataGridCell)
     ).toList();
     assertTrue(result.isEmpty());
   }
@@ -441,7 +440,7 @@ void testPolygonIntersectingDataPartly() {
         osmEntity -> true,
         false
     )).iterateByContribution(
-        oshdbDataGridCell
+        OSHEntitySource.fromGridOSHEntity(oshdbDataGridCell)
     ).toList();
     assertEquals(2, result.size());
   }
@@ -471,7 +470,7 @@ void testTagFilterAndPolygonIntersectingDataPartly() {
         osmEntity -> osmEntity.getTags().hasTagKey(osmXmlTestData.keys().get("shop")),
         false
     )).iterateByContribution(
-        oshdbDataGridCell
+        OSHEntitySource.fromGridOSHEntity(oshdbDataGridCell)
     ).toList();
     // result size =2 because if tag filtered for disappears it's a deletion
     assertEquals(2, result.size()); // one version with tag shop
@@ -498,8 +497,10 @@ void testCoordinatesRelativeToPolygon() throws IOException {
         oshEntity -> oshEntity.getId() >= 10 && oshEntity.getId() < 20,
         osmEntity -> true,
         false
-    )).iterateByContribution(GridOSHFactory.getGridOSHNodes(osmXmlTestData, 6, (new XYGrid(6))
-            .getId(1.0, 1.0)/* approx. 0, 0, 5.6, 5.6*/)).toList();
+    )).iterateByContribution(OSHEntitySource.fromGridOSHEntity(
+        GridOSHFactory.getGridOSHNodes(osmXmlTestData, 6, (new XYGrid(6))
+            .getId(1.0, 1.0)/* approx. 0, 0, 5.6, 5.6*/)
+    )).toList();
 
     assertEquals(2, result.size());
     assertEquals(13, result.get(0).osmEntity().getId());