Tile hashing fix

planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java

@@ -523,6 +523,19 @@ private boolean containsOnlyFillsOrEdges(boolean allowEdges) {
     return !empty;
   }
 
+  public boolean likelyToBeDuplicate() {
+    if (layers.size() <= 0) {
+      return true;
+    } else if (layers.size() == 1 && layers.values().stream().findFirst().get().encodedFeatures.isEmpty()) {
+      return true;
+    }
+    var result = containsOnlyFillsOrEdges();
+    if (result) {
+      return true;
+    }
+    return result;
+  }
+
   private enum Command {
     MOVE_TO(1),
     LINE_TO(2),

planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java

@@ -12,7 +12,6 @@
 import com.onthegomap.planetiler.util.CloseableConusmer;
 import com.onthegomap.planetiler.util.CommonStringEncoder;
 import com.onthegomap.planetiler.util.DiskBacked;
-import com.onthegomap.planetiler.util.Hashing;
 import com.onthegomap.planetiler.util.LayerStats;
 import com.onthegomap.planetiler.worker.Worker;
 import java.io.Closeable;
@@ -368,22 +367,6 @@ public TileCoord tileCoord() {
       return tileCoord;
     }
 
-    /**
-     * Generates a hash over the feature's relevant data: layer, geometry, and attributes. The coordinates are
-     * <b>not</b> part of the hash.
-     * <p>
-     * Used as an optimization to avoid writing the same (ocean) tiles over and over again.
-     */
-    public long generateContentHash() {
-      long hash = Hashing.FNV1_64_INIT;
-      for (var feature : entries) {
-        byte layerId = extractLayerIdFromKey(feature.key());
-        hash = Hashing.fnv1a64(hash, layerId);
-        hash = Hashing.fnv1a64(hash, feature.value());
-      }
-      return hash;
-    }
-
     /**
      * Returns true if {@code other} contains features with identical layer, geometry, and attributes, as this tile -
      * even if the tiles have separate coordinates.

planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/MbtilesWriter.java

@@ -16,6 +16,7 @@
 import com.onthegomap.planetiler.util.DiskBacked;
 import com.onthegomap.planetiler.util.FileUtils;
 import com.onthegomap.planetiler.util.Format;
+import com.onthegomap.planetiler.util.Hashing;
 import com.onthegomap.planetiler.util.LayerStats;
 import com.onthegomap.planetiler.worker.WorkQueue;
 import com.onthegomap.planetiler.worker.Worker;
@@ -289,8 +290,8 @@ private void tileEncoder(Iterable<TileBatch> prev, Consumer<TileBatch> next) thr
           lastEncoded = encoded;
           lastBytes = bytes;
           last = tileFeatures;
-          if (compactDb && en.containsOnlyFillsOrEdges()) {
-            tileDataHash = tileFeatures.generateContentHash();
+          if (compactDb && en.likelyToBeDuplicate() && bytes != null) {
+            tileDataHash = generateContentHash(bytes);
           } else {
             tileDataHash = null;
           }
@@ -412,6 +413,17 @@ private long tilesEmitted() {
     return Stream.of(tilesByZoom).mapToLong(c -> c.get()).sum();
   }
 
+  /**
+   * Generates a hash over encoded and compressed tile.
+   * <p>
+   * Used as an optimization to avoid writing the same (mostly ocean) tiles over and over again.
+   */
+  public static long generateContentHash(byte[] bytes) {
+    long hash = Hashing.FNV1_64_INIT;
+    hash = Hashing.fnv1a64(hash, bytes);
+    return hash;
+  }
+
   /**
    * Container for a batch of tiles to be processed together in the encoder and writer threads.
    * <p>

planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java

@@ -27,10 +27,12 @@
 import java.io.UncheckedIOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.sql.Connection;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Comparator;
 import java.util.HashSet;
@@ -42,6 +44,7 @@
 import java.util.TreeSet;
 import java.util.function.Function;
 import java.util.stream.Collectors;
+import org.apache.commons.lang3.reflect.FieldUtils;
 import org.locationtech.jts.algorithm.Orientation;
 import org.locationtech.jts.geom.Coordinate;
 import org.locationtech.jts.geom.CoordinateSequence;
@@ -685,4 +688,33 @@ public static void assertFeatureNear(Mbtiles db, String layer, Map<String, Objec
       fail(e);
     }
   }
+
+  public static void assertTileDuplicates(Mbtiles db, int expected) {
+    try {
+      Connection connection = (Connection) FieldUtils.readField(db, "connection", true);
+      Statement statement = connection.createStatement();
+      ResultSet rs = statement.executeQuery("SELECT tile_data FROM tiles_data");
+      ArrayList<byte[]> tilesList = new ArrayList<>();
+      while (rs.next()) {
+        tilesList.add(rs.getBytes("tile_data"));
+      }
+
+      var tiles = tilesList.toArray(new byte[0][0]);
+      Set<Integer> dups = new HashSet<>();
+      for (int i = 0; i < tiles.length; i++) {
+        for (int j = i + 1; j < tiles.length; j++) {
+          if (Arrays.equals(tiles[i], tiles[j])) {
+            if (!dups.contains(j)) {
+              dups.add(j);
+            }
+          }
+        }
+      }
+
+      int dupCount = dups.size();
+      assertEquals(expected, dupCount, "%d duplicates expected, %d found".formatted(expected, dupCount));
+    } catch (IllegalAccessException | SQLException e) {
+      fail(e);
+    }
+  }
 }

planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureGroupTest.java

@@ -12,9 +12,12 @@
 import com.onthegomap.planetiler.VectorTile;
 import com.onthegomap.planetiler.geo.GeometryType;
 import com.onthegomap.planetiler.geo.TileCoord;
+import com.onthegomap.planetiler.mbtiles.MbtilesWriter;
 import com.onthegomap.planetiler.render.RenderedFeature;
 import com.onthegomap.planetiler.stats.Stats;
 import com.onthegomap.planetiler.util.CloseableConusmer;
+import com.onthegomap.planetiler.util.Gzip;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -356,17 +359,22 @@ void testHasSameContents(String testName, boolean expectSame, PuTileArgs args0,
 
   @ParameterizedTest(name = "{0}")
   @ArgumentsSource(SameFeatureGroupTestArgs.class)
-  void testGenerateContentHash(String testName, boolean expectSame, PuTileArgs args0, PuTileArgs args1) {
+  void testGenerateContentHash(String testName, boolean expectSame, PuTileArgs args0, PuTileArgs args1)
+    throws IOException {
     put(args0);
     put(args1);
     sorter.sort();
     var iter = features.iterator();
-    var tile0 = iter.next();
-    var tile1 = iter.next();
+    var tile0_hash = MbtilesWriter.generateContentHash(
+      Gzip.gzip(iter.next().getVectorTileEncoder().encode())
+    );
+    var tile1_hash = MbtilesWriter.generateContentHash(
+      Gzip.gzip(iter.next().getVectorTileEncoder().encode())
+    );
     if (expectSame) {
-      assertEquals(tile0.generateContentHash(), tile1.generateContentHash());
+      assertEquals(tile0_hash, tile1_hash);
     } else {
-      assertNotEquals(tile0.generateContentHash(), tile1.generateContentHash());
+      assertNotEquals(tile0_hash, tile1_hash);
     }
   }
 

planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/BikeRouteOverlayTest.java

@@ -110,6 +110,8 @@ void integrationTest(@TempDir Path tmpDir) throws Exception {
           "name", "EuroVelo 8 - Mediterranean Route - part Monaco",
           "ref", "EV8"
         ), GeoUtils.WORLD_LAT_LON_BOUNDS, 25, LineString.class);
+
+      TestUtils.assertTileDuplicates(mbtiles, 0);
     }
   }
 }