Expose pmtiles writer (#520)

onthegomap · Mar 18, 2023 · 74db638 · 74db638
1 parent 9945ad4
commit 74db638
Show file tree

Hide file tree

Showing 48 changed files with 1,374 additions and 652 deletions.
diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
@@ -85,7 +85,7 @@ jobs:
       run: mv target/*with-deps.jar ./run.jar
       working-directory: planetiler-examples
     - name: Run
-      run: java -jar run.jar --osm-path=../planetiler-core/src/test/resources/monaco-latest.osm.pbf --mbtiles=data/out.mbtiles
+      run: java -jar run.jar --osm-path=../planetiler-core/src/test/resources/monaco-latest.osm.pbf --output=data/out.mbtiles
       working-directory: planetiler-examples
     - name: Verify
       run: java -cp run.jar com.onthegomap.planetiler.mbtiles.Verify data/out.mbtiles

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
@@ -22,73 +22,73 @@ jobs:
     timeout-minutes: 20
     continue-on-error: true
     steps:
-      - name: 'Cancel previous runs'
-        uses: styfle/cancel-workflow-action@0.11.0
-        with:
-          access_token: ${{ github.token }}
-      - name: 'Checkout branch'
-        uses: actions/checkout@v3
-        with:
-          path: branch
-          submodules: true
-      - name: 'Checkout base'
-        uses: actions/checkout@v3
-        with:
-          path: base
-          ref: ${{ github.event.pull_request.base.sha }}
-          submodules: true
-      - name: Cache data/sources
-        uses: ./branch/.github/cache-sources-action
-        with:
-          basedir: branch
-      - name: Set up JDK
-        uses: actions/setup-java@v3
-        with:
-          java-version: 17
-          distribution: 'temurin'
-          cache: 'maven'
-      - uses: actions/setup-node@v3
-        with:
-          node-version: '14'
-      - run: npm install -g strip-ansi-cli@3.0.2
+    - name: 'Cancel previous runs'
+      uses: styfle/cancel-workflow-action@0.11.0
+      with:
+        access_token: ${{ github.token }}
+    - name: 'Checkout branch'
+      uses: actions/checkout@v3
+      with:
+        path: branch
+        submodules: true
+    - name: 'Checkout base'
+      uses: actions/checkout@v3
+      with:
+        path: base
+        ref: ${{ github.event.pull_request.base.sha }}
+        submodules: true
+    - name: Cache data/sources
+      uses: ./branch/.github/cache-sources-action
+      with:
+        basedir: branch
+    - name: Set up JDK
+      uses: actions/setup-java@v3
+      with:
+        java-version: 17
+        distribution: 'temurin'
+        cache: 'maven'
+    - uses: actions/setup-node@v3
+      with:
+        node-version: '14'
+    - run: npm install -g strip-ansi-cli@3.0.2
 
-      - name: 'Build branch'
-        run: ./scripts/build.sh
-        working-directory: branch
-      - name: 'Build base'
-        run: ./scripts/build.sh
-        working-directory: base
+    - name: 'Build branch'
+      run: ./scripts/build.sh
+      working-directory: branch
+    - name: 'Build base'
+      run: ./scripts/build.sh
+      working-directory: base
 
-      - name: 'Download data'
-        run: |
-          set -eo pipefail
-          cp base/planetiler-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}"
-          cp branch/planetiler-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}"
+    - name: 'Download data'
+      run: |
+        set -eo pipefail
+        cp base/planetiler-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}"
+        cp branch/planetiler-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}"
 
-      - name: 'Store build info'
-        run: |
-          mkdir build-info
-          echo "${{ github.event.pull_request.base.sha }}" > build-info/base_sha
-          echo "${{ github.sha }}" > build-info/branch_sha
-          echo "${{ github.event.number }}" > build-info/pull_request_number
+    - name: 'Store build info'
+      run: |
+        mkdir build-info
+        echo "${{ github.event.pull_request.base.sha }}" > build-info/base_sha
+        echo "${{ github.sha }}" > build-info/branch_sha
+        echo "${{ github.event.number }}" > build-info/pull_request_number
 
-      - name: 'Run branch'
-        run: |
-          rm -rf data/out.mbtiles data/tmp
-          cp branch/planetiler-dist/target/*with-deps.jar run.jar
-          java -Xms${{ env.RAM }} -Xmx${{ env.RAM }} -jar run.jar --area="${{ env.AREA }}" "${{ env.BOUNDS_ARG }}" --mbtiles=data/out.mbtiles 2>&1 | tee log
-          ls -alh run.jar | tee -a log
-          cat log | strip-ansi > build-info/branchlogs.txt
-      - name: 'Run base'
-        run: |
-          rm -rf data/out.mbtiles data/tmp
-          cp base/planetiler-dist/target/*with-deps.jar run.jar
-          java -Xms${{ env.RAM }} -Xmx${{ env.RAM }} -jar run.jar --area="${{ env.AREA }}" "${{ env.BOUNDS_ARG }}" --mbtiles=data/out.mbtiles 2>&1 | tee log
-          ls -alh run.jar | tee -a log
-          cat log | strip-ansi > build-info/baselogs.txt
+    - name: 'Run branch'
+      run: |
+        rm -rf data/out.mbtiles data/tmp
+        cp branch/planetiler-dist/target/*with-deps.jar run.jar
+        java -Xms${{ env.RAM }} -Xmx${{ env.RAM }} -jar run.jar --area="${{ env.AREA }}" "${{ env.BOUNDS_ARG }}" --output=data/out.mbtiles 2>&1 | tee log
+        ls -alh run.jar | tee -a log
+        cat log | strip-ansi > build-info/branchlogs.txt
+    - name: 'Run base'
+      run: |
+        rm -rf data/out.mbtiles data/tmp
+        cp base/planetiler-dist/target/*with-deps.jar run.jar
+        java -Xms${{ env.RAM }} -Xmx${{ env.RAM }} -jar run.jar --area="${{ env.AREA }}" "${{ env.BOUNDS_ARG }}" --output=data/out.mbtiles 2>&1 | tee log
+        ls -alh run.jar | tee -a log
+        cat log | strip-ansi > build-info/baselogs.txt
 
-      - name: 'Upload build-info'
-        uses: actions/upload-artifact@v3
-        with:
-          name: build-info
-          path: ./build-info
+    - name: 'Upload build-info'
+      uses: actions/upload-artifact@v3
+      with:
+        name: build-info
+        path: ./build-info
diff --git a/PLANET.md b/PLANET.md
@@ -50,7 +50,7 @@ java -Xmx110g \
   --download-threads=10 --download-chunk-size-mb=1000 \
   `# Also download name translations from wikidata` \
   --fetch-wikidata \
-  --mbtiles=output.mbtiles \
+  --output=output.mbtiles \
   `# Store temporary node locations in memory` \
   --nodemap-type=array --storage=ram
 ```
@@ -67,7 +67,7 @@ java -Xmx20g \
   --download-threads=10 --download-chunk-size-mb=1000 \
   `# Also download name translations from wikidata` \
   --fetch-wikidata \
-  --mbtiles=output.mbtiles \
+  --output=output.mbtiles \
   `# Store temporary node locations at fixed positions in a memory-mapped file` \
   --nodemap-type=array --storage=mmap
 ```
@@ -103,7 +103,7 @@ java -Xmx100g \
   --download-threads=10 --download-chunk-size-mb=1000 \
   `# Also download name translations from wikidata` \
   --fetch-wikidata \
-  --mbtiles=output.mbtiles \
+  --output=output.mbtiles \
   --nodemap-type=sparsearray --nodemap-storage=ram 2>&1 | tee logs.txt
 ```
 

diff --git a/README.md b/README.md
@@ -8,9 +8,10 @@ or database.
 
 Vector tiles contain raw point, line, and polygon geometries that clients like [MapLibre](https://github.com/maplibre)
 can use to render custom maps in the browser, native apps, or on a server. Planetiler packages tiles into
-an [MBTiles](https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md) (sqlite) file that can be served using
-tools like [TileServer GL](https://github.com/maptiler/tileserver-gl) or even
-[queried directly from the browser](https://github.com/phiresky/sql.js-httpvfs).
+an [MBTiles](https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md) (sqlite)
+or [PMTiles](https://github.com/protomaps/PMTiles) file that can be served using tools
+like [TileServer GL](https://github.com/maptiler/tileserver-gl) or [Martin](https://github.com/maplibre/martin) or
+even [queried directly from the browser](https://github.com/protomaps/PMTiles/tree/main/js).
 See [awesome-vector-tiles](https://github.com/mapbox/awesome-vector-tiles) for more projects that work with data in this
 format.
 
@@ -87,7 +88,7 @@ Using [Node.js](https://nodejs.org/en/download/):
 
 ```bash
 npm install -g tileserver-gl-light
-tileserver-gl-light --mbtiles data/output.mbtiles
+tileserver-gl-light data/output.mbtiles
 ```
 
 Or using [Docker](https://docs.docker.com/get-docker/):
@@ -100,6 +101,8 @@ Then open http://localhost:8080 to view tiles.
 
 Some common arguments:
 
+- `--output` tells planetiler where to write output to, and what format to write it in. For
+  example `--output=australia.pmtiles` creates a pmtiles archive named `australia.pmtiles`.
 - `--download` downloads input sources automatically and `--only-download` exits after downloading
 - `--area=monaco` downloads a `.osm.pbf` extract from [Geofabrik](https://download.geofabrik.de/)
 - `--osm-path=path/to/file.osm.pbf` points Planetiler at an existing OSM extract on disk
@@ -209,6 +212,8 @@ download regularly-updated tilesets.
   OpenStreetMap [.osm.pbf](https://wiki.openstreetmap.org/wiki/PBF_Format),
   [`geopackage`](https://www.geopackage.org/),
   and [Esri Shapefiles](https://en.wikipedia.org/wiki/Shapefile) data sources
+- Writes to [MBTiles](https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md) or
+  or [PMTiles](https://github.com/protomaps/PMTiles) output.
 - Java-based [Profile API](planetiler-core/src/main/java/com/onthegomap/planetiler/Profile.java) to customize how source
   elements map to vector tile features, and post-process generated tiles
   using [JTS geometry utilities](https://github.com/locationtech/jts)
@@ -328,6 +333,7 @@ Planetiler is made possible by these awesome open source projects:
 - [cel-java](https://github.com/projectnessie/cel-java) for the Java implementation of
   Google's [Common Expression Language](https://github.com/google/cel-spec) that powers dynamic expressions embedded in
   schema config files.
+- [PMTiles](https://github.com/protomaps/PMTiles) optimized tile storage format
 
 See [NOTICE.md](NOTICE.md) for a full list and license details.
 

diff --git a/...benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesWriter.java b/...benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesWriter.java
@@ -66,9 +66,9 @@ public static void main(String[] args) throws IOException {
     for (int repetition = 0; repetition < repetitions; repetition++) {
 
       Path outputPath = getTempOutputPath();
-      try (var mbtiles = Mbtiles.newWriteToFileDatabase(outputPath, config.compactDb())) {
+      try (var mbtiles = Mbtiles.newWriteToFileDatabase(outputPath, config.arguments())) {
 
-        if (config.skipIndexCreation()) {
+        if (mbtiles.skipIndexCreation()) {
           mbtiles.createTablesWithoutIndexes();
         } else {
           mbtiles.createTablesWithIndexes();