CI #8
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| # This workflow converts OpenStreetMap data from Geofabrik to compressed MBTiles format | |
| # It runs daily at 12:00 UTC and can also be triggered manually | |
| # Each country is processed in parallel with max 2 concurrent jobs (see strategy.max-parallel below) | |
| on: | |
| schedule: | |
| - cron: '0 12 * * *' | |
| workflow_dispatch: | |
| jobs: | |
| convert-osm-to-mbtiles: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| max-parallel: 2 | |
| fail-fast: false | |
| matrix: | |
| country: | |
| - { name: south-korea, url: https://download.geofabrik.de/asia/south-korea-latest.osm.pbf, ram: 6g, maxzoom: 13 } | |
| - { name: japan, url: https://download.geofabrik.de/asia/japan-latest.osm.pbf, ram: 7g, maxzoom: 12 } | |
| - { name: germany, url: https://download.geofabrik.de/europe/germany-latest.osm.pbf, ram: 7g, maxzoom: 12 } | |
| - { name: france, url: https://download.geofabrik.de/europe/france-latest.osm.pbf, ram: 7g, maxzoom: 12 } | |
| - { name: united-kingdom, url: https://download.geofabrik.de/europe/great-britain-latest.osm.pbf, ram: 7g, maxzoom: 12 } | |
| - { name: italy, url: https://download.geofabrik.de/europe/italy-latest.osm.pbf, ram: 7g, maxzoom: 12 } | |
| - { name: spain, url: https://download.geofabrik.de/europe/spain-latest.osm.pbf, ram: 7g, maxzoom: 12 } | |
| - { name: canada, url: https://download.geofabrik.de/north-america/canada-latest.osm.pbf, ram: 7g, maxzoom: 11 } | |
| - { name: australia, url: https://download.geofabrik.de/australia-oceania/australia-latest.osm.pbf, ram: 7g, maxzoom: 11 } | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup JDK 21 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: '21' | |
| distribution: 'temurin' | |
| - name: Install tools | |
| run: sudo apt-get update && sudo apt-get install -y zstd wget unzip | |
| - name: Cache planetiler data | |
| uses: actions/cache@v4 | |
| with: | |
| path: data | |
| key: planetiler-data-v3 | |
| - name: Prepare planetiler sources | |
| run: | | |
| mkdir -p data/sources | |
| # Download water polygons with retry logic | |
| if [ ! -f data/sources/water-polygons-split-3857.zip ]; then | |
| echo "Downloading water polygons..." | |
| wget --tries=5 --timeout=60 \ | |
| https://osmdata.openstreetmap.de/download/water-polygons-split-3857.zip \ | |
| -O data/sources/water-polygons-split-3857.zip || { | |
| echo "Failed to download water polygons" | |
| exit 1 | |
| } | |
| else | |
| echo "Using cached water polygons" | |
| fi | |
| # Download lake centerlines with retry logic | |
| if [ ! -f data/sources/lake_centerline.shp.zip ]; then | |
| echo "Downloading lake centerlines..." | |
| wget --tries=5 --timeout=60 \ | |
| https://github.com/acalcutt/osm-lakelines/releases/download/v12/lake_centerline.shp.zip \ | |
| -O data/sources/lake_centerline.shp.zip || { | |
| echo "Failed to download lake centerlines" | |
| exit 1 | |
| } | |
| else | |
| echo "Using cached lake centerlines" | |
| fi | |
| # Download Natural Earth data with retry logic | |
| if [ ! -f data/sources/natural_earth_vector.sqlite.zip ]; then | |
| echo "Downloading Natural Earth data..." | |
| wget --tries=5 --timeout=60 \ | |
| https://naciscdn.org/naturalearth/packages/natural_earth_vector.sqlite.zip \ | |
| -O data/sources/natural_earth_vector.sqlite.zip || { | |
| echo "Failed to download Natural Earth data" | |
| exit 1 | |
| } | |
| else | |
| echo "Using cached Natural Earth data" | |
| fi | |
| - name: Download OSM | |
| run: | | |
| if [ ! -f "${{ matrix.country.name }}.osm.pbf" ]; then | |
| echo "Downloading OSM data for ${{ matrix.country.name }}..." | |
| wget --tries=5 --timeout=300 -O "${{ matrix.country.name }}.osm.pbf" "${{ matrix.country.url }}" || { | |
| echo "Failed to download OSM data for ${{ matrix.country.name }}" | |
| exit 1 | |
| } | |
| else | |
| echo "Using cached OSM data for ${{ matrix.country.name }}" | |
| fi | |
| - name: Download Planetiler | |
| run: | | |
| wget --tries=5 --timeout=60 -q https://github.com/onthegomap/planetiler/releases/download/v0.10.0/planetiler.jar || { | |
| echo "Failed to download Planetiler after 5 attempts" | |
| exit 1 | |
| } | |
| - name: Convert OSM → MBTiles | |
| run: | | |
| echo "Converting ${{ matrix.country.name }} OSM data to MBTiles..." | |
| java -Xmx${{ matrix.country.ram }} \ | |
| -jar planetiler.jar \ | |
| --osm-path="${{ matrix.country.name }}.osm.pbf" \ | |
| --output="${{ matrix.country.name }}.mbtiles" \ | |
| --minzoom=0 \ | |
| --maxzoom=${{ matrix.country.maxzoom }} \ | |
| --boundary-layer=true \ | |
| --simplify-tolerance=0.1 \ | |
| --natural-earth-path=data/sources/natural_earth_vector.sqlite.zip \ | |
| --water-polygons-path=data/sources/water-polygons-split-3857.zip \ | |
| --force | |
| # Verify output file was created | |
| if [ ! -f "${{ matrix.country.name }}.mbtiles" ]; then | |
| echo "ERROR: MBTiles file was not created" | |
| exit 1 | |
| fi | |
| echo "Conversion completed successfully. File size: $(du -h ${{ matrix.country.name }}.mbtiles | cut -f1)" | |
| - name: Get current date | |
| id: date | |
| run: | | |
| echo "yymmdd=$(date +'%y%m%d')" >> $GITHUB_OUTPUT | |
| - name: Compress MBTiles | |
| run: | | |
| # Create dated version with maximum compression (level 19) | |
| zstd -19 --threads=0 "${{ matrix.country.name }}.mbtiles" \ | |
| -o "${{ matrix.country.name }}-${{ steps.date.outputs.yymmdd }}.mbtiles.zst" | |
| # Create latest version (copy of dated version) | |
| cp "${{ matrix.country.name }}-${{ steps.date.outputs.yymmdd }}.mbtiles.zst" \ | |
| "${{ matrix.country.name }}.mbtiles.zst" | |
| - name: Upload Artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.country.name }} | |
| path: | | |
| ${{ matrix.country.name }}-${{ steps.date.outputs.yymmdd }}.mbtiles.zst | |
| ${{ matrix.country.name }}.mbtiles.zst | |
| compression-level: 0 | |
| - name: Cleanup | |
| if: always() | |
| run: | | |
| # Clean up large temporary files to save disk space | |
| echo "Cleaning up temporary files..." | |
| rm -f *.osm.pbf | |
| rm -f *.mbtiles | |
| echo "Cleanup completed" | |
| create-release: | |
| needs: convert-osm-to-mbtiles | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Get current date | |
| id: date | |
| run: | | |
| echo "tag=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT | |
| echo "title=$(date +'%Y/%m/%d')" >> $GITHUB_OUTPUT | |
| echo "yymmdd=$(date +'%y%m%d')" >> $GITHUB_OUTPUT | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: artifacts | |
| - name: Create Release | |
| uses: softprops/action-gh-release@v2 | |
| with: | |
| tag_name: ${{ steps.date.outputs.tag }} | |
| name: ${{ steps.date.outputs.title }} | |
| body: | | |
| ## Daily MBTiles Update | |
| **Date:** ${{ steps.date.outputs.title }} | |
| ### Countries | |
| South Korea, Japan, Germany, France, United Kingdom, Italy, Spain, Canada, Australia | |
| ### File Formats | |
| Each country has two versions: | |
| - **`country-${{ steps.date.outputs.yymmdd }}.mbtiles.zst`** - Dated version for archival | |
| - **`country.mbtiles.zst`** - Latest version (always current) | |
| ### Details | |
| - Min Zoom: 0 | |
| - Max Zoom: 11-13 (country-dependent: 13 for S. Korea, 12 for most, 11 for Canada/Australia) | |
| - Boundary Layer: Enabled | |
| - Compression: zstd level 19 (maximum) | |
| - Tool: Planetiler v0.10.0 | |
| - Source: Geofabrik OSM data | |
| - Additional Data: Natural Earth, Water Polygons, Lake Centerlines | |
| ### Decompress | |
| ```bash | |
| # Decompress latest version | |
| zstd -d country.mbtiles.zst | |
| # Or decompress dated version | |
| zstd -d country-${{ steps.date.outputs.yymmdd }}.mbtiles.zst | |
| ``` | |
| --- | |
| 🤖 Automated via GitHub Actions | |
| files: artifacts/**/*.zst | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| draft: false | |
| prerelease: false |