Update parts database #519
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: "Update parts database" | |
| on: | |
| schedule: # 2 hours after jlcparts updates their database | |
| - cron: "0 5 * * *" | |
| workflow_dispatch: # allow for manually trigger workflow | |
| jobs: | |
| build_and_update: | |
| name: "Update component database and frontend" | |
| runs-on: ubuntu-22.04 | |
| environment: github-pages | |
| steps: | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends \ | |
| python3 python3-pip wget zip unzip p7zip-full sqlite3 | |
| - name: Maximize build space | |
| uses: easimon/maximize-build-space@master | |
| with: | |
| root-reserve-mb: 512 | |
| swap-size-mb: 1024 | |
| remove-dotnet: 'true' | |
| - name: Free more Space, Found in https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 | |
| run: | | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf "/usr/local/share/boost" | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Show versions | |
| run: | | |
| sqlite3 --version | |
| python3 --version | |
| python3 -c "import sqlite3; import pprint; db = sqlite3.connect(':memory:'); cursor = db.execute('PRAGMA COMPILE_OPTIONS'); pprint.pprint(cursor.fetchall())" | |
| - name: Install python dependencies | |
| run: | | |
| pip install humanize | |
| - name: Update database | |
| run: | | |
| set -x | |
| # Print the current disk usage, might help with future "no disk space left" issues | |
| echo "=============================================================================================================================" | |
| df -h | |
| echo "=============================================================================================================================" | |
| mkdir -p db_build | |
| cd db_build | |
| wget -q https://yaqwsx.github.io/jlcparts/data/cache.zip | |
| VOLUMES=$(7z l cache.zip | grep "Volume Index = " | grep -Eoh "[0-9]+") | |
| for seq in $(seq 1 $VOLUMES); do | |
| CACHE=$(printf '%02d' $seq) | |
| wget -q https://yaqwsx.github.io/jlcparts/data/cache.z$CACHE || true | |
| done | |
| 7z x cache.zip | |
| rm -rf cache.z* | |
| ls -lah | |
| cd .. | |
| # Print the current disk usage, might help with future "no disk space left" issues | |
| echo "=============================================================================================================================" | |
| df -h | |
| echo "=============================================================================================================================" | |
| # creates the converted database in 80M split-zip files in folder db_build | |
| python3 jlcparts_db_convert.py | |
| # remove the source db, as we don't want to package that one | |
| rm db_build/cache.sqlite3 | |
| # some info output for sanity check | |
| ls -lah db_build | |
| du -cslh db_build | |
| # Print the current disk usage, might help with future "no disk space left" issues | |
| echo "=============================================================================================================================" | |
| df -h | |
| echo "=============================================================================================================================" | |
| - name: Upload pages artifact | |
| uses: actions/upload-pages-artifact@v3 | |
| with: | |
| name: github-pages | |
| path: db_build | |
| deploy: | |
| name: "Deploy" | |
| runs-on: ubuntu-20.04 | |
| needs: build_and_update | |
| permissions: | |
| actions: write | |
| contents: write | |
| pages: write | |
| id-token: write | |
| # Deploy to the github-pages environment | |
| environment: | |
| name: github-pages | |
| url: ${{ steps.deployment.outputs.page_url }} | |
| if: github.ref == 'refs/heads/main' | |
| steps: | |
| - name: Deploy to GitHub Pages | |
| id: deployment | |
| uses: actions/deploy-pages@v4 | |
| # delete-artifact | |
| - name: Clean | |
| uses: geekyeggo/delete-artifact@v4 | |
| with: | |
| name: github-pages |