Compress Documentation DB and Upload to Google Drive #133
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Compress Documentation DB and Upload to Google Drive | |
| permissions: | |
| id-token: write | |
| contents: write | |
| actions: write | |
| on: | |
| schedule: | |
| - cron: '0 11 * * *' # Daily at 11:00 AM UTC | |
| workflow_dispatch: | |
| jobs: | |
| download_documentation: | |
| name: Download and compress documentation DB | |
| runs-on: self-hosted | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Authenticate to Google Cloud for Drive access | |
| id: auth_drive | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| workload_identity_provider: ${{ secrets.WIF_PROVIDER }} | |
| service_account: ${{ secrets.IDENTITY_EMAIL }} | |
| token_format: 'access_token' | |
| access_token_scopes: 'https://www.googleapis.com/auth/drive' | |
| - name: Download latest documentation.db from Google Drive | |
| id: download | |
| run: | | |
| DB_FILE_ID="${{ secrets.DOCUMENTATION_DB_FILE_ID }}" | |
| ACCESS_TOKEN="${{ steps.auth_drive.outputs.access_token }}" | |
| if [ -z "$DB_FILE_ID" ]; then | |
| echo "ERROR: DOCUMENTATION_DB_FILE_ID secret not set" | |
| echo "Please set the DOCUMENTATION_DB_FILE_ID secret in repository settings" | |
| exit 1 | |
| fi | |
| echo "Downloading documentation.db from Google Drive..." | |
| curl -sL -H "Authorization: Bearer $ACCESS_TOKEN" \ | |
| "https://www.googleapis.com/drive/v3/files/${DB_FILE_ID}?alt=media&supportsAllDrives=true&acknowledgeAbuse=true" \ | |
| -o documentation.db | |
| if [ ! -f documentation.db ]; then | |
| echo "ERROR: Failed to download documentation.db" | |
| exit 1 | |
| fi | |
| FILE_SIZE_BYTES=$(stat -c%s documentation.db 2>/dev/null || stat -f%z documentation.db 2>/dev/null) | |
| FILE_SIZE_HUMAN=$(du -h documentation.db | cut -f1) | |
| if [ "$FILE_SIZE_BYTES" -lt 1000000 ]; then | |
| echo "ERROR: Downloaded file is too small ($FILE_SIZE_HUMAN)" | |
| echo "This usually means the file was not found or service account lacks access" | |
| jq -n '{ | |
| blocks: [ | |
| { | |
| type: "header", | |
| text: { | |
| type: "plain_text", | |
| text: ":x: Compression Failed - Documentation DB Download Error", | |
| emoji: true | |
| } | |
| }, | |
| { | |
| type: "section", | |
| text: { | |
| type: "mrkdwn", | |
| text: "@here The compression run failed because *documentation.db* could not be downloaded from Google Drive." | |
| } | |
| }, | |
| { | |
| type: "section", | |
| text: { | |
| type: "mrkdwn", | |
| text: "*Possible Causes:*\n• File not found or incorrect file ID\n• Service account does not have access to the file\n• File was deleted or moved" | |
| } | |
| }, | |
| { | |
| type: "section", | |
| text: { | |
| type: "mrkdwn", | |
| text: "*Action Required:*\n1. Verify the file exists in Google Drive\n2. Check that DOCUMENTATION_DB_FILE_ID secret is correct\n3. Ensure service account has Viewer access to the file" | |
| } | |
| } | |
| ] | |
| }' > error_payload.json | |
| curl -X POST -H "Content-type: application/json" --data @error_payload.json "${{ secrets.SLACK_WEBHOOK_URL }}" | |
| rm -f error_payload.json | |
| exit 1 | |
| fi | |
| echo "Successfully downloaded documentation.db ($FILE_SIZE_HUMAN)" | |
| - name: Check if documentation.db has changed | |
| id: checksum | |
| run: | | |
| echo "Checking if documentation.db has changed..." | |
| ACCESS_TOKEN="${{ steps.auth_drive.outputs.access_token }}" | |
| LAST_MD5_FILE_ID="${{ secrets.DOCUMENTATION_MD5_FILE_ID }}" | |
| # Download last_md5 from Drive | |
| LAST_MD5=$(curl -s -H "Authorization: Bearer $ACCESS_TOKEN" \ | |
| "https://www.googleapis.com/drive/v3/files/${LAST_MD5_FILE_ID}?alt=media" \ | |
| | tr -d '\n') | |
| echo "Last md5Checksum: $LAST_MD5" | |
| # Compute checksum of the downloaded documentation.db | |
| CURRENT_MD5=$(md5sum documentation.db | awk '{print $1}') | |
| echo "Current md5Checksum: $CURRENT_MD5" | |
| # Compare | |
| if [ "$CURRENT_MD5" = "$LAST_MD5" ]; then | |
| echo "skip=true" >> $GITHUB_OUTPUT | |
| echo "No change detected. Skipping download and compression." | |
| exit 0 | |
| fi | |
| echo "skip=false" >> $GITHUB_OUTPUT | |
| echo "$CURRENT_MD5" > last_md5 | |
| - name: Compress documentation.db with Brotli | |
| id: brotli | |
| if: steps.checksum.outputs.skip != 'true' | |
| run: | | |
| echo "Installing Brotli..." | |
| sudo apt-get update && sudo apt-get install -y brotli | |
| echo "Compressing documentation.db with maximum compression..." | |
| brotli -f -Z -o documentation.db.br documentation.db | |
| if [ ! -f documentation.db.br ]; then | |
| echo "ERROR: Failed to create documentation.db.br" | |
| exit 1 | |
| fi | |
| ORIGINAL_SIZE=$(du -h documentation.db | cut -f1) | |
| COMPRESSED_SIZE=$(du -h documentation.db.br | cut -f1) | |
| echo "Successfully compressed documentation.db" | |
| echo "Original size: $ORIGINAL_SIZE" | |
| echo "Compressed size: $COMPRESSED_SIZE" | |
| - name: Upload documentation.db.br and last_md5 to Drive | |
| id: upload | |
| if: steps.checksum.outputs.skip != 'true' | |
| run: | | |
| echo "Uploading documentation.db.br and last_md5 to Drive..." | |
| ACCESS_TOKEN="${{ steps.auth_drive.outputs.access_token }}" | |
| BR_FILE_ID="${{ secrets.DOCUMENTATION_BR_FILE_ID }}" | |
| BR_MD5_FILE_ID="${{ secrets.DOCUMENTATION_BR_MD5_FILE_ID }}" | |
| LAST_MD5_FILE_ID="${{ secrets.DOCUMENTATION_MD5_FILE_ID }}" | |
| # Compute current md5Checksum of documentation.db | |
| CURRENT_MD5=$(md5sum documentation.db | awk '{print $1}') | |
| echo "$CURRENT_MD5" > last_md5 | |
| # Compute md5Checksum of documentation.db.br | |
| BR_MD5=$(md5sum documentation.db.br | awk '{print $1}') | |
| echo "$BR_MD5" > br_md5 | |
| # Upload documentation.db.br | |
| response=$(curl -s -o /dev/null -w "%{http_code}" --fail -X PATCH \ | |
| -H "Authorization: Bearer $ACCESS_TOKEN" \ | |
| -H "Content-Type: application/octet-stream" \ | |
| --data-binary "@documentation.db.br" \ | |
| "https://www.googleapis.com/upload/drive/v3/files/${BR_FILE_ID}?uploadType=media") | |
| if [[ "$response" -ne 200 ]]; then | |
| echo "Failed upload of documentation.db.br with HTTP status $response" | |
| exit 1 | |
| fi | |
| # Upload last_md5 | |
| response=$(curl -s -o /dev/null -w "%{http_code}" --fail -X PATCH \ | |
| -H "Authorization: Bearer $ACCESS_TOKEN" \ | |
| -H "Content-Type: text/plain" \ | |
| --data-binary "@last_md5" \ | |
| "https://www.googleapis.com/upload/drive/v3/files/${LAST_MD5_FILE_ID}?uploadType=media") | |
| if [[ "$response" -ne 200 ]]; then | |
| echo "Failed upload of documentation.db.md5 with HTTP status $response" | |
| exit 1 | |
| fi | |
| # Upload br_md5 | |
| response=$(curl -s -o /dev/null -w "%{http_code}" --fail -X PATCH \ | |
| -H "Authorization: Bearer $ACCESS_TOKEN" \ | |
| -H "Content-Type: text/plain" \ | |
| --data-binary "@br_md5" \ | |
| "https://www.googleapis.com/upload/drive/v3/files/${BR_MD5_FILE_ID}?uploadType=media") | |
| if [[ "$response" -ne 200 ]]; then | |
| echo "Failed upload of documentation.db.br.md5 with HTTP status $response" | |
| exit 1 | |
| fi | |
| echo "Upload complete." | |