Generic GHA to sync dists with gh releases

.github/workflows/sync-release-assets.yml

@@ -0,0 +1,139 @@
+name: Sync github release assets with dist.ipfs.io
+on:
+  workflow_call:
+    inputs:
+      dist:
+        required: true
+        type: string
+
+concurrency:
+  group: release-assets-dist-sync
+  cancel-in-progress: true
+
+jobs:
+  sync-github-and-dist-ipfs-io:
+    runs-on: "ubuntu-latest"
+    steps:
+      - uses: ipfs/download-ipfs-distribution-action@v1
+      - uses: ipfs/start-ipfs-daemon-action@v1
+        with:
+          args: --init --init-profile=flatfs,server --enable-gc=false
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 14
+      - name: Sync the latest 5 github releases
+        uses: actions/github-script@v4
+        with:
+          script: |
+            const fs = require('fs').promises
+            const max_synced = 5
+
+            // fetch github releases
+            resp = await github.repos.listReleases({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              page: 1,
+              per_page: max_synced
+            })
+            const release_assets = [];
+            num_synced = 0;
+            for (const release of resp.data) {
+              console.log("checking release tagged", release.tag_name)
+              if (release.draft || release.prerelease) {
+                console.log("skipping draft/prerelease tagged", release.tag_name)
+                continue
+              }
+              if (num_synced > max_synced) {
+                console.log("done: synced", max_synced, "latest releases")
+                break;
+              }
+              num_synced += 1
+
+              const github_assets = new Set()
+              for (const asset of release.assets) {
+                github_assets.add(asset.name)
+              }
+
+              // fetch asset info from dist.ipfs.io
+              p = '/ipns/dist.ipfs.io/${{ inputs.dist }}/' + release.tag_name
+              let stdout = ''
+              const options = {}
+              options.listeners = {
+                stdout: (data) => {
+                  stdout += data.toString();
+                }
+              }
+              await exec.exec('ipfs', ['ls', p], options)
+
+              const dist_assets = new Set()
+              const missing_files = []
+              for (const raw_line of stdout.split("\n")) {
+                line = raw_line.trim();
+                if (line.length != 0) {
+                  file = line.split(/(\s+)/).filter( function(e) { return e.trim().length > 0; } )[2]
+                  dist_assets.add(file)
+                  if (!github_assets.has(file)) {
+                    missing_files.push(file)
+                  }
+                }
+              }
+
+              // if dist.ipfs.io has files not found in github, copy them over
+              for (const file of missing_files) {
+                file_sha = file + ".sha512"
+                file_cid = file + ".cid"
+
+                // skip files that don't have .cid and .sha512 checksum files
+                if (!dist_assets.has(file_sha) || !dist_assets.has(file_cid)) {
+                  if (!file.endsWith('.cid') && !file.endsWith('.sha512')) { // silent skip of .sha512.sha512 :)
+                    console.log(`skipping "${file}" as dist.ipfs.io does not provide .cid and .sha512 checksum files for it`)
+                  }
+                  continue
+                }
+
+                console.log("fetching", file, "from dist.ipfs.io")
+                await exec.exec('ipfs', ['get', p + '/' + file])
+                await exec.exec('ipfs', ['get', p + '/' + file_sha])
+                await exec.exec('ipfs', ['get', p + '/' + file_cid])
+                console.log("verifying contents of", file)
+
+                // compute sha512 output for file
+                let sha_stdout = ''
+                const sha_options = {}
+                sha_options.listeners = {
+                  stdout: (data) => {
+                    sha_stdout += data.toString();
+                  }
+                }
+                await exec.exec('sha512sum', [file], sha_options)
+                // read expected sha512 output
+                const sha_data = await fs.readFile(file_sha, "utf8")
+                const digest = (s) => s.split(' ').shift()
+                if (digest(sha_data) != digest(sha_stdout)) {
+                  console.log(`${file}.sha512: ${sha_data}`)
+                  console.log(`sha512sum ${file}: ${sha_stdout}`)
+                  throw "checksum verification failed for " + file
+                }
+
+                console.log("uploading", file, "to github release", release.tag_name)
+                const uploadReleaseAsset = async (file) => github.repos.uploadReleaseAsset({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  release_id: release.id,
+                  headers: {
+                    "content-type": "application/octet-stream",
+                    "content-length": `${(await fs.stat(file)).size}`
+                  },
+                  name: file,
+                  data: await fs.readFile(file)
+                })
+                await uploadReleaseAsset(file)
+                await uploadReleaseAsset(file_sha)
+                await uploadReleaseAsset(file_cid)
+
+              }
+              // summary of assets on both sides
+              release_assets.push({ tag: release.tag_name, github_assets, dist_assets })
+            }
+            console.log(release_assets)
+            return release_assets

configs/README.md

@@ -7,6 +7,7 @@ This directory contains config files used for workflow dispatch.
 | go | repositories containing Go code |
 | js | repositories containing JS code |
 | testing | repositories used for testing Unified CI workflows |
+| custom | repositories not ready for full unified CI |
 
 ## Adding new repository to existing config file
 

configs/custom.json

@@ -0,0 +1,15 @@
+{
+  "defaults": {
+    "files": []
+  },
+  "repositories": [
+    {
+      "target": "ipfs/go-ipfs",
+      "files": [
+        ".github/workflows/automerge.yml",
+        ".github/workflows/sync-release-assets.yml"
+      ],
+      "dist": "go-ipfs"
+    }
+  ]
+}

configs/go.json

@@ -446,7 +446,11 @@
       "target": "libp2p/go-libp2p-record"
     },
     {
-      "target": "libp2p/go-libp2p-relay-daemon"
+      "target": "libp2p/go-libp2p-relay-daemon",
+      "extra_files": [
+        ".github/workflows/sync-release-assets.yml"
+      ],
+      "dist": "libp2p-relay-daemon"
     },
     {
       "target": "libp2p/go-libp2p-routing-helpers"

templates/.github/workflows/sync-release-assets.yml

@@ -0,0 +1,11 @@
+name: Sync github release assets with dist.ipfs.io
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  sync-github-and-dist-ipfs-io:
+    uses: protocol/.github/.github/workflows/sync-release-assets.yml@master
+    with:
+      dist: ${{{ config.dist }}}