Convert & Check Images #197
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Convert & Check Images | |
| on: | |
| schedule: | |
| # Do conversion every day at 00:03 clock UTC | |
| - cron: "3 0 * * *" | |
| workflow_dispatch: | |
| env: | |
| CARGO_TERM_COLOR: always | |
| REGISTRY: ghcr.io | |
| ORGANIZATION: ${{ github.repository }} | |
| IMAGE_LIST_PATH: misc/top_images/image_list.txt | |
| FSCK_PATCH_PATH: misc/top_images/fsck.patch | |
| jobs: | |
| nydusify-build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Setup Golang | |
| uses: actions/setup-go@v3 | |
| with: | |
| go-version: ~1.18 | |
| - name: Golang Cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.cache/go-build | |
| ~/go/pkg/mod | |
| key: ${{ runner.os }}-golang-${{ hashFiles('**/go.sum') }} | |
| restore-keys: | | |
| ${{ runner.os }}-golang- | |
| - name: Build Contrib | |
| run: | | |
| curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/local/bin v1.51.2 | |
| make -e DOCKER=false nydusify-release | |
| - name: Upload Nydusify | |
| uses: actions/upload-artifact@master | |
| with: | |
| name: nydusify-artifact | |
| path: contrib/nydusify/cmd/nydusify | |
| nydus-build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Rust Cache | |
| uses: Swatinem/rust-cache@v2.2.0 | |
| with: | |
| cache-on-failure: true | |
| shared-key: nydus-build | |
| - name: Build Nydus | |
| run: | | |
| rustup component add rustfmt clippy | |
| make | |
| - name: Upload Nydus Binaries | |
| uses: actions/upload-artifact@master | |
| with: | |
| name: nydus-artifact | |
| path: | | |
| target/release/nydus-image | |
| target/release/nydusd | |
| fsck-erofs-build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Build fsck.erofs | |
| run: | | |
| sudo apt-get update && sudo apt-get install -y build-essential git autotools-dev automake libtool pkg-config uuid-dev liblz4-dev | |
| git clone https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs-utils.git | |
| cd erofs-utils && git apply ../${{ env.FSCK_PATCH_PATH }} && ./autogen.sh && ./configure && make && cd .. | |
| sudo cp erofs-utils/fsck/fsck.erofs /usr/local/bin/ | |
| - name: Upload fsck.erofs | |
| uses: actions/upload-artifact@master | |
| with: | |
| name: fsck-erofs-artifact | |
| path: | | |
| /usr/local/bin/fsck.erofs | |
| convert-zran: | |
| runs-on: ubuntu-latest | |
| needs: [nydusify-build, nydus-build, fsck-erofs-build] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Login ghcr registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Download Nydus | |
| uses: actions/download-artifact@master | |
| with: | |
| name: nydus-artifact | |
| path: /usr/local/bin | |
| - name: Download Nydusify | |
| uses: actions/download-artifact@master | |
| with: | |
| name: nydusify-artifact | |
| path: /usr/local/bin | |
| - name: Download fsck.erofs | |
| uses: actions/download-artifact@master | |
| with: | |
| name: fsck-erofs-artifact | |
| path: /usr/local/bin | |
| - name: Convert and check zran images | |
| run: | | |
| sudo chmod +x /usr/local/bin/nydus* | |
| sudo chmod +x /usr/local/bin/fsck.erofs | |
| sudo docker run -d --restart=always -p 5000:5000 registry | |
| sudo mkdir convert-zran | |
| for I in $(cat ${{ env.IMAGE_LIST_PATH }}); do | |
| echo "converting $I:latest to $I:nydus-nightly-oci-ref" | |
| ghcr_repo=${{ env.REGISTRY }}/${{ env.ORGANIZATION }} | |
| # push oci image to ghcr/local for zran reference | |
| sudo docker pull $I:latest | |
| sudo docker tag $I:latest $ghcr_repo/$I | |
| sudo docker tag $I:latest localhost:5000/$I | |
| sudo DOCKER_CONFIG=$HOME/.docker docker push $ghcr_repo/$I | |
| sudo docker push localhost:5000/$I | |
| # for pre-built images | |
| sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ | |
| --oci-ref \ | |
| --source $ghcr_repo/$I \ | |
| --target $ghcr_repo/$I:nydus-nightly-oci-ref \ | |
| --platform linux/amd64,linux/arm64 | |
| # use local registry for speed | |
| sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ | |
| --oci-ref \ | |
| --source localhost:5000/$I \ | |
| --target localhost:5000/$I:nydus-nightly-oci-ref \ | |
| --platform linux/amd64,linux/arm64 \ | |
| --output-json convert-zran/${I}.json | |
| # check zran image and referenced oci image | |
| sudo rm -rf ./tmp | |
| sudo DOCKER_CONFIG=$HOME/.docker nydusify check \ | |
| --source localhost:5000/$I \ | |
| --target localhost:5000/$I:nydus-nightly-oci-ref | |
| sudo fsck.erofs -d1 output/nydus_bootstrap | |
| sudo rm -rf ./output | |
| done | |
| - name: Save Nydusify Metric | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: convert-zran-metric | |
| path: convert-zran | |
| convert-native-v5: | |
| runs-on: ubuntu-latest | |
| needs: [nydusify-build, nydus-build] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Login ghcr registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Download Nydus | |
| uses: actions/download-artifact@master | |
| with: | |
| name: nydus-artifact | |
| path: /usr/local/bin | |
| - name: Download Nydusify | |
| uses: actions/download-artifact@master | |
| with: | |
| name: nydusify-artifact | |
| path: /usr/local/bin | |
| - name: Convert and check RAFS v5 images | |
| run: | | |
| sudo chmod +x /usr/local/bin/nydus* | |
| sudo docker run -d --restart=always -p 5000:5000 registry | |
| sudo mkdir convert-native-v5 | |
| for I in $(cat ${{ env.IMAGE_LIST_PATH }}); do | |
| echo "converting $I:latest to $I:nydus-nightly-v5" | |
| # for pre-built images | |
| sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ | |
| --source $I:latest \ | |
| --target ${{ env.REGISTRY }}/${{ env.ORGANIZATION }}/$I:nydus-nightly-v5 \ | |
| --fs-version 5 \ | |
| --platform linux/amd64,linux/arm64 | |
| # use local registry for speed | |
| sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ | |
| --source $I:latest \ | |
| --target localhost:5000/$I:nydus-nightly-v5 \ | |
| --fs-version 5 \ | |
| --platform linux/amd64,linux/arm64 \ | |
| --output-json convert-native-v5/${I}.json | |
| sudo rm -rf ./tmp | |
| sudo DOCKER_CONFIG=$HOME/.docker nydusify check --source $I:latest \ | |
| --target localhost:5000/$I:nydus-nightly-v5 | |
| done | |
| - name: Save Nydusify Metric | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: convert-native-v5-metric | |
| path: convert-native-v5 | |
| convert-native-v6: | |
| runs-on: ubuntu-latest | |
| needs: [nydusify-build, nydus-build, fsck-erofs-build] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Login ghcr registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Download Nydus | |
| uses: actions/download-artifact@master | |
| with: | |
| name: nydus-artifact | |
| path: /usr/local/bin | |
| - name: Download Nydusify | |
| uses: actions/download-artifact@master | |
| with: | |
| name: nydusify-artifact | |
| path: /usr/local/bin | |
| - name: Download fsck.erofs | |
| uses: actions/download-artifact@master | |
| with: | |
| name: fsck-erofs-artifact | |
| path: /usr/local/bin | |
| - name: Convert and check RAFS v6 images | |
| run: | | |
| sudo chmod +x /usr/local/bin/nydus* | |
| sudo chmod +x /usr/local/bin/fsck.erofs | |
| sudo docker run -d --restart=always -p 5000:5000 registry | |
| sudo mkdir convert-native-v6 | |
| for I in $(cat ${{ env.IMAGE_LIST_PATH }}); do | |
| echo "converting $I:latest to $I:nydus-nightly-v6" | |
| # for pre-built images | |
| sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ | |
| --source $I:latest \ | |
| --target ${{ env.REGISTRY }}/${{ env.ORGANIZATION }}/$I:nydus-nightly-v6 \ | |
| --fs-version 6 \ | |
| --platform linux/amd64,linux/arm64 | |
| # use local registry for speed | |
| sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ | |
| --source $I:latest \ | |
| --target localhost:5000/$I:nydus-nightly-v6 \ | |
| --fs-version 6 \ | |
| --platform linux/amd64,linux/arm64 \ | |
| --output-json convert-native-v6/${I}.json | |
| sudo rm -rf ./tmp | |
| sudo DOCKER_CONFIG=$HOME/.docker nydusify check --source $I:latest \ | |
| --target localhost:5000/$I:nydus-nightly-v6 | |
| sudo fsck.erofs -d1 output/nydus_bootstrap | |
| sudo rm -rf ./output | |
| done | |
| - name: Save Nydusify Metric | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: convert-native-v6-metric | |
| path: convert-native-v6 | |
| convert-metric: | |
| runs-on: ubuntu-latest | |
| needs: [convert-zran, convert-native-v5, convert-native-v6] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Download Zran Metric | |
| uses: actions/download-artifact@master | |
| with: | |
| name: convert-zran-metric | |
| path: convert-zran | |
| - name: Download V5 Metric | |
| uses: actions/download-artifact@master | |
| with: | |
| name: convert-native-v5-metric | |
| path: convert-native-v5 | |
| - name: Download V6 Metric | |
| uses: actions/download-artifact@master | |
| with: | |
| name: convert-native-v6-metric | |
| path: convert-native-v6 | |
| - name: Summary | |
| run: | | |
| echo "## Image Size(MB)" > $GITHUB_STEP_SUMMARY | |
| echo "|image name|nydus-zran|nydus-v5|nydus-v6|" >> $GITHUB_STEP_SUMMARY | |
| echo "|:---:|:--:|:-------:|:-------:|" >> $GITHUB_STEP_SUMMARY | |
| for I in $(cat ${{ env.IMAGE_LIST_PATH }}); do | |
| zranSourceImageSize=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.SourceImageSize' convert-zran/${I}.json) / 1048576")") | |
| zranTargetImageSize=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.TargetImageSize' convert-zran/${I}.json) / 1048576")") | |
| v5SourceImageSize=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.SourceImageSize' convert-native-v5/${I}.json) / 1048576")") | |
| v5TargetImageSize=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.TargetImageSize' convert-native-v5/${I}.json) / 1048576")") | |
| v6SourceImageSize=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.SourceImageSize' convert-native-v6/${I}.json) / 1048576")") | |
| v6TargetImageSize=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.TargetImageSize' convert-native-v6/${I}.json) / 1048576")") | |
| echo "|${I}|${zranSourceImageSize}/${zranTargetImageSize}|${v5SourceImageSize}/${v5TargetImageSize}|${v6SourceImageSize}/${v6TargetImageSize}|" >> $GITHUB_STEP_SUMMARY | |
| done | |
| echo "## Conversion Time(ms)" >> $GITHUB_STEP_SUMMARY | |
| echo "|image name|nydus-zran|nydus-v5|nydus-v6|" >> $GITHUB_STEP_SUMMARY | |
| echo "|:---:|:--:|:-------:|:-------:|" >> $GITHUB_STEP_SUMMARY | |
| for I in $(cat ${{ env.IMAGE_LIST_PATH }}); do | |
| zranConversionElapsed=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.ConversionElapsed' convert-zran/${I}.json) / 1000000")") | |
| v5ConversionElapsed=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.ConversionElapsed' convert-native-v5/${I}.json) / 1000000")") | |
| v6ConversionElapsed=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.ConversionElapsed' convert-native-v6/${I}.json) / 1000000")") | |
| echo "|${I}|${zranConversionElapsed}|${v5ConversionElapsed}|${v6ConversionElapsed}|" >> $GITHUB_STEP_SUMMARY | |
| done | |
| - uses: geekyeggo/delete-artifact@v2 | |
| with: | |
| name: '*' |