Overhaul the Makefile

This new setup actually makes use of make's dependency tracking so that
builds aren't randomly broken. It also correctly avoids rebuilds of things
that haven't changed and thus don't need to be rebuilt.

The previous Makefile setup was very buggy/racy if make was run with more
than one job in parallel. It would also have issues if make was interrupted
in the middle of a build. This one does not suffer from any of those issues.

This new setup takes into account both debug vs release modes for the hook.yaml
along with all the arches (new arches should be pretty easy to add in too).

One of the bigger features of this new Makefile setup is that everything
needed to build hook (except for the kernel) is built locally from source. This
means we don't need to push images only to have linuxkit fetch them. This was
only possible previously in dev-mode when we'd load the images from buildx
into the local docker. This was limited to dev mode because docker doesn't
support multi-arch manifests in the local cache. This makes it possible to
tinker locally without the risk of pushing to quay.io unintentionally.

This is also fully parameterized by both TAG and ORG, though ORG isn't
accounted for very well. If ORG is modified in the Makefile please follow
the directions and clean out the build dir `rm -rf out`. The TAG build
parameter is fully supported and can be different between multiple `make`
calls. We make use of this in CI now.

Signed-off-by: Manuel Mendez <github@i.m.mmlb.dev>

Author: mmlb

.github/workflows/ci.yaml

@@ -6,11 +6,6 @@ on:
 jobs:
   validation:
     runs-on: ubuntu-20.04
-    services:
-      registry:
-        image: registry:2
-        ports:
-          - 5000:5000
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
@@ -19,26 +14,21 @@ jobs:
         uses: docker/setup-qemu-action@v2
 
       - name: Set up Docker Buildx
-        id: buildx
         uses: docker/setup-buildx-action@v2
-        with:
-          driver-opts: network=host
 
       - name: Build and push hook-bootkit
         uses: docker/build-push-action@v3
         with:
           context: ./hook-bootkit/
           platforms: linux/amd64,linux/arm64
-          push: true
-          tags: localhost:5000/tinkerbell/hook-bootkit:latest
+          tags: quay.io/tinkerbell/hook-bootkit:latest
 
       - name: Build and push hook-docker
         uses: docker/build-push-action@v3
         with:
           context: ./hook-docker/
           platforms: linux/amd64,linux/arm64
-          push: true
-          tags: localhost:5000/tinkerbell/hook-docker:latest
+          tags: quay.io/tinkerbell/hook-docker:latest
 
       - uses: cachix/install-nix-action@v17
         with:
@@ -50,14 +40,11 @@ jobs:
       - name: Run formatters and linters
         run: nix-shell --run .github/workflows/formatters-and-linters.sh
 
-      # Replace hook-{bootkit,docker} but not hook-kernel
-      - run: sed -E -i 's,quay.io/tinkerbell/hook-(bootkit|docker),localhost:5000/tinkerbell/hook-\1,g' hook.in.yaml
-
       - name: Build
-        run: ./hack/ci-build.sh
+        run: nix-shell --run 'make TAG=${{ github.sha }} dist'
 
       # TODO: add artifacts for the built images
       - uses: actions/upload-artifact@v3
         with:
           name: hook-${{ github.sha }}.tar.gz
-          path: hook-${{ github.sha }}.tar.gz
+          path: out/${{ github.sha }}/rel/hook-${{ github.sha }}.tar.gz

.github/workflows/push.yaml

@@ -58,10 +58,10 @@ jobs:
 
       - name: Build & Deploy
         run: |
-          ./hack/build-and-deploy.sh
-          TAG=sha-${{steps.commitid.outputs.short}} ./hack/build-and-deploy.sh
+          nix-shell --run 'make TAG=latest deploy'
+          nix-shell --run 'make TAG=sha-${{steps.commitid.outputs.short}} deploy'
 
       - uses: actions/upload-artifact@v3
         with:
           name: hook-${{steps.commitid.outputs.short}}.tar.gz
-          path: hook-${{steps.commitid.outputs.short}}.tar.gz
+          path: out/${{steps.commitid.outputs.short}}/rel/hook-${{steps.commitid.outputs.short}}.tar.gz

.gitignore

@@ -1,10 +1,10 @@
 bin/
+dbg/
 dist/
 .env
-hook-bootkit/local/
-hook-debug.*.yaml
-hook-docker/local/
-hook-*.tar.gz
-hook.*.yaml
+hook-*
+!/hook-bootkit/
+!/hook-docker/
+hook*.*.yaml
 out/
 *.swp

Makefile

@@ -1,134 +1,37 @@
-ORG ?= quay.io/tinkerbell
-ARCH := $(shell uname -m)
-
-ifeq ($(strip $(TAG)),)
-  # ^ guards against TAG being defined but empty string which makes `TAG ?=` not work
-  TAG := latest
+# set the ORG
+### !!NOTE!!
+# If this is changed then a fresh output dir is required (`git clean -fxd` or just `rm -rf out`)
+# Handling this better shows some of make's suckiness compared to newer build tools (redo, tup ...) where the command lines to tools invoked isn't tracked by make
+ORG := quay.io/mmlb
+# makes sure there's no trailing / so we can just add them in the recipes which looks nicer
+ORG := $(shell echo "${ORG}" | sed 's|/*$$||')
+
+# The following `ifeq` are the equivalent of FOO ?= except that they work correctly if FOO is set but empty
+ifeq ($(strip $(LINUXKIT_CONFIG)),)
+  LINUXKIT_CONFIG := hook.yaml
 endif
-default: hook-bootkitBuild hook-dockerBuild image
 
-dev: dev-hook-bootkitBuild dev-hook-dockerBuild
-ifeq ($(ARCH),x86_64)
-dev: dev-image-amd64
-endif
-ifeq ($(ARCH),aarch64)
-dev: dev-image-arm64
+ifeq ($(strip $(TAG)),)
+  TAG := sha-$(shell git rev-parse --short HEAD)
 endif
+T := $(strip $(TAG))
 
-# This option is for running docker manifest command
-export DOCKER_CLI_EXPERIMENTAL := enabled
-
-LINUXKIT_CONFIG ?= hook.yaml
-hook.$(TAG).yaml: $(LINUXKIT_CONFIG)
-	sed '/quay.io/ s|:latest|:$(TAG)|' $^ > $@.tmp
-	mv $@.tmp $@
-
-hook-debug.$(TAG).yaml: hook.$(TAG).yaml
-	sed '/^\s*#dbg/ s|#dbg||' $^ > $@.tmp
-	mv $@.tmp $@
-
-image-amd64: hook.$(TAG).yaml
-	mkdir -p out
-	linuxkit build -docker -pull -format kernel+initrd -name hook-x86_64 -dir out $^
-
-image-arm64: hook.$(TAG).yaml
-	mkdir -p out
-	linuxkit build -docker -pull -arch arm64 -format kernel+initrd -name hook-aarch64 -dir out $^
-
-dev-image-amd64: hook.$(TAG).yaml
-	mkdir -p out
-	linuxkit build -docker -format kernel+initrd -name hook-x86_64 -dir out $^
-
-dev-image-arm64: hook.$(TAG).yaml
-	mkdir -p out
-	linuxkit build -docker -arch arm64 -format kernel+initrd -name hook-aarch64 -dir out $^
-
-image: image-amd64 image-arm64
-
-debug-image-amd64: hook-debug.$(TAG).yaml
-	mkdir -p out/amd64
-	linuxkit build --docker -format kernel+initrd -name debug-x86_64 -dir out $^
+help: ## Print this help
+	@grep --no-filename -E '^[[:alnum:]_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sed 's/:.*## /·/' | sort | column -t -W 2 -s '·' -c $$(tput cols)
 
-debug-image-arm64: hook-debug.$(TAG).yaml
-	mkdir -p out/arm64
-	linuxkit build --docker -arch arm64 -format kernel+initrd -name debug-aarch64 -dir out $^
+include rules.mk
+include lint.mk
 
-debug-image: debug-image-amd64 debug-image-arm64
+all: containers images ## Build release mode boot files and container images for all supported architectures
 
-run-amd64:
-	sudo ~/go/bin/linuxkit run qemu --mem 2048 out/hook-x86_64
+dev: image-dbg-$(ARCH) ## Build debug mode boot files and container images for currently running architecture
 
-run-arm64:
-	sudo ~/go/bin/linuxkit run qemu --mem 2048 out/hook-aarch64
+images: ## Build release mode boot files for all supported architectures
 
-run:
-	sudo ~/go/bin/linuxkit run qemu --mem 2048 out/hook-${ARCH}
+containers: hook-bootkit hook-docker ## Build container images
 
-dev-hook-bootkitBuild:
-	cd hook-bootkit; docker buildx build --load -t $(ORG)/hook-bootkit:$(TAG) .
-
-hook-bootkitBuild:
-	cd hook-bootkit; docker buildx build --platform linux/amd64,linux/arm64 --push -t $(ORG)/hook-bootkit:$(TAG) .
-
-dev-hook-dockerBuild:
-	cd hook-docker; docker buildx build --load -t $(ORG)/hook-docker:$(TAG) .
-
-hook-dockerBuild:
-	cd hook-docker; docker buildx build --platform linux/amd64,linux/arm64 --push -t $(ORG)/hook-docker:$(TAG) .
-
-dev-convert:
-	rm -rf ./convert
-	mkdir ./convert
-	cp out/hook-${ARCH}-initrd.img ./convert/initrd.gz
-	cd convert/; gunzip ./initrd.gz; cpio -idv < initrd; rm initrd; find . -print0 | cpio --null -ov --format=newc > ../initramfs-${ARCH}; gzip ../initramfs-${ARCH}
-
-.PHONY: convert
-convert:
-	for a in x86_64 aarch64; do \
-		rm -rf ./convert; \
-		mkdir ./convert; \
-		cp out/hook-$$a-initrd.img ./convert/initrd.gz; \
-		cd convert/; gunzip ./initrd.gz; cpio -idv < initrd; rm initrd; find . -print0 | cpio --null -ov --format=newc > ../initramfs-$$a; gzip ../initramfs-$$a; cd ../;\
-	done
-
-dist: default convert
-	rm -rf ./dist ./convert
-	mkdir ./dist
-	for a in x86_64 aarch64; do \
-		mv ./initramfs-$$a.gz ./dist/initramfs-$$a; \
-		mv ./out/hook-$$a-kernel ./dist/vmlinuz-$$a; \
-	done
-	rm -rf out
-	cd ./dist && tar -czvf ../hook-${TAG}.tar.gz ./*
-
-dist-existing-images: image convert
-	rm -rf ./dist ./convert
-	mkdir ./dist
-	for a in x86_64 aarch64; do \
-		mv ./initramfs-$$a.gz ./dist/initramfs-$$a; \
-		mv ./out/hook-$$a-kernel ./dist/vmlinuz-$$a; \
-	done
-	rm -rf out
-	cd ./dist && tar -czvf ../hook-${TAG}.tar.gz ./*
-
-
-dev-dist: dev dev-convert
-	rm -rf ./dist ./convert
-	mkdir ./dist
-	mv ./initramfs-${ARCH}.gz ./dist/initramfs-${ARCH}
-	mv ./out/hook-${ARCH}-kernel ./dist/vmlinuz-${ARCH}
-	rm -rf out
-	cd ./dist && tar -czvf ../hook-${TAG}.tar.gz ./*
-
-deploy: dist
-ifeq ($(shell git rev-parse --abbrev-ref HEAD),main)
-	s3cmd sync ./hook-${TAG}.tar.gz s3://s.gianarb.it/hook/${TAG}.tar.gz
-	s3cmd cp s3://s.gianarb.it/hook/hook-${TAG}.tar.gz s3://s.gianarb.it/hook/hook-main.tar.gz
-endif
+debug: ## Build debug mode boot files and container images for all supported architectures
 
-.PHONY: clean
-clean:
-	rm ./hook-${TAG}.tar.gz
-	rm -rf dist/ out/ hook-docker/local/ hook-bootkit/local/
+push: push-hook-bootkit push-hook-docker ## Push container images to registry
 
--include lint.mk
+run: run-$(ARCH) ## Boot system using qemu

README.md

@@ -105,8 +105,7 @@ The `dist` make target will do a couple of things:
 
 1. Build the required docker images using `docker buildx`.
 2. It will use `linuxkit build` to prepare the init ramdisk and the kernel.
-3. It will convert the init ramkdisk in the right format that iPXE can boot
-4. It will create a `tar.gz` archive in the root of the project containing all the files in the proper format, ready to be served via Tinkerbell.
+3. It will create a `tar.gz` archive in the root of the project containing all the files in the proper format, ready to be served via boots.
 
 ## Build for local testing (only the local architecture)
 
@@ -121,10 +120,8 @@ make dev-dist
 
 ## Troubleshooting
 
-Sometimes this has occurred with changed one letter of a string inside some source code and rebuilding... not sure why yet.
-
-It is also possible to build a debug version of hook, that will have an `sshd` server running with any public keys you have.
-This is achieved through the command `make debug-image`
+It is possible to build a debug version of hook, that will have an `sshd` server running with any public keys you have.
+This is achieved through the command `make debug-images`
 
 ## Nix for CI/CD
 

kernel/Makefile

@@ -15,8 +15,15 @@
 # This option is for running docker manifest command
 export DOCKER_CLI_EXPERIMENTAL := enabled
 
-# Name and Org on Hub
-ORG?=quay.io/tinkerbell
+# set ORG in the env or make call
+# having it in the makefile is dangerous as its too easy to overwrite previous pushes/tags
+ifeq ($(strip $(ORG)),)
+# ^ guards against ORG being defined but empty string which makes `ORG ?=` not work
+  ORG := quay.io/tinkerbell
+endif
+# makes sure there's no trailing / so we can just add them in the recipes which looks nicer
+ORG := $(shell echo "${ORG}" | sed 's|/*$$||')
+
 IMAGE:=hook-kernel
 IMAGE_BCC:=hook-kernel-bcc
 IMAGE_PERF:=hook-kernel-perf

rules.mk

@@ -0,0 +1,107 @@
+# Only use the recipes defined in these makefiles
+MAKEFLAGS += --no-builtin-rules
+.SUFFIXES:
+# Delete target files if there's an error
+# This avoids a failure to then skip building on next run if the output is created by shell redirection for example
+# Not really necessary for now, but just good to have already if it becomes necessary later.
+.DELETE_ON_ERROR:
+# Treat the whole recipe as a one shell script/invocation instead of one-per-line
+.ONESHELL:
+# Use bash instead of plain sh
+SHELL := bash
+.SHELLFLAGS := -o pipefail -euc
+
+# This option is for running docker manifest command
+export DOCKER_CLI_EXPERIMENTAL := enabled
+
+ARCH := $(shell uname -m)
+ifeq ($(ARCH),x86_64)
+ARCH = amd64
+endif
+ifeq ($(ARCH),aarch64)
+ARCH = arm64
+endif
+
+arches := amd64 arm64
+modes := rel dbg
+
+hook-bootkit-deps := $(wildcard hook-bootkit/*)
+hook-docker-deps := $(wildcard hook-docker/*)
+
+define foreach_mode_arch_rules =
+# mode := $(1)
+# arch := $(2)
+
+$$(shell mkdir -p out/$T/$(1)/$(2))
+
+.PHONY: hook-$(1)-$(2)
+image-$(1)-$(2): out/$T/$(1)/$(2)/hook.tar
+out/$T/$(1)/$(2)/hook.tar: out/$T/$(1)/$(2)/hook.yaml out/$T/hook-bootkit-$(2) out/$T/hook-docker-$(2)
+	linuxkit build -docker -arch $(2) -format tar-kernel-initrd -name hook -dir $$(@D) $$<
+	mv $$(@D)/hook-initrd.tar $$@
+
+out/$T/$(1)/$(2)/cmdline out/$T/$(1)/$(2)/initrd.img out/$T/$(1)/$(2)/kernel: out/$T/$(1)/$(2)/hook.tar
+	tar xf $$^ -C $$(@D) $$(@F)
+	touch $$@
+
+out/$T/$(1)/$(2)/hook.yaml: $$(LINUXKIT_CONFIG)
+	sed '/hook-\(bootkit\|docker\):/ { s|:latest|:$T-$(2)|; s|quay.io/tinkerbell|$(ORG)|; }' $$< > $$@
+	if [[ $(1) == dbg ]]; then
+	    sed -i '/^\s*dbg/ s|#dbg||' $$@
+	fi
+endef
+$(foreach m,$(modes),$(foreach a,$(arches),$(eval $(call foreach_mode_arch_rules,$m,$a))))
+
+define foreach_arch_rules =
+# arch := $(1)
+
+debug: out/$T/dbg/$(1)/hook.tar
+images: out/$T/rel/$(1)/hook.tar
+image-dbg-$(1): out/$T/dbg/$(1)/hook.tar
+
+out/$T/rel/hook.tar: out/$T/rel/$(1)/initrd.img out/$T/rel/$(1)/kernel
+hook-bootkit: out/$T/hook-bootkit-$(1)
+hook-docker: out/$T/hook-docker-$(1)
+
+out/$T/hook-bootkit-$(1): $$(hook-bootkit-deps)
+out/$T/hook-docker-$(1): $$(hook-docker-deps)
+out/$T/hook-bootkit-$(1) out/$T/hook-docker-$(1): platform=linux/$$(lastword $$(subst -, ,$$(notdir $$@)))
+out/$T/hook-bootkit-$(1) out/$T/hook-docker-$(1): container=hook-$$(word 2,$$(subst -, ,$$(notdir $$@)))
+out/$T/hook-bootkit-$(1) out/$T/hook-docker-$(1):
+	docker buildx build --platform $$(platform) --load -t $(ORG)/$$(container):$T-$(1) $$(container)
+	touch $$@
+
+run-$(1): out/$T/dbg/$(1)/hook.tar
+run-$(1):
+	linuxkit run qemu --mem 2048 $$^
+endef
+$(foreach a,$(arches),$(eval $(call foreach_arch_rules,$a)))
+
+push-hook-bootkit: $(hook-bootkit-deps)
+push-hook-docker: $(hook-docker-deps)
+push-hook-bootkit push-hook-docker: platforms=$(addprefix linux/,$(arches))
+push-hook-bootkit push-hook-docker: container=hook-$(lastword $(subst -, ,$(basename $@)))
+push-hook-bootkit push-hook-docker:
+	platforms="$(platforms)"
+	platforms=$${platforms// /,}
+	docker buildx build --platform $$platforms --push -t $(ORG)/$(container):$T $(container)
+
+.PHONY: dist
+dist: out/$T/rel/hook-$T.tar.gz ## Build tarball for distribution
+out/$T/rel/hook-$T.tar.gz: out/$T/rel/hook.tar
+	pigz < $< > $@
+
+out/$T/rel/hook.tar:
+	rm -rf out/$T/rel/dist
+	mkdir -p out/$T/rel/dist
+	for a in $(arches); do
+		cp out/$T/rel/$$a/initrd.img out/$T/rel/dist/initramfs-$$a
+		cp out/$T/rel/$$a/kernel out/$T/rel/dist/vmlinuz-$$a
+	done
+	cd out/$T/rel/dist && tar -cvf ../$(@F) ./*
+
+deploy: dist ## Push tarball to S3
+ifeq ($(shell git rev-parse --abbrev-ref HEAD),main)
+	s3cmd sync ./out/hook-$T.tar.gz s3://s.gianarb.it/hook/$T.tar.gz
+	s3cmd cp s3://s.gianarb.it/hook/hook-$T.tar.gz s3://s.gianarb.it/hook/hook-main.tar.gz
+endif