Chore: Reduce verbosity of buildall (#247)

zdw · web-flow · commit 84f15d062a0f · 2025-05-01T14:55:13.000Z
diff --git a/buildall/Makefile b/buildall/Makefile
@@ -26,13 +26,13 @@ SCRATCH_DIR := scratch
 CHARTS_DIR  := charts
 
 $(REPO_DIR):
-	mkdir -p $@
+	@mkdir -p $@
 
 $(SCRATCH_DIR):
-	mkdir -p $@
+	@mkdir -p $@
 
 $(CHARTS_DIR):
-	mkdir -p $@
+	@mkdir -p $@
 
 check: ## check for prereqs (currently noOp)
 
@@ -48,7 +48,8 @@ VENV_DIR     := venv_$(PROJECT_NAME)
 
 # virtualenv activate script has undefined variables, disable then re-enable in bash
 $(VENV_DIR): requirements.txt
-	python3 -m venv $@ ;\
+	@echo "## Building Python Virtualenv ##"
+	@python3 -m venv $@ ;\
   set +u; . ./$@/bin/activate; set -u ;\
   python -m pip install --upgrade pip ;\
   python -m pip install -r requirements.txt
@@ -86,18 +87,16 @@ $(REPO_DIR)/%: | $(REPO_LIST)
 	git clone https://github.com/$(GITHUB_ORG)/$*.git $(REPO_DIR)/$*
 
 checkout-repos: $(REPOS) | $(REPO_LIST) ## checkout all repos
-	@echo "All checked out"
+	@echo "## All repos have been checked out ##"
 
 GIT_BRANCH ?= main
-branch-checkout: checkout-repos ## refresh and checkout all repos with a specific branch
-	for repo in $(REPOS); do \
+branch-checkout: checkout-repos ## refresh and checkout all repos at branch GIT_BRANCH (default: main)
+	@for repo in $(REPOS); do \
 		pushd $$repo ;\
 		git checkout $(GIT_BRANCH) ; git pull --ff-only ;\
 	  popd ;\
 	done
 
-#	git fetch ; git checkout $(GIT_BRANCH) ; git pull --ff-only ;\
-	#
 # Check if there are the right make targets needed in the repo
 MAKEFILE_CHECKS := $(addprefix $(SCRATCH_DIR)/make_check_, $(shell tr '\n' ' ' < $(REPO_LIST)))
 check-makefiles: $(REPOS) | $(MAKEFILE_CHECKS)
@@ -106,64 +105,65 @@ $(SCRATCH_DIR)/make_check_%:
 	$(SCRIPTS_DIR)/check_makefile.sh repos/$* |tee $@
 
 ## targets used to run buildall ##
-
 buildall: $(SCRATCH_DIR) ## perform the entire build
-	$(SCRIPTS_DIR)/buildall.sh
+	@$(SCRIPTS_DIR)/buildall.sh
 
-timed-buildall: $(SCRATCH_DIR)
+timed-buildall: $(SCRATCH_DIR) ## perform the entire build with build times and verbose output
 	$(SCRIPTS_DIR)/timed_buildall.sh
 
 # list all artifacts used in each repo
 ARTIFACT_LIST := $(addprefix $(SCRATCH_DIR)/artifacts_, $(addsuffix .yaml, $(shell tr '\n' ' ' < $(REPO_LIST))))
 list-artifacts: $(REPOS) | $(SCRATCH_DIR) $(ARTIFACT_LIST) ## list charts/images provided by each repo
+	@echo "## All artifact lists have been created ##"
 
 $(SCRATCH_DIR)/artifacts_%.yaml:
-	$(SCRIPTS_DIR)/list_artifacts.sh repos/$* |tee $@
+	@echo "### Generating artifact list in 'repos/$*' ###" ;\
+	$(SCRIPTS_DIR)/list_artifacts.sh repos/$* > $@
 
 # determine work needed with orch-deploy manifest
 chart-manifest: $(SCRATCH_DIR)/manifest_charts.yaml ## build chart manifest
 
 $(SCRATCH_DIR)/manifest_charts.yaml: $(SCRATCH_DIR)
-	pushd .. ;\
-	echo "### Generating releaseManifest ####" ;\
+	@pushd .. > /dev/null ;\
+	echo "## Generating releaseManifest of required helm charts ##" ;\
 	mage gen:releaseManifest $(PROJECT_NAME)/$@ ;\
-	echo "### Generated releaseManifest ####" ;\
-	popd
+	popd > /dev/null
 
 sort-charts: $(VENV_DIR)  ## determine which repos can build charts required by manifest, and write to htags_
-	set +u; . ./$</bin/activate; set -u ;\
+	@set +u; . ./$</bin/activate; set -u ;\
 	python scripts/sort_charts.py
 
 image-manifest: $(SCRATCH_DIR)/manifest_images.yaml  ## build image manifest
 $(SCRATCH_DIR)/manifest_images.yaml: $(SCRATCH_DIR)
-	pushd .. ;\
-	echo "### Generating localReleaseImageManifest from charts ####" ;\
+	@pushd .. > /dev/null ;\
+	echo "## Generating localReleaseImageManifest of docker images from charts ###" ;\
 	mage gen:localReleaseImageManifest $(PROJECT_NAME)/$@ > $(PROJECT_NAME)/$@.log ;\
-	echo "### Generated localReleaseImageManifest ####" ;\
-	popd
+	popd > /dev/null
 
 sort-images: $(VENV_DIR)  ## determine which repos can build images required by manifest, and write to itags_
-	set +u; . ./$</bin/activate; set -u ;\
+	@set +u; . ./$</bin/activate; set -u ;\
 	python scripts/sort_images.py
 
 # build charts
 HELM_BUILDS := $(addprefix $(SCRATCH_DIR)/hbuild_, $(shell tr '\n' ' ' < $(REPO_LIST)))
 helm-build: $(HELM_BUILDS) $(CHARTS_DIR) ## build all helm charts
+	@echo "## All helm charts have been created ##"
 
 $(SCRATCH_DIR)/hbuild_%: $(CHARTS_DIR)
-	$(SCRIPTS_DIR)/helm_build.sh $* $@
+	@$(SCRIPTS_DIR)/helm_build.sh $* $@
 
-clean-helm:  ## clean up helm charts to rebuild
+clean-helm:  ## clean up helm chart scratchfiles to force rebuild
 	rm -rf $(SCRATCH_DIR)/hbuild_*
 
 # build images
 IMAGE_BUILDS := $(addprefix $(SCRATCH_DIR)/ibuild_, $(shell tr '\n' ' ' < $(REPO_LIST)))
 image-build: $(IMAGE_BUILDS) ## build all container images
+	@echo "## All docker images have been created ##"
 
 $(SCRATCH_DIR)/ibuild_%:
-	$(SCRIPTS_DIR)/image_build.sh $* $@
+	@$(SCRIPTS_DIR)/image_build.sh $* $@
 
-clean-image: ## clean up docker images to rebuild
+clean-image: ## clean up docker image scratchfiles to force rebuild
 	rm -rf $(SCRATCH_DIR)/ibuild_*
 
 clean: clean-helm clean-image ## clean up scratch files
diff --git a/buildall/README.md b/buildall/README.md
@@ -7,6 +7,11 @@ To ensure that a tested Best Known Configuration set of components are used,
 the versions rebuilt are derived from the versions specified in the current
 checkout of the `edge-manageability-framework` repository.
 
+Please see the [Edge Orchestrator docs on
+buildall](https://docs.openedgeplatform.intel.com/edge-manage-docs/main/developer_guide/platform/buildall.html)
+to decide whether this process is suitable for your development or deployment
+process.
+
 ## Prerequisites
 
 Performing a buildall requires the following tools to be installed in a
@@ -18,13 +23,26 @@ Linux/Unix like environment:
   - The built docker images require around 5GB of space
   - Approximately roughly 50GB for build cache. See `DOCKER_PRUNE` below
     to optionally prune the build cache during the build process.
-- Helm 3
+- Helm 3 (3.17 or later recommended)
 - Python 3 (in-support version such as 3.10 or later, tested on 3.12)
-- Mage
-
-Additionally, some of the containers that are build require the following
-
-- Open Policy Agent v0.69.0 (or possibly later) for infra-core repo
+- Mage 1.15 or later
+
+Additionally, some of the Makefiles in component repos require the following
+dependencies to run properly:
+
+- Open Policy Agent (`opa` cli tool) v0.69.0 (or possibly later) for infra-core repo
+- golangci-lint (v1.64.8 or later)
+- go-junit-report (v2.1.0 or later)
+- oapi-codegen (v1.12.0 or later)
+- protoc-gen-doc (1.5.1 or later)
+- buf (1.45.0 or later)
+- protoc-gen-go-grpc (1.2.0 or later)
+- protoc-gen-go (v1.30.0 or later)
+- protoc-gen-openapi (latest from
+  https://github.com/kollalabs/protoc-gen-openapi)
+- protoc-gen-grpc-gateway (v2.26.3)
+- swagger (4.0.4 or later)
+- yq (4.44.3 or later)
 
 ## Configuration
 
@@ -47,6 +65,8 @@ docker build, set `DOCKER_PRUNE` to `Y`.
 
 ## Usage
 
+### Build
+
 All commands are run using `make`.
 
 You can get a list of all the make targets with `make help`
@@ -57,7 +77,8 @@ To run the full process with the BKC from the currently checked out copy of
 To cleanup internal scratch state, run `make clean`
 
 To fully cleanup (including tools), run `make clean-all`.  Note that this only
-cleans up local files, not container images that were built.
+cleans up local files, not container images that were built, which can be done
+with `docker rmi` or similar commands.
 
 ### How it works
 
diff --git a/buildall/env.sh b/buildall/env.sh
@@ -33,6 +33,9 @@ export DOCKER_PRUNE=
 # don't color output
 export NO_COLOR=1
 
+# avoid checking for tool versions/presence
+export TOOL_VERSION_CHECK=0
+
 # Github Token - used to build certain images
 # Obtain from: https://github.com/settings/tokens
 export GITHUB_TOKEN=
diff --git a/buildall/scripts/buildall.sh b/buildall/scripts/buildall.sh
@@ -5,7 +5,7 @@
 
 # buildall.sh
 
-set -xeu -o pipefail
+set -eu -o pipefail
 
 # bring in environmental variables. Where MAKE_JOBS is set
 source env.sh
diff --git a/buildall/scripts/helm_build.sh b/buildall/scripts/helm_build.sh
@@ -34,13 +34,13 @@ fi
 TAGS_PATH="scratch/htags_${REPO}"
 
 if [ ! -f "${TAGS_PATH}" ]; then
-  echo "'${TAGS_PATH}' does not exist, so repo does not create any helm charts"
+  echo "### No helm charts built in repos/${REPO} ###"
   exit 0
 fi
 
 TAG_FILE=$(cat "${TAGS_PATH}")
 
-pushd "${REPO_PATH}"
+pushd "${REPO_PATH}" >> "${LOGFILE}"
   for tline in $TAG_FILE; do
 
     tag=$(cut -d '|' -f 1 <<< "$tline")
@@ -51,33 +51,32 @@ pushd "${REPO_PATH}"
       outDir="."
     fi
 
-    git switch --detach "${tag}"
+    git switch --detach "${tag}" >> "${LOGFILE}" 2>&1
 
     # check if helm-build target exists in Makefile
-    set +eu
+    set +e
     HELM_BUILD_TARGET=$(grep ^helm-build Makefile)
-    set -eu
+    set -e
 
     if [ ! "${HELM_BUILD_TARGET}" ]; then
-      echo "### Copying Makefile from main branch as helm-build target doesn't exist ###"
-      git checkout main Makefile
+      echo "#### Copying Makefile from main branch as helm-build target doesn't exist ####" \
+        | tee -a "${LOGFILE}"
+      git checkout main Makefile  >> "${LOGFILE}" 2>&1
     fi
 
-    echo "### Starting make helm-build in: '${REPO}', tag: '${tag}' ###"
+    echo "### Running 'make helm-build' in 'repos/${REPO}', tag: '${tag}' ###"
 
     make helm-build >> "${LOGFILE}" 2>&1
 
     mv "${outDir}"/*.tgz ../../charts
 
     # clean up repo
-    git checkout HEAD .
-
-    echo "### Finished make helm-build in: '${REPO}', tag: '${tag}', output in: '${outDir}' ###"
+    git checkout HEAD . >> "${LOGFILE}" 2>&1
 
   done
-popd
+popd >> "${LOGFILE}"
 
 END_TS=$(date +"%s")
 ELAPSED=$(( END_TS - START_TS ))
 
-echo "### helm-build took ${ELAPSED} seconds in repo: '${REPO}' ###"
+echo "### All helm chart builds complete in 'repos/${REPO}', took ${ELAPSED}s ###"
diff --git a/buildall/scripts/image_build.sh b/buildall/scripts/image_build.sh
@@ -34,13 +34,13 @@ fi
 TAGS_PATH="scratch/itags_${REPO}"
 
 if [ ! -f "${TAGS_PATH}" ]; then
-  echo "'${TAGS_PATH}' does not exist, so repo does not create any container images"
+  echo "### No docker images built in repos/${REPO} ###"
   exit 0
 fi
 
 TAG_FILE=$(cat "${TAGS_PATH}")
 
-pushd "${REPO_PATH}"
+pushd "${REPO_PATH}" >> "${LOGFILE}"
   for tline in $TAG_FILE; do
 
     tag=$(cut -d '|' -f 1 <<< "$tline")
@@ -51,9 +51,9 @@ pushd "${REPO_PATH}"
       target='docker-build'
     fi
 
-    echo "*** Starting docker build in: '${REPO}', tag: '${tag}', target: '${target}' ***"
+    echo "### Running 'make ${target}' in 'repos/${REPO}', tag: '${tag}' ###"
 
-    git switch --detach "${tag}"
+    git switch --detach "${tag}" >> "${LOGFILE}" 2>&1
 
     # regex matches all the end-of-string numbers/dots
     if [[ "${tag}" =~ ([0-9.]+)$ ]]
@@ -68,16 +68,15 @@ pushd "${REPO_PATH}"
     GITHUB_TOKEN=$GITHUB_TOKEN \
     make "${target}" >> "${LOGFILE}" 2>&1
 
-    echo "*** Finished docker build in: '${REPO}', tag: '${tag}', target: '${target}' ***"
   done
-popd
+popd >> "${LOGFILE}"
 
 # optionally cleanup docker buildx cache
 if [ -n "${DOCKER_PRUNE}" ]; then
-  echo "### Pruning docker buildx cache ###"
-  docker buildx prune -f
+  echo "#### Pruning docker buildx cache ####" | tee "${LOGFILE}"
+  docker buildx prune -f  >> "${LOGFILE}" 2>&1
 fi
 
 END_TS=$(date +"%s")
 ELAPSED=$(( END_TS - START_TS ))
-echo "### Docker build in: '${REPO}' took ${ELAPSED} seconds ###"
+echo "### All docker image builds complete in: 'repos/${REPO}', took ${ELAPSED}s ###"
diff --git a/buildall/scripts/sort_charts.py b/buildall/scripts/sort_charts.py
@@ -73,8 +73,6 @@ def match_charts(chart_mf, chart_to_repo, artifacts):
                 for line in data["charts"]:
                     htagout.write(f"{line}\n")
 
-    print("chart builds needed in each repo:")
-    yaml.dump(repo_tags_to_build, sys.stdout)
-
-    if req_charts:  # only print missing charts if they exist
-        print(f"missing charts: {req_charts}")
+    if req_charts:  # die if missing charts if they exist
+        print(f"ERROR - Missing charts: {req_charts}")
+        sys.exit(1)
diff --git a/buildall/scripts/sort_images.py b/buildall/scripts/sort_images.py
@@ -53,12 +53,12 @@ def parse_image_manifest(image_mf_raw):
                 if len(split) == 1:
                     name, version = split[0]
                 else:
-                    print(f"Underspecified image name: {rawi}")
+                    print(f"INFO: Underspecified image name: {rawi}")
 
         if name in image_mf_fix["images"]:
             if image_mf_fix["images"][name]["registry"] != registry:
                 print(
-                    f"Duplicate image: '{name}' has registry '{registry}' which "
+                    f"INFO: Duplicate image: '{name}' has registry '{registry}' which "
                     f"doesn't match earlier '{image_mf_fix['images'][name]['registry']}'"
                 )
             else:
@@ -102,9 +102,6 @@ def match_images(image_mf, image_to_repo, artifacts):
 
     match_images(image_manifest, itr, repo_artifacts)
 
-    print("builds needed in each repo")
-    print(repo_tags_to_build)
-
     for rname, data in repo_tags_to_build.items():
         if data["images"]:  # If there are charts in the repo
 
