Speed-up make generate (gardener#3489)

* Speed-up make generate

* Speed-up make generate code review changes

.ci/verify

@@ -10,7 +10,7 @@ git config --global user.email "gardener@sap.com"
 git config --global user.name "Gardener CI/CD"
 
 apt-get update
-apt-get install -y unzip jq
+apt-get install -y unzip jq parallel
 
 mkdir -p /go/src/github.com/gardener/gardener
 cp -r . /go/src/github.com/gardener/gardener

Makefile

@@ -159,8 +159,18 @@ check:
 	@hack/check.sh --golangci-lint-config=./.golangci.yaml ./cmd/... ./extensions/... ./pkg/... ./plugin/... ./test/...
 	@hack/check-charts.sh ./charts
 
+# We need to explicitly pass GO111MODULE=off to k8s.io/code-generator as it is significantly slower otherwise,
+# see https://github.com/kubernetes/code-generator/issues/100.
 .PHONY: generate
 generate:
+	@GO111MODULE=off hack/update-protobuf.sh
+	@GO111MODULE=off hack/update-codegen.sh --parallel
+	@hack/generate-parallel.sh cmd extensions pkg plugin test
+
+.PHONY: generate-sequential
+generate-sequential:
+	@GO111MODULE=off hack/update-protobuf.sh
+	@GO111MODULE=off hack/update-codegen.sh 
 	@hack/generate.sh ./cmd/... ./extensions/... ./pkg/... ./plugin/... ./test/...
 
 .PHONY: generate-extensions-crds

docs/development/local_setup.md

@@ -101,6 +101,16 @@ go get -u github.com/bronze1man/yaml2json
 brew install jq
 ```
 
+## Installing GNU Parallel
+
+[GNU Parallel](https://www.gnu.org/software/parallel/) is a shell tool for executing jobs in parallel, used by the code generation scripts (`make generate`).
+
+On MacOS run
+
+```bash
+brew install parallel
+```
+
 ## [MacOS only] Install GNU core utilities
 
 When running on MacOS you have to install the GNU core utilities:

hack/clean.sh

@@ -20,5 +20,14 @@ echo "> Clean"
 
 for source_tree in $@; do
   find "$(dirname "$source_tree")" -type f -name "zz_*.go" -exec rm '{}' \;
-  grep -lr --include="*.go" "//go:generate packr2" . | xargs -I {} packr2 clean "{}/.."
+  find "$(dirname "$source_tree")" -type f -name "generated.proto" -exec rm '{}' \;
+  find "$(dirname "$source_tree")" -type f -name "generated.pb.go" -exec rm '{}' \;
+  find "$(dirname "$source_tree")" -type f -name "openapi_generated.go" -exec rm '{}' \;
+  grep -lr '// Code generated by MockGen. DO NOT EDIT' "$(dirname "$source_tree")" | xargs rm -f
+  grep -lr '// Code generated by client-gen. DO NOT EDIT' "$(dirname "$source_tree")" | xargs rm -f
+  grep -lr '// Code generated by informer-gen. DO NOT EDIT' "$(dirname "$source_tree")" | xargs rm -f
+  grep -lr '// Code generated by lister-gen. DO NOT EDIT' "$(dirname "$source_tree")" | xargs rm -f
+  grep -lr --include="*.go" "//go:generate packr2" "$(dirname "$source_tree")" | xargs -I {} packr2 clean "{}/.."
 done
+
+find ./hack/api-reference/ -type f -name "*.md" -exec rm '{}' \;

hack/generate-parallel.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+CURRENT_DIR="$(dirname $0)"
+PROJECT_ROOT="${CURRENT_DIR}"/..
+if [ "${PROJECT_ROOT#/}" == "${PROJECT_ROOT}" ]; then
+  PROJECT_ROOT="./$PROJECT_ROOT"
+fi
+
+pushd "$PROJECT_ROOT" > /dev/null
+ROOTS=${ROOTS:-$(git grep --files-with-matches -e '//go:generate' "$@" | \
+	xargs -n 1 dirname | \
+	sed 's,^,github.com/gardener/gardener/,;' | \
+	sort | uniq
+)}
+popd > /dev/null
+
+read -ra PACKAGES <<< $(echo ${ROOTS})
+
+# We need to explicitly pass GO111MODULE=off to k8s.io/code-generator as it is significantly slower otherwise,
+# see https://github.com/kubernetes/code-generator/issues/100.
+GO111MODULE=off parallel --will-cite echo Generate {}';' go generate -mod=vendor {} ::: ${PACKAGES[@]}

hack/generate.sh

@@ -18,4 +18,6 @@ set -e
 
 echo "> Generate"
 
-GO111MODULE=on go generate -mod=vendor $@
+# We need to explicitly pass GO111MODULE=off to k8s.io/code-generator as it is significantly slower otherwise,
+# see https://github.com/kubernetes/code-generator/issues/100.
+GO111MODULE=off go generate -mod=vendor $@

hack/install-requirements.sh

@@ -25,10 +25,8 @@ curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/insta
 curl -s "https://raw.githubusercontent.com/helm/helm/v2.17.0/scripts/get" | bash -s -- --version 'v2.17.0'
 
 platform=$(uname -s)
-if [[ ${platform} == "Linux" ]]
-then
-  if ! which jq &>/dev/null
-  then
+if [[ ${platform} == "Linux" ]]; then
+  if ! which jq &>/dev/null; then
     echo "Installing jq ..."
     curl -L -o /usr/local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
     chmod +x /usr/local/bin/jq
@@ -44,9 +42,10 @@ Please make sure you have installed the following requirements:
 - GNU Core Utils
 - GNU Tar
 - GNU Sed
+- GNU Parallel
 
 Brew command:
-$ brew install coreutils gnu-tar gnu-sed jq
+$ brew install coreutils gnu-tar gnu-sed jq parallel
 
 Please allow them to be used without their "g" prefix:
 $ export PATH=/usr/local/opt/coreutils/libexec/gnubin:\$PATH