Enabling a new DevKit build job and support use within pipeline configuration

pipelines/build/common/openjdk_build_pipeline.groovy

@@ -12,6 +12,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+/* groovylint-disable MethodCount */
+
 import common.IndividualBuildConfig
 import common.MetaData
 import common.VersionInfo
@@ -1482,6 +1484,15 @@ class Build {
             context.println '    buildConfig.BUILD_REF: ' + buildConfig.BUILD_REF
             context.println '    buildConfig.HELPER_REF: ' + buildConfig.HELPER_REF
 
+            def openjdk_build_dir = context.WORKSPACE + '/workspace/build/src/build'
+            def openjdk_build_dir_arg = ""
+            if (getJavaVersionNumber() >= 21) {
+                // For reproducible jdk-21+ builds ensure not to build within the openjdk source folder
+                // so that debug symbols can be reproducibly mapped (https://bugs.openjdk.org/browse/JDK-8326685)
+                openjdk_build_dir =  context.WORKSPACE + '/workspace/build/openjdkbuild'
+                openjdk_build_dir_arg = " --user-openjdk-build-root-directory ${openjdk_build_dir}"
+            }
+
             if (cleanWorkspace) {
                 try {
                     try {
@@ -1562,9 +1573,9 @@ class Build {
                                     context.println "Processing exploded build, sign JMODS, and assemble build, for platform ${buildConfig.TARGET_OS} version ${buildConfig.JAVA_TO_BUILD}"
                                     def signBuildArgs
                                     if (env.BUILD_ARGS != null && !env.BUILD_ARGS.isEmpty()) {
-                                        signBuildArgs = env.BUILD_ARGS + ' --make-exploded-image'
+                                        signBuildArgs = env.BUILD_ARGS + ' --make-exploded-image' + openjdk_build_dir_arg
                                     } else {
-                                        signBuildArgs = '--make-exploded-image'
+                                        signBuildArgs = '--make-exploded-image' + openjdk_build_dir_arg
                                     }
                                     context.withEnv(['BUILD_ARGS=' + signBuildArgs]) {
                                         context.println 'Building an exploded image for signing'
@@ -1675,16 +1686,24 @@ class Build {
 
                                     def assembleBuildArgs
                                     if (env.BUILD_ARGS != null && !env.BUILD_ARGS.isEmpty()) {
-                                        assembleBuildArgs = env.BUILD_ARGS + ' --assemble-exploded-image'
+                                        assembleBuildArgs = env.BUILD_ARGS + ' --assemble-exploded-image' + openjdk_build_dir_arg
                                     } else {
-                                        assembleBuildArgs = '--assemble-exploded-image'
+                                        assembleBuildArgs = '--assemble-exploded-image' + openjdk_build_dir_arg
                                     }
                                     context.withEnv(['BUILD_ARGS=' + assembleBuildArgs]) {
                                         context.println 'Assembling the exploded image'
                                         context.sh(script: "./${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']}")
                                     }
                                 } else {
-                                    context.sh(script: "./${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']}")
+                                    def buildArgs
+                                    if (env.BUILD_ARGS != null && !env.BUILD_ARGS.isEmpty()) {
+                                        buildArgs = env.BUILD_ARGS + openjdk_build_dir_arg
+                                    } else {
+                                        buildArgs = openjdk_build_dir_arg
+                                    }
+                                    context.withEnv(['BUILD_ARGS=' + buildArgs]) {
+                                        context.sh(script: "./${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']}")
+                                    }
                                 }
                                 context.println '[CHECKOUT] Reverting pre-build adoptium/temurin-build checkout...'
                                 // Special case for the pr tester as checking out to the user's pipelines doesn't play nicely
@@ -1700,7 +1719,15 @@ class Build {
                                 repoHandler.setUserDefaultsJson(context, DEFAULTS_JSON)
                                 repoHandler.checkoutUserBuild(context)
                                 printGitRepoInfo()
-                                context.sh(script: "./${DEFAULTS_JSON['scriptDirectories']['buildfarm']}")
+                                def buildArgs
+                                if (env.BUILD_ARGS != null && !env.BUILD_ARGS.isEmpty()) {
+                                    buildArgs = env.BUILD_ARGS + openjdk_build_dir_arg
+                                } else {
+                                    buildArgs = openjdk_build_dir_arg
+                                }
+                                context.withEnv(['BUILD_ARGS=' + buildArgs]) {
+                                    context.sh(script: "./${DEFAULTS_JSON['scriptDirectories']['buildfarm']}")
+                                }
                                 context.println '[CHECKOUT] Reverting pre-build user temurin-build checkout...'
                                 repoHandler.checkoutUserPipelines(context)
                                 printGitRepoInfo()
@@ -1766,10 +1793,12 @@ class Build {
                                         context.println "Failed to clean ${e}"
                                     }
                                 } else if (cleanWorkspaceBuildOutputAfter) {
-                                    context.println 'Cleaning workspace build output files: ' + context.WORKSPACE + '/workspace/build/src/build'
-                                    context.sh(script: 'rm -rf ' + context.WORKSPACE + '/workspace/build/src/build')
+                                    context.println 'Cleaning workspace build output files: ' + openjdk_build_dir
+                                    context.sh(script: 'rm -rf ' + openjdk_build_dir)
                                     context.println 'Cleaning workspace build output files: ' + context.WORKSPACE + '/workspace/target'
                                     context.sh(script: 'rm -rf ' + context.WORKSPACE + '/workspace/target')
+                                    context.println 'Cleaning workspace build output files: ' + context.WORKSPACE + '/workspace/build/devkit'
+                                    context.sh(script: 'rm -rf ' + context.WORKSPACE + '/workspace/build/devkit')
                                 }
                             } else {
                                 context.println 'Warning: Unable to clean workspace as context.WORKSPACE is null/empty'

pipelines/build/devkit/Tools.gmk.patch

@@ -0,0 +1,141 @@
+diff --git a/make/devkit/Tools.gmk b/make/devkit/Tools.gmk
+index 187320ca2..9f1454d5e 100644
+--- a/make/devkit/Tools.gmk
++++ b/make/devkit/Tools.gmk
+@@ -62,6 +62,23 @@ ifeq ($(BASE_OS), OL)
+     BASE_URL := http://yum.oracle.com/repo/OracleLinux/OL6/4/base/$(ARCH)/
+     LINUX_VERSION := OL6.4
+   endif
++else ifeq ($(BASE_OS), Centos)
++  DEFAULT_OS_VERSION := 7
++  ifeq ($(BASE_OS_VERSION), )
++    BASE_OS_VERSION := $(DEFAULT_OS_VERSION)
++  endif
++  BASE_OS_MAJOR_VERSION := $(shell echo $(BASE_OS_VERSION) | cut -d'.' -f1)
++  ifeq ($(BASE_OS_VERSION),7.9.2009)
++     CENTOS_MIRROR := mirror.centos.org
++  else
++     CENTOS_MIRROR := vault.centos.org
++  endif
++  ifeq ($(ARCH), x86_64)
++    BASE_URL := http://$(CENTOS_MIRROR)/centos/$(BASE_OS_VERSION)/os/$(ARCH)/Packages/
++  else
++    BASE_URL := http://$(CENTOS_MIRROR)/altarch/$(BASE_OS_VERSION)/os/$(ARCH)/Packages/
++  endif
++  LINUX_VERSION := Centos$(BASE_OS_VERSION)
+ else ifeq ($(BASE_OS), Fedora)
+   ifeq ($(ARCH), riscv64)
+     DEFAULT_OS_VERSION := rawhide/68692
+@@ -87,6 +104,7 @@ else ifeq ($(BASE_OS), Fedora)
+       BASE_URL := https://archives.fedoraproject.org/pub/archive/$(FEDORA_TYPE)/releases/$(BASE_OS_VERSION)/Everything/$(ARCH)/os/Packages/
+     endif
+   endif
++  BASE_OS_MAJOR_VERSION := $(shell echo $(BASE_OS_VERSION) | cut -d'.' -f1)
+   LINUX_VERSION := Fedora_$(BASE_OS_VERSION)
+ else
+   $(error Unknown base OS $(BASE_OS))
+@@ -95,16 +113,38 @@ endif
+ ##########################################################################################
+ # Define external dependencies
+
++# Centos GPG KEYS
++Centos_7_GPG_KEY_aarch64 := EF8F3CA66EFDF32B36CDADF76C7CB6EF305D49D6 #gitleaks:allow
++Centos_7_GPG_KEY_x86_64 := 6341AB2753D78A78A7C27BB124C6A8A7F4A80EB5  #gitleaks:allow
++Centos_7_GPG_KEY_ppc64le := BAFA3436FC50768E3C3C2E4EA963BBDBF533F4FA #gitleaks:allow
++# Fedora GPG KEYS
++Fedora_19_GPG_KEY_s390x := CA81B2C85E4F4D4A1A3F723407477E65FB4B18E6   #gitleaks:allow
++
++# PGP Signature file extensions
++GCC_SIG := sig
++BINUTILS_SIG := sig
++CCACHE_SIG := asc
++GMP_SIG := sig
++MPC_SIG := sig
++GDB_SIG := sig
++
+ # Latest that could be made to work.
+ GCC_VER := 11.3.0
+ ifeq ($(GCC_VER), 11.3.0)
+   gcc_ver := gcc-11.3.0
++  GCC_GPG_KEY := 7F74F97C103468EE5D750B583AB00996FC26A641 #gitleaks:allow
+   binutils_ver := binutils-2.39
++  BINUTILS_GPG_KEY := 3A24BC1E8FB409FA9F14371813FCEF89DD9E3C4F #gitleaks:allow
+   ccache_ver := ccache-3.7.12
++  CCACHE_GPG_KEY := 5A939A71A46792CF57866A51996DDA075594ADB8 #gitleaks:allow
+   mpfr_ver := mpfr-4.1.1
++  MPFR_SHA256 := 85fdf11614cc08e3545386d6b9c8c9035e3db1e506211a45f4e108117fe3c951 #gitleaks:allow
+   gmp_ver := gmp-6.2.1
++  GMP_GPG_KEY := 343C2FF0FBEE5EC2EDBEF399F3599FF828C67298 #gitleaks:allow
+   mpc_ver := mpc-1.2.1
++  MPC_GPG_KEY := AD17A21EF8AED8F1CC02DBD9F7D5C9BF765C61E3 #gitleaks:allow
+   gdb_ver := gdb-11.2
++  GDB_GPG_KEY := F40ADB902B24264AA42E50BF92EDB04BFF325CF3 #gitleaks:allow
+   REQUIRED_MIN_MAKE_MAJOR_VERSION := 4
+ else ifeq ($(GCC_VER), 11.2.0)
+   gcc_ver := gcc-11.2.0
+@@ -246,7 +286,13 @@ download-rpms:
+         # Only run this if rpm dir is empty.
+         ifeq ($(wildcard $(DOWNLOAD_RPMS)/*.rpm), )
+ 	  cd $(DOWNLOAD_RPMS) && \
+-	      wget -r -np -nd $(patsubst %, -A "*%*.rpm", $(RPM_LIST)) $(BASE_URL)
++	    wget -e robots=off -r -np -nd $(patsubst %, -A "*%*.rpm", $(RPM_LIST)) $(BASE_URL)
++	  gpg --keyserver keyserver.ubuntu.com --recv-keys \
++	    $($(BASE_OS)_$(BASE_OS_MAJOR_VERSION)_GPG_KEY_$(ARCH))
++	  rm -f $(DOWNLOAD_RPMS)/rpm.sig && gpg --armor --export \
++	    $($(BASE_OS)_$(BASE_OS_MAJOR_VERSION)_GPG_KEY_$(ARCH)) > $(DOWNLOAD_RPMS)/rpm.sig
++	  rpm --import $(DOWNLOAD_RPMS)/rpm.sig || true
++	  rpm -K $(DOWNLOAD_RPMS)/*.rpm
+         endif
+
+ ##########################################################################################
+@@ -271,6 +317,14 @@ define Download
+
+   $$($(1)_FILE) :
+ 	wget -P $(DOWNLOAD) $$($(1))
++        ifeq ($(1),MPFR)
++	  echo $($(1)_SHA256) $$@ | sha256sum -c
++        else
++	  wget -P $(DOWNLOAD) $$($(1)).$($(1)_SIG)
++	  gpg --keyserver keyserver.ubuntu.com --recv-keys $($(1)_GPG_KEY)
++	  echo -e "5\ny\n" | gpg --batch --command-fd 0 --expert --edit-key $($(1)_GPG_KEY) trust;
++	  gpg --verify $$@.$($(1)_SIG) $$@
++        endif
+ endef
+
+ # Download and unpack all source packages
+@@ -323,6 +377,9 @@ $(foreach p,$(RPM_FILE_LIST),$(eval $(call unrpm,$(p))))
+ # have it anyway, but just to make sure...
+ # Patch libc.so and libpthread.so to force linking against libraries in sysroot
+ # and not the ones installed on the build machine.
++# Remove comment sections from static libraries and C runtime objects
++# to prevent leaking RHEL-specific package versions into
++# devkit-produced binaries.
+ $(libs) : $(rpms)
+ 	@echo Patching libc and pthreads
+ 	@(for f in `find $(SYSROOT) -name libc.so -o -name libpthread.so`; do \
+@@ -332,6 +389,7 @@ $(libs) : $(rpms)
+ 	      -e 's|/lib/||g' ) > $$f.tmp ; \
+ 	  mv $$f.tmp $$f ; \
+ 	done)
++	@find $(SYSROOT) -name '*.[ao]' -exec objcopy --remove-section .comment '{}' ';'
+ 	@mkdir -p $(SYSROOT)/usr/lib
+ 	@touch $@
+
+@@ -440,6 +498,9 @@ endif
+
+ # Makefile creation. Simply run configure in build dir.
+ # Setting CFLAGS to -O2 generates a much faster ld.
++# Use --enable-deterministic-archives so that make targets that
++# generate "ar" archives, such as "static-libs-image", produce
++# deterministic .a files.
+ $(bfdmakes) \
+ $(BUILDDIR)/$(binutils_ver)/Makefile \
+     : $(BINUTILS_CFG)
+@@ -454,6 +515,7 @@ $(BUILDDIR)/$(binutils_ver)/Makefile \
+ 	      --with-sysroot=$(SYSROOT) \
+ 	      --disable-nls \
+ 	      --program-prefix=$(TARGET)- \
++	      --enable-deterministic-archives \
+ 	      --enable-multilib \
+ 	      --enable-threads \
+ 	      --enable-plugins \

pipelines/build/devkit/binutils-2.39.patch

@@ -0,0 +1,38 @@
+diff --git a/gprofng/Makefile.am b/gprofng/Makefile.am
+index 2fcd695140ca6e3f06f3cc6844d498847a21a63d..96f8015700885e00b9e121cd61adede63c2b832e 100644 (file)
+--- a/gprofng/Makefile.am
++++ b/gprofng/Makefile.am
+@@ -23,11 +23,14 @@ AUTOMAKE_OPTIONS = dejagnu foreign
+ if BUILD_COLLECTOR
+     COLLECTOR_SUBDIRS = libcollector
+ endif
++if BUILD_MAN
++    DOC_SUBDIR = doc
++endif
+ if BUILD_SRC
+-    SRC_SUBDIRS = src gp-display-html doc
++    SRC_SUBDIRS = src gp-display-html $(DOC_SUBDIR)
+ endif
+ SUBDIRS = $(COLLECTOR_SUBDIRS) $(SRC_SUBDIRS)
+-DIST_SUBDIRS = libcollector src gp-display-html doc
++DIST_SUBDIRS = libcollector src gp-display-html $(DOC_SUBDIR)
+
+ # Setup the testing framework, if you have one
+ EXPECT = expect
+diff --git a/gprofng/Makefile.in b/gprofng/Makefile.in
+index fd5279b4df13ca21061bfc833797a0bb6ada62bc..d660f99233fedef9df9cae31629cd3ba57a51150 100644 (file)
+--- a/gprofng/Makefile.in
++++ b/gprofng/Makefile.in
+@@ -381,9 +381,10 @@ zlibinc = @zlibinc@
+ ACLOCAL_AMFLAGS = -I . -I .. 
+ AUTOMAKE_OPTIONS = dejagnu foreign
+ @BUILD_COLLECTOR_TRUE@COLLECTOR_SUBDIRS = libcollector
+-@BUILD_SRC_TRUE@SRC_SUBDIRS = src gp-display-html doc
++@BUILD_MAN_TRUE@DOC_SUBDIR = doc
++@BUILD_SRC_TRUE@SRC_SUBDIRS = src gp-display-html $(DOC_SUBDIR)
+ SUBDIRS = $(COLLECTOR_SUBDIRS) $(SRC_SUBDIRS)
+-DIST_SUBDIRS = libcollector src gp-display-html doc
++DIST_SUBDIRS = libcollector src gp-display-html $(DOC_SUBDIR)
+ RUNTEST = runtest
+ RUNTESTFLAGS = 
+ BASEDIR = $(srcdir)/..