diff --git a/.buildkite/hooks/pre-command.bat b/.buildkite/hooks/pre-command.bat new file mode 100644 index 0000000000000..e77a372727ab3 --- /dev/null +++ b/.buildkite/hooks/pre-command.bat @@ -0,0 +1,22 @@ +@ECHO OFF + +FOR /F "tokens=* eol=#" %%i in ('type .ci\java-versions.properties') do set %%i + +SET JAVA_HOME=%USERPROFILE%\.java\%ES_BUILD_JAVA% +SET JAVA11_HOME=%USERPROFILE%\.java\java11 +SET JAVA16_HOME=%USERPROFILE%\.java\openjdk16 + +SET GRADLEW=./gradlew --parallel --no-daemon --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ +SET GRADLEW_BAT=./gradlew.bat --parallel --no-daemon --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ + +(if not exist "%USERPROFILE%/.gradle" mkdir "%USERPROFILE%/.gradle") && (echo. >> "%USERPROFILE%/.gradle/gradle.properties" && echo org.gradle.daemon=false >> "%USERPROFILE%/.gradle/gradle.properties") + +set WORKSPACE=%cd% +set BUILD_NUMBER=%BUILDKITE_BUILD_NUMBER% +set COMPOSE_HTTP_TIMEOUT=120 +set JOB_BRANCH=%BUILDKITE_BRANCH% + +set GRADLE_BUILD_CACHE_USERNAME=vault read -field=username secret/ci/elastic-elasticsearch/migrated/gradle-build-cache +set GRADLE_BUILD_CACHE_PASSWORD=vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache + +exit /b 0 diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index a83720c79bbe7..2abc791ace259 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -45,3 +45,46 @@ steps: buildDirectory: /dev/shm/bk env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" + - group: packaging-tests-windows + steps: + - label: "{{matrix.image}} / packaging-tests-windows" + command: | + .\.buildkite\scripts\run-script.ps1 .\.ci\scripts\packaging-test.ps1 + timeout_in_minutes: 180 + matrix: + setup: + image: + - windows-2016 + - windows-2019 + - windows-2022 + agents: + provider: gcp + image: family/elasticsearch-{{matrix.image}} + machineType: custom-32-98304 + diskType: pd-ssd + diskSizeGb: 350 + env: {} + - group: platform-support-windows + steps: + - label: "{{matrix.image}} / {{matrix.GRADLE_TASK}} / platform-support-windows" + command: | + .\.buildkite\scripts\run-script.ps1 bash .buildkite/scripts/windows-run-gradle.sh + timeout_in_minutes: 360 + matrix: + setup: + image: + - windows-2016 + - windows-2019 + - windows-2022 + GRADLE_TASK: + - checkPart1 + - checkPart2 + - bwcTestSnapshots + agents: + provider: gcp + image: family/elasticsearch-{{matrix.image}} + machineType: custom-32-98304 + diskType: pd-ssd + diskSizeGb: 350 + env: + GRADLE_TASK: "{{matrix.GRADLE_TASK}}" diff --git a/.buildkite/scripts/run-script.ps1 b/.buildkite/scripts/run-script.ps1 new file mode 100644 index 0000000000000..95174817ea408 --- /dev/null +++ b/.buildkite/scripts/run-script.ps1 @@ -0,0 +1,154 @@ +# Usage: .buildkite/scripts/run-script.ps1 +# Example: .buildkite/scripts/run-script.ps1 bash .buildkite/scripts/tests.sh +# Example: .buildkite/scripts/run-script.ps1 .buildkite/scripts/other-tests.ps1 +# +# NOTE: Apparently passing arguments in powershell is a nightmare, so you shouldn't do it unless it's really simple. Just use the wrapper to call a script instead. +# See: https://stackoverflow.com/questions/6714165/powershell-stripping-double-quotes-from-command-line-arguments +# and: https://github.com/PowerShell/PowerShell/issues/3223#issuecomment-487975049 +# +# See here: https://github.com/buildkite/agent/issues/2202 +# Background processes after the buildkite step script finishes causes the job to hang. +# So, until this is fixed/changed in buildkite-agent (if ever), we can use this wrapper. + +# This wrapper: +# - Creates a Windows job object (which is like a process group) +# - Sets JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, which means that when the job object is closed, all processes in the job object are killed +# - Starts running your given script, and assigns it to the job object +# - Now, any child processes created by your script will also end up in the job object +# - Waits for your script (and only your script, not child processes) to finish +# - Closes the job object, which kills all processes in the job object (including any leftover child processes) +# - Exits with the exit status from your script + +Add-Type -TypeDefinition @' +using Microsoft.Win32.SafeHandles; +using System; +using System.ComponentModel; +using System.Runtime.InteropServices; + +public class NativeMethods +{ + public enum JOBOBJECTINFOCLASS + { + AssociateCompletionPortInformation = 7, + BasicLimitInformation = 2, + BasicUIRestrictions = 4, + EndOfJobTimeInformation = 6, + ExtendedLimitInformation = 9, + SecurityLimitInformation = 5, + GroupInformation = 11 + } + + [StructLayout(LayoutKind.Sequential)] + struct JOBOBJECT_BASIC_LIMIT_INFORMATION + { + public Int64 PerProcessUserTimeLimit; + public Int64 PerJobUserTimeLimit; + public UInt32 LimitFlags; + public UIntPtr MinimumWorkingSetSize; + public UIntPtr MaximumWorkingSetSize; + public UInt32 ActiveProcessLimit; + public Int64 Affinity; + public UInt32 PriorityClass; + public UInt32 SchedulingClass; + } + + [StructLayout(LayoutKind.Sequential)] + struct IO_COUNTERS + { + public UInt64 ReadOperationCount; + public UInt64 WriteOperationCount; + public UInt64 OtherOperationCount; + public UInt64 ReadTransferCount; + public UInt64 WriteTransferCount; + public UInt64 OtherTransferCount; + } + + [StructLayout(LayoutKind.Sequential)] + struct JOBOBJECT_EXTENDED_LIMIT_INFORMATION + { + public JOBOBJECT_BASIC_LIMIT_INFORMATION BasicLimitInformation; + public IO_COUNTERS IoInfo; + public UIntPtr ProcessMemoryLimit; + public UIntPtr JobMemoryLimit; + public UIntPtr PeakProcessMemoryUsed; + public UIntPtr PeakJobMemoryUsed; + } + + [DllImport("Kernel32.dll", EntryPoint = "AssignProcessToJobObject", SetLastError = true)] + private static extern bool NativeAssignProcessToJobObject(SafeHandle hJob, SafeHandle hProcess); + + public static void AssignProcessToJobObject(SafeHandle job, SafeHandle process) + { + if (!NativeAssignProcessToJobObject(job, process)) + throw new Win32Exception(); + } + + [DllImport( + "Kernel32.dll", + CharSet = CharSet.Unicode, + EntryPoint = "CreateJobObjectW", + SetLastError = true + )] + private static extern SafeFileHandle NativeCreateJobObjectW( + IntPtr lpJobAttributes, + string lpName + ); + + [DllImport("Kernel32.dll", EntryPoint = "CloseHandle", SetLastError = true)] + private static extern bool NativeCloseHandle(SafeHandle hJob); + + [DllImport("kernel32.dll")] + public static extern bool SetInformationJobObject( + SafeHandle hJob, + JOBOBJECTINFOCLASS JobObjectInfoClass, + IntPtr lpJobObjectInfo, + uint cbJobObjectInfoLength + ); + + private const UInt32 JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x2000; + + public static SafeHandle CreateJobObjectW(string name) + { + SafeHandle job = NativeCreateJobObjectW(IntPtr.Zero, name); + JOBOBJECT_BASIC_LIMIT_INFORMATION info = new JOBOBJECT_BASIC_LIMIT_INFORMATION(); + info.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + JOBOBJECT_EXTENDED_LIMIT_INFORMATION extendedInfo = + new JOBOBJECT_EXTENDED_LIMIT_INFORMATION(); + extendedInfo.BasicLimitInformation = info; + int length = Marshal.SizeOf(typeof(JOBOBJECT_EXTENDED_LIMIT_INFORMATION)); + IntPtr extendedInfoPtr = Marshal.AllocHGlobal(length); + Marshal.StructureToPtr(extendedInfo, extendedInfoPtr, false); + SetInformationJobObject( + job, + JOBOBJECTINFOCLASS.ExtendedLimitInformation, + extendedInfoPtr, + (uint)length + ); + if (job.IsInvalid) + throw new Win32Exception(); + return job; + } + + public static void CloseHandle(SafeHandle job) + { + if (!NativeCloseHandle(job)) + throw new Win32Exception(); + } +} + +'@ + +$guid = [guid]::NewGuid().Guid +Write-Output "Creating job object with name $guid" +$job = [NativeMethods]::CreateJobObjectW($guid) +$process = Start-Process -PassThru -NoNewWindow powershell -ArgumentList "$args" +[NativeMethods]::AssignProcessToJobObject($job, $process.SafeHandle) + +try { + Write-Output "Waiting for process $($process.Id) to complete..." + $process | Wait-Process + Write-Output "Process finished with exit code $($process.ExitCode), terminating job and exiting..." +} finally { + [NativeMethods]::CloseHandle($job) + exit $process.ExitCode +} diff --git a/.buildkite/scripts/windows-run-gradle.sh b/.buildkite/scripts/windows-run-gradle.sh new file mode 100755 index 0000000000000..75999e105c873 --- /dev/null +++ b/.buildkite/scripts/windows-run-gradle.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -euo pipefail + +.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true $GRADLE_TASK diff --git a/.ci/scripts/packaging-test.ps1 b/.ci/scripts/packaging-test.ps1 index d75fa8267ae62..b601d7970d18c 100644 --- a/.ci/scripts/packaging-test.ps1 +++ b/.ci/scripts/packaging-test.ps1 @@ -12,7 +12,7 @@ $env:ES_BUILD_JAVA=$AppProps.ES_BUILD_JAVA $env:JAVA_TOOL_OPTIONS='' $ErrorActionPreference="Stop" -$gradleInit = "C:\Users\$env:username\.gradle\init.d\" +$gradleInit = "$env:USERPROFILE\.gradle\init.d\" echo "Remove $gradleInit" Remove-Item -Recurse -Force $gradleInit -ErrorAction Ignore New-Item -ItemType directory -Path $gradleInit @@ -20,13 +20,14 @@ echo "Copy .ci/init.gradle to $gradleInit" Copy-Item .ci/init.gradle -Destination $gradleInit [Environment]::SetEnvironmentVariable("JAVA_HOME", $null, "Machine") -$env:PATH="C:\Users\jenkins\.java\$env:ES_BUILD_JAVA\bin\;$env:PATH" +$env:PATH="$env:USERPROFILE\.java\$env:ES_BUILD_JAVA\bin\;$env:PATH" $env:JAVA_HOME=$null -$env:SYSTEM_JAVA_HOME="C:\Users\jenkins\.java\java8" +$env:SYSTEM_JAVA_HOME="$env:USERPROFILE\.java\java8" Remove-Item -Recurse -Force \tmp -ErrorAction Ignore New-Item -ItemType directory -Path \tmp $ErrorActionPreference="Continue" -& .\gradlew.bat -g "C:\Users\$env:username\.gradle" --parallel --no-daemon --scan --console=plain $GradleTasks +Add-Content -Path $profile.AllUsersAllHosts -Value '$ErrorActionPreference="Continue"' +& .\gradlew.bat -g "$env:USERPROFILE\.gradle" --parallel --no-daemon --scan --console=plain $GradleTasks exit $LastExitCode diff --git a/.ci/scripts/run-gradle.sh b/.ci/scripts/run-gradle.sh index 2c59817929443..ea68ab7718579 100755 --- a/.ci/scripts/run-gradle.sh +++ b/.ci/scripts/run-gradle.sh @@ -1,28 +1,35 @@ #!/bin/bash -# drop page cache and kernel slab objects on linux -[[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches rm -Rfv ~/.gradle/init.d mkdir -p ~/.gradle/init.d && cp -v $WORKSPACE/.ci/init.gradle ~/.gradle/init.d -if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then - MAX_WORKERS=16 -elif [ -f /proc/cpuinfo ]; then - MAX_WORKERS=`grep '^cpu\scores' /proc/cpuinfo | uniq | sed 's/\s\+//g' | cut -d':' -f 2` -else - if [[ "$OSTYPE" == "darwin"* ]]; then - MAX_WORKERS=`sysctl -n hw.physicalcpu | sed 's/\s\+//g'` - else - echo "Unsupported OS Type: $OSTYPE" - exit 1 - fi -fi -if pwd | grep -v -q ^/dev/shm ; then - echo "Not running on a ramdisk, reducing number of workers" - MAX_WORKERS=$(($MAX_WORKERS*2/3)) -fi -# Export glibc version as environment variable since some BWC tests are incompatible with later versions -export GLIBC_VERSION=$(ldd --version | grep '^ldd' | sed 's/.* \([1-9]\.[0-9]*\).*/\1/') +MAX_WORKERS=4 + +# Don't run this stuff on Windows +if ! uname -a | grep -q MING; then + # drop page cache and kernel slab objects on linux + [[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches + + if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then + MAX_WORKERS=16 + elif [ -f /proc/cpuinfo ]; then + MAX_WORKERS=`grep '^cpu\scores' /proc/cpuinfo | uniq | sed 's/\s\+//g' | cut -d':' -f 2` + else + if [[ "$OSTYPE" == "darwin"* ]]; then + MAX_WORKERS=`sysctl -n hw.physicalcpu | sed 's/\s\+//g'` + else + echo "Unsupported OS Type: $OSTYPE" + exit 1 + fi + fi + if pwd | grep -v -q ^/dev/shm ; then + echo "Not running on a ramdisk, reducing number of workers" + MAX_WORKERS=$(($MAX_WORKERS*2/3)) + fi + + # Export glibc version as environment variable since some BWC tests are incompatible with later versions + export GLIBC_VERSION=$(ldd --version | grep '^ldd' | sed 's/.* \([1-9]\.[0-9]*\).*/\1/') +fi set -e $GRADLEW -S --max-workers=$MAX_WORKERS $@ diff --git a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle index 1781cc2f244a4..c3c6e093b8989 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle @@ -17,6 +17,7 @@ long startTime = project.gradle.services.get(BuildRequestMetaData).getStartTime( buildScan { URL jenkinsUrl = System.getenv('JENKINS_URL') ? new URL(System.getenv('JENKINS_URL')) : null + String buildKiteUrl = System.getenv('BUILDKITE_BUILD_URL') ? System.getenv('BUILDKITE_BUILD_URL') : null // Automatically publish scans from Elasticsearch CI if (jenkinsUrl?.host?.endsWith('elastic.co') || jenkinsUrl?.host?.endsWith('elastic.dev') || System.getenv('BUILDKITE') == 'true') { @@ -98,6 +99,47 @@ buildScan { value 'Git Commit ID', BuildParams.gitRevision link 'Source', "https://github.com/elastic/elasticsearch/tree/${BuildParams.gitRevision}" } + } else if (buildKiteUrl) { //Buildkite-specific build scan metadata + // Disable async upload in CI to ensure scan upload completes before CI agent is terminated + uploadInBackground = false + + def branch = System.getenv('BUILDKITE_BRANCH') + def repoMatcher = System.getenv('BUILDKITE_REPO') =~ /(https:\/\/github\.com\/|git@github\.com:)(\S+)\.git/ + def repository = repoMatcher.matches() ? repoMatcher.group(2) : "" + tag 'CI' + link 'CI Build', buildKiteUrl + value 'Job Number', System.getenv('BUILDKITE_BUILD_NUMBER') + + + // Add SCM information + def prId = System.getenv('BUILDKITE_PULL_REQUEST') + if (prId != 'false') { + def prBaseUrl = (System.getenv('BUILDKITE_PULL_REQUEST_REPO') - ".git") + value 'Git Commit ID', System.getenv('BUILDKITE_COMMIT') + tag "pr/${prId}" + tag 'pull-request' + link 'Source', "${prBaseUrl}/tree/${System.getenv('BUILDKITE_COMMIT')}" + link 'Pull Request', "https://github.com/${repository}/pull/${prId}" + } else { + value 'Git Commit ID', BuildParams.gitRevision + link 'Source', "https://github.com/${repository}/tree/${BuildParams.gitRevision}" + tag branch + } + + buildScanPublished { scan -> + // Attach build scan link as build metadata + // See: https://buildkite.com/docs/pipelines/build-meta-data + new ProcessBuilder('buildkite-agent', 'meta-data', 'set', "build-scan-${System.getenv('BUILDKITE_JOB_ID')}", "${scan.buildScanUri}") + .start() + .waitFor() + + // Add a build annotation + // See: https://buildkite.com/docs/agent/v3/cli-annotate + def body = """
${System.getenv('BUILDKITE_LABEL')} :gradle: build ran: gradle ${gradle.startParameter.taskNames.join(' ')}
""" + new ProcessBuilder('buildkite-agent', 'annotate', '--context', 'gradle-build-scans', '--append', '--style', 'info', body) + .start() + .waitFor() + } } else { tag 'LOCAL' } diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/ReaperService.java b/build-tools/src/main/java/org/elasticsearch/gradle/ReaperService.java index 4a0a76798f6fe..d02ad7e7e2f32 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/ReaperService.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/ReaperService.java @@ -41,7 +41,7 @@ public abstract class ReaperService implements BuildServiceconditional() - .onWindows(() -> new String[] { "Taskkill", "/F", "/PID", String.valueOf(pid) }) + .onWindows(() -> new String[] { "Taskkill", "/F", "/T", "/PID", String.valueOf(pid) }) .onUnix(() -> new String[] { "kill", "-9", String.valueOf(pid) }) .supply(); registerCommand(serviceId, killPidCommand);