From d60f67bbee8ff2ee078cd3c1f965c46023ae8187 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 21 Aug 2023 13:50:31 -0400 Subject: [PATCH] Add initial windows docker builder and host AMI (#334) * Add initial windows docker builder and host AMI Signed-off-by: Peter Zhu * More tweaks Signed-off-by: Peter Zhu * Reset the disk size to original to allow agent setups to increase Signed-off-by: Peter Zhu * More tweaks Signed-off-by: Peter Zhu --------- Signed-off-by: Peter Zhu --- ...enkins-agent-win2019-x64-gradle-check.json | 4 +- packer/jenkins-agent-win2019-x64.json | 3 +- .../jenkins-agent-win2016-x64.json | 6 +- .../jenkins-agent-win2019-x64-alpine-wsl.json | 10 +- ...-agent-win2019-x64-pre-docker-support.json | 96 +++++++++++++++++++ .../windows/{ => legacy}/pip-install.ps1 | 0 .../{ => legacy}/scoop-install-alpinewsl.ps1 | 0 .../{ => legacy}/scoop-install-commons.ps1 | 0 .../windows/{ => legacy}/smb-setup.ps1 | 0 .../windows/{ => legacy}/wsl-setup.ps1 | 0 .../scoop-install-commons-docker-support.ps1 | 90 +++++++++++++++++ 11 files changed, 197 insertions(+), 12 deletions(-) rename packer/{ => legacy}/jenkins-agent-win2016-x64.json (93%) rename packer/{ => legacy}/jenkins-agent-win2019-x64-alpine-wsl.json (90%) create mode 100644 packer/legacy/jenkins-agent-win2019-x64-pre-docker-support.json rename packer/scripts/windows/{ => legacy}/pip-install.ps1 (100%) rename packer/scripts/windows/{ => legacy}/scoop-install-alpinewsl.ps1 (100%) rename packer/scripts/windows/{ => legacy}/scoop-install-commons.ps1 (100%) rename packer/scripts/windows/{ => legacy}/smb-setup.ps1 (100%) rename packer/scripts/windows/{ => legacy}/wsl-setup.ps1 (100%) create mode 100644 packer/scripts/windows/scoop-install-commons-docker-support.ps1 diff --git a/packer/jenkins-agent-win2019-x64-gradle-check.json b/packer/jenkins-agent-win2019-x64-gradle-check.json index e6ea3ca..5732559 100644 --- a/packer/jenkins-agent-win2019-x64-gradle-check.json +++ b/packer/jenkins-agent-win2019-x64-gradle-check.json @@ -77,8 +77,8 @@ "scripts/windows/smb-setup-2019-plus.ps1", "scripts/windows/longpath-setup.ps1", "scripts/windows/scoop-setup.ps1", - "scripts/windows/scoop-install-commons.ps1", - "scripts/windows/pip-install.ps1" + "scripts/windows/legacy/scoop-install-commons.ps1", + "scripts/windows/legacy/pip-install.ps1" ], "max_retries": 3 }, diff --git a/packer/jenkins-agent-win2019-x64.json b/packer/jenkins-agent-win2019-x64.json index 6306931..7aaf8bf 100644 --- a/packer/jenkins-agent-win2019-x64.json +++ b/packer/jenkins-agent-win2019-x64.json @@ -77,8 +77,7 @@ "scripts/windows/smb-setup-2019-plus.ps1", "scripts/windows/longpath-setup.ps1", "scripts/windows/scoop-setup.ps1", - "scripts/windows/scoop-install-commons.ps1", - "scripts/windows/pip-install.ps1" + "scripts/windows/scoop-install-commons-docker-support.ps1" ], "max_retries": 3 }, diff --git a/packer/jenkins-agent-win2016-x64.json b/packer/legacy/jenkins-agent-win2016-x64.json similarity index 93% rename from packer/jenkins-agent-win2016-x64.json rename to packer/legacy/jenkins-agent-win2016-x64.json index fad818b..23e6ef2 100644 --- a/packer/jenkins-agent-win2016-x64.json +++ b/packer/legacy/jenkins-agent-win2016-x64.json @@ -63,10 +63,10 @@ { "type":"powershell", "scripts": [ - "scripts/windows/smb-setup.ps1", + "scripts/windows/legacy/smb-setup.ps1", "scripts/windows/scoop-setup.ps1", - "scripts/windows/scoop-install-commons.ps1", - "scripts/windows/pip-install.ps1" + "scripts/windows/legacy/scoop-install-commons.ps1", + "scripts/windows/legacy/pip-install.ps1" ], "max_retries": 3 }, diff --git a/packer/jenkins-agent-win2019-x64-alpine-wsl.json b/packer/legacy/jenkins-agent-win2019-x64-alpine-wsl.json similarity index 90% rename from packer/jenkins-agent-win2019-x64-alpine-wsl.json rename to packer/legacy/jenkins-agent-win2019-x64-alpine-wsl.json index c764735..73c0db1 100644 --- a/packer/jenkins-agent-win2019-x64-alpine-wsl.json +++ b/packer/legacy/jenkins-agent-win2019-x64-alpine-wsl.json @@ -64,10 +64,10 @@ "type":"powershell", "scripts": [ "scripts/windows/smb-setup-2019-plus.ps1", - "scripts/windows/scoop-setup.ps1", - "scripts/windows/scoop-install-commons.ps1", - "scripts/windows/pip-install.ps1", - "scripts/windows/wsl-setup.ps1" + "scripts/windows/legacy/scoop-setup.ps1", + "scripts/windows/legacy/scoop-install-commons.ps1", + "scripts/windows/legacy/pip-install.ps1", + "scripts/windows/legacy/wsl-setup.ps1" ], "max_retries": 3 }, @@ -77,7 +77,7 @@ { "type":"powershell", "scripts": [ - "scripts/windows/scoop-install-alpinewsl.ps1" + "scripts/windows/legacy/scoop-install-alpinewsl.ps1" ] }, { diff --git a/packer/legacy/jenkins-agent-win2019-x64-pre-docker-support.json b/packer/legacy/jenkins-agent-win2019-x64-pre-docker-support.json new file mode 100644 index 0000000..cf02ef6 --- /dev/null +++ b/packer/legacy/jenkins-agent-win2019-x64-pre-docker-support.json @@ -0,0 +1,96 @@ +{ + "variables": { + "name-base":"Jenkins-Agent-Windows2019-X64-Pre-Docker-Support", + "os-version": "Windows2019", + "build-region":"us-east-1", + "build-vpc":"vpc-<>", + "build-subnet":"subnet-<>", + "build-secgrp":"sg-<>", + "build-time":"{{isotime \"2006-01-02T03-04-05Z\"}}", + "aws_ami_region":"us-east-1" + }, + "builders": [ + { + "name":"{{user `name-base`}}-{{user `build-time`}}" , + "ami_description":"{{user `name-base`}}", + "type":"amazon-ebs", + "encrypt_boot":"false", + "region":"{{user `build-region`}}", + "ami_regions":"{{user `aws_ami_region`}}", + "instance_type":"c5.4xlarge", + "ami_name":"{{user `name-base`}}-{{user `build-time`}}", + "vpc_id":"{{user `build-vpc`}}", + "subnet_id":"{{user `build-subnet`}}", + "security_group_ids":"{{user `build-secgrp`}}", + "source_ami_filter": { + "filters": { + "virtualization-type":"hvm", + "architecture":"x86_64", + "name": "Windows_Server-2019-English-Full-Base-20*", + "block-device-mapping.volume-type":"gp2", + "root-device-type":"ebs" + }, + "owners": [ + "amazon" + ], + "most_recent":true + }, + "user_data_file":"scripts/windows/userdata.ps1", + "associate_public_ip_address":false, + "communicator":"winrm", + "winrm_username":"Administrator", + "winrm_timeout":"40m", + "windows_password_timeout":"30m", + "winrm_insecure":true, + "tags": { + "Name": "{{user `name-base`}}-{{user `build-time`}}", + "OS_Version":"{{user `os-version`}}", + "User":"Packer", + "Encrypted_AMI":"False", + "Created":"{{user `build-time`}}" + }, + "launch_block_device_mappings": [ + { + "device_name":"/dev/sda1", + "volume_size":300, + "delete_on_termination":true, + "volume_type":"gp3" + } + ] + } + ], + "provisioners": [ + { + "type": "file", + "source": "scripts/windows/winrm_max_memory.ps1", + "destination": "C:\\Users\\Administrator\\jenkins\\winrm_max_memory.ps1" + }, + { + "type":"powershell", + "inline": [ + "C:\\Users\\Administrator\\jenkins\\winrm_max_memory.ps1 30" + ] + }, + { + "type":"powershell", + "scripts": [ + "scripts/windows/smb-setup-2019-plus.ps1", + "scripts/windows/longpath-setup.ps1", + "scripts/windows/scoop-setup.ps1", + "scripts/windows/legacy/scoop-install-commons.ps1", + "scripts/windows/legacy/pip-install.ps1" + ], + "max_retries": 3 + }, + { + "type": "windows-restart" + }, + { + "type":"powershell", + "inline": [ + "C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Scripts\\InitializeInstance.ps1 -Schedule", + "C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Scripts\\SysprepInstance.ps1 -NoShutdown" + ] + } + ] +} diff --git a/packer/scripts/windows/pip-install.ps1 b/packer/scripts/windows/legacy/pip-install.ps1 similarity index 100% rename from packer/scripts/windows/pip-install.ps1 rename to packer/scripts/windows/legacy/pip-install.ps1 diff --git a/packer/scripts/windows/scoop-install-alpinewsl.ps1 b/packer/scripts/windows/legacy/scoop-install-alpinewsl.ps1 similarity index 100% rename from packer/scripts/windows/scoop-install-alpinewsl.ps1 rename to packer/scripts/windows/legacy/scoop-install-alpinewsl.ps1 diff --git a/packer/scripts/windows/scoop-install-commons.ps1 b/packer/scripts/windows/legacy/scoop-install-commons.ps1 similarity index 100% rename from packer/scripts/windows/scoop-install-commons.ps1 rename to packer/scripts/windows/legacy/scoop-install-commons.ps1 diff --git a/packer/scripts/windows/smb-setup.ps1 b/packer/scripts/windows/legacy/smb-setup.ps1 similarity index 100% rename from packer/scripts/windows/smb-setup.ps1 rename to packer/scripts/windows/legacy/smb-setup.ps1 diff --git a/packer/scripts/windows/wsl-setup.ps1 b/packer/scripts/windows/legacy/wsl-setup.ps1 similarity index 100% rename from packer/scripts/windows/wsl-setup.ps1 rename to packer/scripts/windows/legacy/wsl-setup.ps1 diff --git a/packer/scripts/windows/scoop-install-commons-docker-support.ps1 b/packer/scripts/windows/scoop-install-commons-docker-support.ps1 new file mode 100644 index 0000000..93fcbcd --- /dev/null +++ b/packer/scripts/windows/scoop-install-commons-docker-support.ps1 @@ -0,0 +1,90 @@ +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# Disable "current" alias directory as it is not preserved after AMI creation +# Use static path in environment variable +scoop config no_junction true + +# Install git +scoop install git +git --version +# Path for git windows usr bin +$fileName = 'nohup.exe' +$fileDir = 'C:\\Users\\Administrator\\scoop\\apps\\git' +$fileFound = (Get-ChildItem -Path $fileDir -Filter $fileName -Recurse | %{$_.FullName} | select -first 1) +$fileFound +$gitPathFound = $fileFound.replace("$fileName", '') +$gitPathFound +# Add to EnvVar +$userenv = [System.Environment]::GetEnvironmentVariable("Path", [System.EnvironmentVariableTarget]::User) +[System.Environment]::SetEnvironmentVariable("PATH", $userenv + ";$gitPathFound", [System.EnvironmentVariableTarget]::User) +# Make sure mem size are set to avoid "Out of memory, malloc failed" issues on Windows +git config --system core.packedGitLimit 128m +git config --system core.packedGitWindowSize 128m +git config --system core.longPaths true +git config --system pack.deltaCacheSize 128m +git config --system pack.packSizeLimit 128m +git config --system pack.windowMemory 128m +git config --system pack.window 0 +git config --system pack.threads 1 +git config --system core.compression 0 +git config --system protocol.version 1 +git config --system --list +# Rename system32 find.exe in case it gets conflicted with POSIX find +mv -v 'C:\\Windows\\System32\\find.exe' 'C:\\Windows\\System32\\find_windows.exe' + +# Add some sleep due to a potential race condition +Start-Sleep -Seconds 10 + +# Setup Repos (This has to happen after git is installed or will error out) +scoop bucket add java +scoop bucket add versions +scoop bucket add extras +scoop bucket add github-gh https://github.com/cli/scoop-gh.git + +# Install jdk +$jdkVersionList = "temurin11-jdk JAVA11_HOME" +Foreach ($jdkVersion in $jdkVersionList) +{ + $jdkVersion + $jdkArray = $jdkVersion.Split(" ") + $jdkArray[0] + $jdkArray[1] + scoop install $jdkArray[0] + $JAVA_HOME_TEMP = [System.Environment]::GetEnvironmentVariable("JAVA_HOME", [System.EnvironmentVariableTarget]::User).replace("\", "/") + $JAVA_HOME_TEMP + [System.Environment]::SetEnvironmentVariable($jdkArray[1], "$JAVA_HOME_TEMP", [System.EnvironmentVariableTarget]::User) + java -version +} + +# Install gh +scoop install gh +gh version + +# Install awscli +scoop install aws +aws --version + +# Install zip +scoop install zip + +# Setup Docker +echo "Enable Hyper-V" +Enable-WindowsOptionalFeature -Online -FeatureName "Microsoft-Hyper-V" -All -NoRestart +echo "Enable Containers" +Enable-WindowsOptionalFeature -Online -FeatureName "Containers" -All -NoRestart +echo "Add Hyper-V-Tools" +Add-WindowsFeature "Hyper-V-Tools" +echo "Add Hyper-V-PowerShell" +Add-WindowsFeature "Hyper-V-PowerShell" +echo "Check HyperV Features" +Get-WindowsFeature "Hyper*" +echo "Setup autostart of hyperviser and the docker services by default" +bcdedit /set hypervisorlaunchtype auto +echo "Install Docker Engine" +scoop install docker +