remote-build.exp

#!/usr/bin/expect --

# Copyright 2017 IBM Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
# Remotely build AT
#
# It executes all the necessary steps to remotely build AT in a list of
# servers, including:
# - call make pack
# - send the tarball to the server
# - remove files from the last build
# - unpack the tarball created by make pack
# - start the build
#

# Set $var to $def_val if not set.
proc set_def {var def_val} {
	upvar ${var} x

	if {![info exists x]} {
		set x ${def_val}
	}
}

# Load config file.
set conf ~/.remote-build.conf
if {[file exists ${conf}]} {
	source ${conf}
}

set prompt "(%|>|\#|\\\$)\[\[:space:\]\]*$"

# Request passwords once per server
set ssh_params {-o "ControlMaster=auto" -o "ControlPath=/tmp/ssh-%r@%h:%p"}

# Flush expect buffer.
proc flush_expect { } {
	expect * { }
}

proc print_usage { } {
	global argv0
	puts "Usage: $argv0 version servers"
	puts ""
	puts "Options:"
	puts "	--buildarch=	Specify the value of BUILD_ARCH."
	puts "	--help		Print this message."
	puts "	--trayprefix=	Writes to the variable \$trayprefix."
	puts "			Default value is '~/tmp/at-build-tray'."
	puts "	--destprefix=	Writes to the variable \$destprefix."
	puts "			Default value is '~/opt'."
	puts "	--atdir=	Writes to the variable \$atdir."
	puts "			Default value is '.'."
	puts ""
	puts "	AT will always be copied to \$trayprefix/\$atdir and installed"
	puts "	to \$destprefix/\$atdir."
	puts ""
	puts "Example:"
	puts "	$argv0 4.0 server1 user@server2"
	puts "	$argv0 --buildarch=ppc64le 7.1 server1.domain"
	puts ""
}

# Detach from current screen session.
proc dettach_screen { } {
	global screen
	if { ${screen} } {
		# Detach from screen session.
		send "
"
	} else {
		# Detach from tmux session.
		send "d"
	}
}

# Default values
set buildarch ""

set_def default_timeout 30; # 30 seconds

# trayprefix, destprefix and atdir can be specified in the config file or
# through parameters. This script will ALWAYS copy the AT files to
# $trayprefix/$atdir and install to $destprefix/$atdir. The order of
# preference for these variables is:
# Parameters > Configuration file > Default values
set_def trayprefix tmp/at-build-tray
set_def destprefix opt
set_def atdir ""

# Parse optional arguments
set argi 0
while { ${argi} < ${argc} } {
	set arg [lindex ${argv} ${argi}]

	switch -regexp ${arg} {
		"--help" {
			print_usage
			exit
		}
		"--buildarch=[^ ]*" {
			if { [catch {regexp -- "--buildarch=(\[^ \]*)" ${arg} \
					 a buildarch}] } {
				error "Could not parse ${arg}"
			}
		}
		"--trayprefix=[^ ]*" {
			if { [catch {regexp -- "--trayprefix=(\[^ \]*)" ${arg} \
					 a tmpvar}] } {
				error "Could not parse ${arg}"
			}
		 	set trayprefix $tmpvar
		}
		"--destprefix=[^ ]*" {
			if { [catch {regexp -- "--destprefix=(\[^ \]*)" ${arg} \
					 a tmpvar}] } {
				error "Could not parse ${arg}"
			}
			set destprefix $tmpvar
		}
		"--atdir=[^ ]*" {
			if { [catch {regexp -- "--atdir=(\[^ \]*)" ${arg} \
					 a tmpvar}] } {
				error "Could not parse ${arg}"
			}
			set atdir $tmpvar
		}
		default break
	}

	incr argi
}

# Parse mandatory arguments
if { ${argc} - ${argi} < 2 } {
	print_usage
	exit
}
set at_version [lindex $argv ${argi}]
incr argi
set servers [lrange $argv ${argi} end]

if { ! [file exists configs/${at_version}] } {
	error "Could not find config set ${at_version}"
}

puts "AT version: $at_version"
puts "Servers: $servers"

set timeout $default_timeout

spawn make pack
expect {
	"at-*.tgz" {
		set at_file $expect_out(0,string)
	}
	default {
		send_user "Failed to create the tarball."
		exit 1
	}
}

flush_expect
send_user "\n"

foreach server $servers {
	eval spawn ssh ${ssh_params} ${server}
	# Save this spawn_id for future usage
	set ssh_spid $spawn_id

	# Deal with credentials
	expect {
		-re $prompt { }
		-re "((assword:\ )|(assphrase))" {
			stty -echo
			expect_user -re "(.*)\n"
			send_user "\n"
			send "$expect_out(1,string)\r"
			stty echo
			exp_continue
			expect {
				-re $prompt { }
			}
		}
		"(yes/no)? " {
			send "yes\n"
			expect {
				-re $prompt { }
			}
		}
		default {
			send_user "Login failed\n"
			exit
		}
	}

	flush_expect

	set host_trayprefix $trayprefix
	set host_destprefix $destprefix

	if { [string range "$host_trayprefix" 0 0] != "/" ||
	     [string range "$host_destprefix" 0 0] != "/" } then {

		# Retrieve HOME from the remote host
		# and prepend it where needed

		send "eval echo ~\r"
		expect -re "(/.*)\r"
		set home $expect_out(1,string)
		if { [string range "$host_trayprefix" 0 0] != "/" } then {
			set host_trayprefix $home/$host_trayprefix
		}
		if { [string range "$host_destprefix" 0 0] != "/" } then {
			set host_destprefix $home/$host_destprefix
		}
	}
	set host_trayprefix [string trimright "$host_trayprefix" /]
	set host_destprefix [string trimright "$host_destprefix" /]

	# Please note that using "set tray_dir" and "set destdir" implies that
	# if you try to set those on your config file, they will be overridden
	# here, so please use trayprefix, destprefix and atdir to specify the
	# desired paths.

	# Directory used by the script to save its files.
	set tray_dir [string trimright "$host_trayprefix/$atdir" /]
	# Directory where AT will be installed (aka DESTDIR).
	set destdir [string trimright "$host_destprefix/$atdir" /]

	send "mkdir -p ${tray_dir}\n"
	expect {
		-re $prompt { }
		default {
			send_user "Failed to create directory ${tray_dir}\n"
			exit 1
		}
	}

	send "cd ${tray_dir}\n"
	expect {
		-re $prompt { }
		default {
			send_user "Failed to change to directory ${tray_dir}\n"
			exit 1
		}
	}

	# Remove files from old builds
	send "test -x remake.sh && ./remake.sh clean\n"
	expect {
		-re $prompt { }
		# Some servers are very slow and require a lot
		# of time to clean a previous build.
		-timeout 600 default {
			send_user "Failed to clean a previous build.\n"
			exit 1
		}
	}

	flush_expect

	send "rm -rf at-* configs Makefile scripts configs README* screenlog*\n"
	expect {
		-re $prompt { }
		-timeout 300 default {
			send_user "Failed to remove files from ${tray_dir}\n"
			exit 1
		}
	}

	send_user "\n"
	flush_expect

	# Indicates when a copy achieved 100%.
	set scp_complete 0
	eval spawn scp ${ssh_params} ${at_file} ${server}:${tray_dir}/
	expect {
		-re "((assword:\ )|(assphrase))" {
			stty -echo
			expect_user -re "(.*)\n"
			send_user "\n"
			send "$expect_out(1,string)\r"
			exp_continue
		}
		"denied" {
			send_user "Login failed\n"
			exit 1
		}
		-timeout 120 timeout {
			send_user "Timeout!\n"
			exit 1
		}
		"100%" {
			# An scp isn't complete until it prints 100% in the
			# output and the command returns (eof).
			set scp_complete 1
			exp_continue
		}
		eof {
			# eof before achieving 100% is an error.
			if { ! ${scp_complete} } {
				send_user "Failed to copy ${at_file} to \
${server}.\n"
				exit 1
			}
		}
	}

	# Recover ssh spawn id
	set spawn_id $ssh_spid
	flush_expect

	# Extract AT tarball.
	send "tar -xzf ${at_file}\n"
	expect {
		-re $prompt { }
		"Error"{
			send_user "Failed to untar ${at_file}\n"
			exit 1
		}
		default {
			send_user "Failed to untar ${at_file}\n"
			exit 1
		}
	}

	flush_expect

	# RHEL8 doesn't provide screen.
	# Check screen is installed. If not, assume tmux is installed.
	set screen 1
	send "which screen\n"
	expect {
		"/screen" { }
		"no screen" {
			set screen 0 }
		default {
			send_user "Failed to check if screen is installed.\n"
			exit 1
		}
	}

	flush_expect

	# Get cpu information
	send "grep -c processor /proc/cpuinfo\n"
	expect {
		-re "\[0-9\]+\r" {
			set cores $expect_out(0,string)
		}
		default {
			send_user "Failed to read processor's information.\n"
			exit 1
		}
	}

	flush_expect

	send "uname -m\n"
	expect {
		-re "ppc.*" {
			# Avoid starting too many jobs on our POWER servers in
			# order to not overload a multi-user server.
			set cores [expr ${cores} / 2 + 1]
		}
		# Use full power on x86
		"i686" {
			incr cores
		}
		"x86_64" {
			incr cores
		}
		default {
			send_user "Failed to identify the type of the  processor.\n"
			exit 1
		}
	}

	flush_expect

	send_user "\nUsing -j${cores}\n"

	set parent_cmd "time -p nice -n 19"
	set make_parms "-j${cores} AT_CONFIGSET=${at_version} \
DESTDIR=${destdir}"
	if { ${buildarch} != "" } {
		append make_parms " BUILD_ARCH=${buildarch}"
	}
	# Force the targets for building and FVTR runs.
	append make_parms " release test"
	if { ${screen} } {
		set screen_parms "-L -S at-build"
		send_user "screen ${screen_parms} ${parent_cmd} make ${make_parms}\n"
		send "screen ${screen_parms} ${parent_cmd} make ${make_parms}\n"
	} else {
		set tmux_parms "new -s at-build"
		send_user "tmux ${tmux_parms} ${parent_cmd} make ${make_parms}\n"
		send "tmux ${tmux_parms} ${parent_cmd} make ${make_parms} \\; pipe-pane 'cat >'${tray_dir}'/screen.log'\n"
	}
	expect {
		"command not found" {
			send_user "Something is missing on ${server}.\n"
			exit 1
		}
		"screen is terminating" {
			send_user "Caught an early build failure on $server.\n"
			exit 1
		}
		-re "exited|failed|Error" {
			dettach_screen
			send_user "\n"
			send_user "Failed to build on $server.\n"
			send_user "Some processes may still continue to run.\n"
			exit 1
		}
		default { }
	}

	dettach_screen
	send_user "\n"
	send_user "Build is running on ${server}.\n"
}

send_user "Started builds on: ${servers}\n"
exit