From 8eacbdc79f60f8998fe45d43397bea6785f5604f Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Wed, 26 Jan 2022 13:23:24 -0500 Subject: [PATCH] Runner config option to disable auto-update. (#1558) * Runner config option to disable auto-update. * Update src/Runner.Listener/Configuration/ConfigurationManager.cs Co-authored-by: Thomas Boop <52323235+thboop@users.noreply.github.com> * Update src/Runner.Listener/Configuration/ConfigurationManager.cs Co-authored-by: Thomas Boop <52323235+thboop@users.noreply.github.com> * Update src/Runner.Listener/Configuration/ConfigurationManager.cs Co-authored-by: Thomas Boop <52323235+thboop@users.noreply.github.com> * Update src/Runner.Listener/Configuration/ConfigurationManager.cs Co-authored-by: Thomas Boop <52323235+thboop@users.noreply.github.com> * feedback. Co-authored-by: Thomas Boop <52323235+thboop@users.noreply.github.com> --- src/Runner.Common/ConfigurationStore.cs | 3 + src/Runner.Common/Constants.cs | 1 + src/Runner.Listener/CommandSettings.cs | 2 + .../Configuration/ConfigurationManager.cs | 33 ++++++++-- src/Runner.Listener/Runner.cs | 1 + src/Runner.Worker/JobRunner.cs | 66 ++++++++++++++++--- src/Sdk/DTWebApi/WebApi/TaskAgentReference.cs | 11 ++++ .../Configuration/ConfigurationManagerL0.cs | 8 ++- 8 files changed, 109 insertions(+), 16 deletions(-) diff --git a/src/Runner.Common/ConfigurationStore.cs b/src/Runner.Common/ConfigurationStore.cs index 94230731433..ca526c77fe1 100644 --- a/src/Runner.Common/ConfigurationStore.cs +++ b/src/Runner.Common/ConfigurationStore.cs @@ -33,6 +33,9 @@ public sealed class RunnerSettings [DataMember(EmitDefaultValue = false)] public string PoolName { get; set; } + [DataMember(EmitDefaultValue = false)] + public bool DisableUpdate { get; set; } + [DataMember(EmitDefaultValue = false)] public bool Ephemeral { get; set; } diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index c43aae122b2..829c8223edf 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -129,6 +129,7 @@ public static class Flags public static readonly string Ephemeral = "ephemeral"; public static readonly string Help = "help"; public static readonly string Replace = "replace"; + public static readonly string DisableUpdate = "disableupdate"; public static readonly string Once = "once"; // Keep this around since customers still relies on it public static readonly string RunAsService = "runasservice"; public static readonly string Unattended = "unattended"; diff --git a/src/Runner.Listener/CommandSettings.cs b/src/Runner.Listener/CommandSettings.cs index 050114560ba..de268d859ab 100644 --- a/src/Runner.Listener/CommandSettings.cs +++ b/src/Runner.Listener/CommandSettings.cs @@ -29,6 +29,7 @@ public sealed class CommandSettings { Constants.Runner.CommandLine.Flags.Check, Constants.Runner.CommandLine.Flags.Commit, + Constants.Runner.CommandLine.Flags.DisableUpdate, Constants.Runner.CommandLine.Flags.Ephemeral, Constants.Runner.CommandLine.Flags.Help, Constants.Runner.CommandLine.Flags.Once, @@ -68,6 +69,7 @@ public sealed class CommandSettings public bool Unattended => TestFlag(Constants.Runner.CommandLine.Flags.Unattended); public bool Version => TestFlag(Constants.Runner.CommandLine.Flags.Version); public bool Ephemeral => TestFlag(Constants.Runner.CommandLine.Flags.Ephemeral); + public bool DisableUpdate => TestFlag(Constants.Runner.CommandLine.Flags.DisableUpdate); // Keep this around since customers still relies on it public bool RunOnce => TestFlag(Constants.Runner.CommandLine.Flags.Once); diff --git a/src/Runner.Listener/Configuration/ConfigurationManager.cs b/src/Runner.Listener/Configuration/ConfigurationManager.cs index 09de9ed1b4f..b70a1e7b91e 100644 --- a/src/Runner.Listener/Configuration/ConfigurationManager.cs +++ b/src/Runner.Listener/Configuration/ConfigurationManager.cs @@ -196,6 +196,7 @@ public async Task ConfigureAsync(CommandSettings command) TaskAgent agent; while (true) { + runnerSettings.DisableUpdate = command.DisableUpdate; runnerSettings.Ephemeral = command.Ephemeral; runnerSettings.AgentName = command.GetRunnerName(); @@ -213,11 +214,22 @@ public async Task ConfigureAsync(CommandSettings command) if (command.GetReplace()) { // Update existing agent with new PublicKey, agent version. - agent = UpdateExistingAgent(agent, publicKey, userLabels, runnerSettings.Ephemeral); + agent = UpdateExistingAgent(agent, publicKey, userLabels, runnerSettings.Ephemeral, command.DisableUpdate); try { agent = await _runnerServer.ReplaceAgentAsync(runnerSettings.PoolId, agent); + if (command.DisableUpdate && + command.DisableUpdate != agent.DisableUpdate) + { + throw new NotSupportedException("The GitHub server does not support configuring a self-hosted runner with 'DisableUpdate' flag."); + } + if (command.Ephemeral && + command.Ephemeral != agent.Ephemeral) + { + throw new NotSupportedException("The GitHub server does not support configuring a self-hosted runner with 'Ephemeral' flag."); + } + _term.WriteSuccessMessage("Successfully replaced the runner"); break; } @@ -236,11 +248,22 @@ public async Task ConfigureAsync(CommandSettings command) else { // Create a new agent. - agent = CreateNewAgent(runnerSettings.AgentName, publicKey, userLabels, runnerSettings.Ephemeral); + agent = CreateNewAgent(runnerSettings.AgentName, publicKey, userLabels, runnerSettings.Ephemeral, command.DisableUpdate); try { agent = await _runnerServer.AddAgentAsync(runnerSettings.PoolId, agent); + if (command.DisableUpdate && + command.DisableUpdate != agent.DisableUpdate) + { + throw new NotSupportedException("The GitHub server does not support configuring a self-hosted runner with 'DisableUpdate' flag."); + } + if (command.Ephemeral && + command.Ephemeral != agent.Ephemeral) + { + throw new NotSupportedException("The GitHub server does not support configuring a self-hosted runner with 'Ephemeral' flag."); + } + _term.WriteSuccessMessage("Runner successfully added"); break; } @@ -466,7 +489,7 @@ private ICredentialProvider GetCredentialProvider(CommandSettings command, strin } - private TaskAgent UpdateExistingAgent(TaskAgent agent, RSAParameters publicKey, ISet userLabels, bool ephemeral) + private TaskAgent UpdateExistingAgent(TaskAgent agent, RSAParameters publicKey, ISet userLabels, bool ephemeral, bool disableUpdate) { ArgUtil.NotNull(agent, nameof(agent)); agent.Authorization = new TaskAgentAuthorization @@ -478,6 +501,7 @@ private TaskAgent UpdateExistingAgent(TaskAgent agent, RSAParameters publicKey, agent.Version = BuildConstants.RunnerPackage.Version; agent.OSDescription = RuntimeInformation.OSDescription; agent.Ephemeral = ephemeral; + agent.DisableUpdate = disableUpdate; agent.MaxParallelism = 1; agent.Labels.Clear(); @@ -494,7 +518,7 @@ private TaskAgent UpdateExistingAgent(TaskAgent agent, RSAParameters publicKey, return agent; } - private TaskAgent CreateNewAgent(string agentName, RSAParameters publicKey, ISet userLabels, bool ephemeral) + private TaskAgent CreateNewAgent(string agentName, RSAParameters publicKey, ISet userLabels, bool ephemeral, bool disableUpdate) { TaskAgent agent = new TaskAgent(agentName) { @@ -506,6 +530,7 @@ private TaskAgent CreateNewAgent(string agentName, RSAParameters publicKey, ISet Version = BuildConstants.RunnerPackage.Version, OSDescription = RuntimeInformation.OSDescription, Ephemeral = ephemeral, + DisableUpdate = disableUpdate }; agent.Labels.Add(new AgentLabel("self-hosted", LabelType.System)); diff --git a/src/Runner.Listener/Runner.cs b/src/Runner.Listener/Runner.cs index aa14776bf29..00afd99c7ea 100644 --- a/src/Runner.Listener/Runner.cs +++ b/src/Runner.Listener/Runner.cs @@ -540,6 +540,7 @@ private void PrintUsage(CommandSettings command) --work string Relative runner work directory (default {Constants.Path.WorkDirectory}) --replace Replace any existing runner with the same name (default false) --pat GitHub personal access token used for checking network connectivity when executing `.{separator}run.{ext} --check` + --disableupdate Disable self-hosted runner automatic update to the latest released version` --ephemeral Configure the runner to only take one job and then let the service un-configure the runner after the job finishes (default false)"); #if OS_WINDOWS diff --git a/src/Runner.Worker/JobRunner.cs b/src/Runner.Worker/JobRunner.cs index e402d4ca824..233be55154b 100644 --- a/src/Runner.Worker/JobRunner.cs +++ b/src/Runner.Worker/JobRunner.cs @@ -1,18 +1,18 @@ -using GitHub.DistributedTask.WebApi; -using Pipelines = GitHub.DistributedTask.Pipelines; -using GitHub.Runner.Common.Util; -using GitHub.Services.Common; -using GitHub.Services.WebApi; -using System; +using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Net.Http; using System.Text; using System.Threading; using System.Threading.Tasks; -using System.Net.Http; +using GitHub.DistributedTask.WebApi; using GitHub.Runner.Common; +using GitHub.Runner.Common.Util; using GitHub.Runner.Sdk; +using GitHub.Services.Common; +using GitHub.Services.WebApi; +using Pipelines = GitHub.DistributedTask.Pipelines; namespace GitHub.Runner.Worker { @@ -25,6 +25,7 @@ public interface IJobRunner : IRunnerService public sealed class JobRunner : RunnerService, IJobRunner { private IJobServerQueue _jobServerQueue; + private RunnerSettings _runnerSettings; private ITempDirectoryManager _tempDirectoryManager; public async Task RunAsync(Pipelines.AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken) @@ -108,8 +109,8 @@ public async Task RunAsync(Pipelines.AgentJobRequestMessage message, jobContext.SetRunnerContext("os", VarUtil.OS); jobContext.SetRunnerContext("arch", VarUtil.OSArchitecture); - var runnerSettings = HostContext.GetService().GetSettings(); - jobContext.SetRunnerContext("name", runnerSettings.AgentName); + _runnerSettings = HostContext.GetService().GetSettings(); + jobContext.SetRunnerContext("name", _runnerSettings.AgentName); string toolsDirectory = HostContext.GetDirectory(WellKnownDirectory.Tools); Directory.CreateDirectory(toolsDirectory); @@ -209,6 +210,53 @@ private async Task CompleteJobAsync(IJobServer jobServer, IExecution jobContext.Debug($"Finishing: {message.JobDisplayName}"); TaskResult result = jobContext.Complete(taskResult); + if (_runnerSettings.DisableUpdate == true) + { + try + { + var currentVersion = new PackageVersion(BuildConstants.RunnerPackage.Version); + ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); + VssCredentials serverCredential = VssUtil.GetVssCredential(systemConnection); + + var runnerServer = HostContext.GetService(); + await runnerServer.ConnectAsync(systemConnection.Url, serverCredential); + var serverPackages = await runnerServer.GetPackagesAsync("agent", BuildConstants.RunnerPackage.PackageName, 5, includeToken: false, cancellationToken: CancellationToken.None); + if (serverPackages.Count > 0) + { + serverPackages = serverPackages.OrderByDescending(x => x.Version).ToList(); + Trace.Info($"Newer packages {StringUtil.ConvertToJson(serverPackages.Select(x => x.Version.ToString()))}"); + + var warnOnFailedJob = false; // any minor/patch version behind. + var warnOnOldRunnerVersion = false; // >= 2 minor version behind + if (serverPackages.Any(x => x.Version.CompareTo(currentVersion) > 0)) + { + Trace.Info($"Current runner version {currentVersion} is behind the latest runner version {serverPackages[0].Version}."); + warnOnFailedJob = true; + } + + if (serverPackages.Where(x => x.Version.Major == currentVersion.Major && x.Version.Minor > currentVersion.Minor).Count() > 1) + { + Trace.Info($"Current runner version {currentVersion} is way behind the latest runner version {serverPackages[0].Version}."); + warnOnOldRunnerVersion = true; + } + + if (result == TaskResult.Failed && warnOnFailedJob) + { + jobContext.Warning($"This job failure may be caused by using an out of date self-hosted runner. You are currently using runner version {currentVersion}. Please update to the latest version {serverPackages[0].Version}"); + } + else if (warnOnOldRunnerVersion) + { + jobContext.Warning($"This self-hosted runner is currently using runner version {currentVersion}. This version is out of date. Please update to the latest version {serverPackages[0].Version}"); + } + } + } + catch (Exception ex) + { + // Ignore any error since suggest runner update is best effort. + Trace.Error($"Caught exception during runner version check: {ex}"); + } + } + try { await ShutdownQueue(throwOnFailure: true); diff --git a/src/Sdk/DTWebApi/WebApi/TaskAgentReference.cs b/src/Sdk/DTWebApi/WebApi/TaskAgentReference.cs index 3031241a8d0..743c8c38924 100644 --- a/src/Sdk/DTWebApi/WebApi/TaskAgentReference.cs +++ b/src/Sdk/DTWebApi/WebApi/TaskAgentReference.cs @@ -25,6 +25,7 @@ protected TaskAgentReference(TaskAgentReference referenceToBeCloned) this.ProvisioningState = referenceToBeCloned.ProvisioningState; this.AccessPoint = referenceToBeCloned.AccessPoint; this.Ephemeral = referenceToBeCloned.Ephemeral; + this.DisableUpdate = referenceToBeCloned.DisableUpdate; if (referenceToBeCloned.m_links != null) { @@ -92,6 +93,16 @@ public bool? Ephemeral set; } + /// + /// Whether or not this agent should auto-update to latest version. + /// + [DataMember(EmitDefaultValue = false)] + public bool? DisableUpdate + { + get; + set; + } + /// /// Whether or not the agent is online. /// diff --git a/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs b/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs index dbc0c05bbf0..197cf7e5202 100644 --- a/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs +++ b/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs @@ -66,7 +66,7 @@ public ConfigurationManagerL0() _serviceControlManager = new Mock(); #endif - var expectedAgent = new TaskAgent(_expectedAgentName) { Id = 1 }; + var expectedAgent = new TaskAgent(_expectedAgentName) { Id = 1, Ephemeral = true, DisableUpdate = true }; expectedAgent.Authorization = new TaskAgentAuthorization { ClientId = Guid.NewGuid(), @@ -154,7 +154,7 @@ public async Task CanEnsureConfigure() tc, new[] { - "configure", + "configure", "--url", _expectedServerUrl, "--name", _expectedAgentName, "--runnergroup", _secondRunnerGroupName, @@ -163,6 +163,8 @@ public async Task CanEnsureConfigure() "--token", _expectedToken, "--labels", userLabels, "--ephemeral", + "--disableupdate", + "--unattended", }); trace.Info("Constructed."); _store.Setup(x => x.IsConfigured()).Returns(false); @@ -185,7 +187,7 @@ public async Task CanEnsureConfigure() // validate GetAgentPoolsAsync gets called twice with automation pool type _runnerServer.Verify(x => x.GetAgentPoolsAsync(It.IsAny(), It.Is(p => p == TaskAgentPoolType.Automation)), Times.Exactly(2)); - var expectedLabels = new List() { "self-hosted", VarUtil.OS, VarUtil.OSArchitecture}; + var expectedLabels = new List() { "self-hosted", VarUtil.OS, VarUtil.OSArchitecture }; expectedLabels.AddRange(userLabels.Split(",").ToList()); _runnerServer.Verify(x => x.AddAgentAsync(It.IsAny(), It.Is(a => a.Labels.Select(x => x.Name).ToHashSet().SetEquals(expectedLabels))), Times.Once);