Skip to content

Commit

Permalink
cache actions on premises (actions#381)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ericsciple
ericsciple authored Mar 25, 2020
1 parent ce3397f commit 388abc2
Show file tree
Hide file tree
Showing 4 changed files with 114 additions and 22 deletions.
27 changes: 27 additions & 0 deletions src/Runner.Common/ConfigurationStore.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ namespace GitHub.Runner.Common
[DataContract]
public sealed class RunnerSettings
{
[DataMember(Name = "IsHostedServer", EmitDefaultValue = false)]
private bool? _isHostedServer;

[DataMember(EmitDefaultValue = false)]
public int AgentId { get; set; }

Expand Down Expand Up @@ -42,6 +45,21 @@ public sealed class RunnerSettings
[DataMember(EmitDefaultValue = false)]
public string MonitorSocketAddress { get; set; }

[IgnoreDataMember]
public bool IsHostedServer
{
get
{
// Old runners do not have this property. Hosted runners likely don't have this property either.
return _isHostedServer ?? true;
}

set
{
_isHostedServer = value;
}
}

/// <summary>
// Computed property for convenience. Can either return:
// 1. If runner was configured at the repo level, returns something like: "myorg/myrepo"
Expand Down Expand Up @@ -69,6 +87,15 @@ public string RepoOrOrgName
return repoOrOrgName;
}
}

[OnSerializing]
private void OnSerializing(StreamingContext context)
{
if (_isHostedServer.HasValue && _isHostedServer.Value)
{
_isHostedServer = null;
}
}
}

[ServiceLocator(Default = typeof(ConfigurationStore))]
Expand Down
6 changes: 2 additions & 4 deletions src/Runner.Listener/Configuration/ConfigurationManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,6 @@ public async Task ConfigureAsync(CommandSettings command)

RunnerSettings runnerSettings = new RunnerSettings();

bool isHostedServer = false;
// Loop getting url and creds until you can connect
ICredentialProvider credProvider = null;
VssCredentials creds = null;
Expand Down Expand Up @@ -117,7 +116,7 @@ public async Task ConfigureAsync(CommandSettings command)
try
{
// Determine the service deployment type based on connection data. (Hosted/OnPremises)
isHostedServer = await IsHostedServer(runnerSettings.ServerUrl, creds);
runnerSettings.IsHostedServer = await IsHostedServer(runnerSettings.ServerUrl, creds);

// Validate can connect.
await _runnerServer.ConnectAsync(new Uri(runnerSettings.ServerUrl), creds);
Expand Down Expand Up @@ -248,7 +247,7 @@ public async Task ConfigureAsync(CommandSettings command)
{
UriBuilder configServerUrl = new UriBuilder(runnerSettings.ServerUrl);
UriBuilder oauthEndpointUrlBuilder = new UriBuilder(agent.Authorization.AuthorizationUrl);
if (!isHostedServer && Uri.Compare(configServerUrl.Uri, oauthEndpointUrlBuilder.Uri, UriComponents.SchemeAndServer, UriFormat.Unescaped, StringComparison.OrdinalIgnoreCase) != 0)
if (!runnerSettings.IsHostedServer && Uri.Compare(configServerUrl.Uri, oauthEndpointUrlBuilder.Uri, UriComponents.SchemeAndServer, UriFormat.Unescaped, StringComparison.OrdinalIgnoreCase) != 0)
{
oauthEndpointUrlBuilder.Scheme = configServerUrl.Scheme;
oauthEndpointUrlBuilder.Host = configServerUrl.Host;
Expand Down Expand Up @@ -381,7 +380,6 @@ public async Task UnconfigureAsync(CommandSettings command)
}

// Determine the service deployment type based on connection data. (Hosted/OnPremises)
bool isHostedServer = await IsHostedServer(settings.ServerUrl, creds);
await _runnerServer.ConnectAsync(new Uri(settings.ServerUrl), creds);

var agents = await _runnerServer.GetAgentsAsync(settings.PoolId, settings.AgentName);
Expand Down
52 changes: 34 additions & 18 deletions src/Runner.Worker/ActionManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,14 @@ public async Task<List<JobExtensionRunner>> PrepareActionsAsync(IExecutionContex
executionContext.Warning("The 'PREVIEW_ACTION_TOKEN' secret is depreciated. Please remove it from the repository's secrets");
}

// Clear the cache (local runner)
IOUtil.DeleteDirectory(HostContext.GetDirectory(WellKnownDirectory.Actions), executionContext.CancellationToken);
// Clear the cache (for self-hosted runners)
// Note, temporarily avoid this step for the on-premises product, to avoid rate limiting.
var configurationStore = HostContext.GetService<IConfigurationStore>();
var isHostedServer = configurationStore.GetSettings().IsHostedServer;
if (isHostedServer)
{
IOUtil.DeleteDirectory(HostContext.GetDirectory(WellKnownDirectory.Actions), executionContext.CancellationToken);
}

foreach (var action in actions)
{
Expand Down Expand Up @@ -448,7 +454,8 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont
ArgUtil.NotNullOrEmpty(repositoryReference.Ref, nameof(repositoryReference.Ref));

string destDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), repositoryReference.Name.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar), repositoryReference.Ref);
if (File.Exists(destDirectory + ".completed"))
string watermarkFile = destDirectory + ".completed";
if (File.Exists(watermarkFile))
{
executionContext.Debug($"Action '{repositoryReference.Name}@{repositoryReference.Ref}' already downloaded at '{destDirectory}'.");
return;
Expand Down Expand Up @@ -498,24 +505,33 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont
using (var httpClientHandler = HostContext.CreateHttpClientHandler())
using (var httpClient = new HttpClient(httpClientHandler))
{
var authToken = Environment.GetEnvironmentVariable("_GITHUB_ACTION_TOKEN");
if (string.IsNullOrEmpty(authToken))
{
// TODO: Depreciate the PREVIEW_ACTION_TOKEN
authToken = executionContext.Variables.Get("PREVIEW_ACTION_TOKEN");
}

if (!string.IsNullOrEmpty(authToken))
var configurationStore = HostContext.GetService<IConfigurationStore>();
var isHostedServer = configurationStore.GetSettings().IsHostedServer;
if (isHostedServer)
{
HostContext.SecretMasker.AddValue(authToken);
var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"PAT:{authToken}"));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
var authToken = Environment.GetEnvironmentVariable("_GITHUB_ACTION_TOKEN");
if (string.IsNullOrEmpty(authToken))
{
// TODO: Depreciate the PREVIEW_ACTION_TOKEN
authToken = executionContext.Variables.Get("PREVIEW_ACTION_TOKEN");
}

if (!string.IsNullOrEmpty(authToken))
{
HostContext.SecretMasker.AddValue(authToken);
var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"PAT:{authToken}"));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
}
else
{
var accessToken = executionContext.GetGitHubContext("token");
var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{accessToken}"));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
}
}
else
{
var accessToken = executionContext.GetGitHubContext("token");
var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{accessToken}"));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
// Intentionally empty. Temporary for GHES alpha release, download from dotcom unauthenticated.
}

httpClient.DefaultRequestHeaders.UserAgent.Add(HostContext.UserAgent);
Expand Down Expand Up @@ -610,7 +626,7 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont
}

Trace.Verbose("Create watermark file indicate action download succeed.");
File.WriteAllText(destDirectory + ".completed", DateTime.UtcNow.ToString());
File.WriteAllText(watermarkFile, DateTime.UtcNow.ToString());

executionContext.Debug($"Archive '{archiveFile}' has been unzipped into '{destDirectory}'.");
Trace.Info("Finished getting action repository.");
Expand Down
51 changes: 51 additions & 0 deletions src/Test/L0/Worker/ActionManagerL0.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,57 @@ public async void PrepareActions_DownloadActionFromGraph()
}
}

[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Worker")]
public async void PrepareActions_SkipDownloadActionFromGraphWhenCached_OnPremises()
{
try
{
// Arrange
Setup();
var actionId = Guid.NewGuid();
var actions = new List<Pipelines.ActionStep>
{
new Pipelines.ActionStep()
{
Name = "action",
Id = actionId,
Reference = new Pipelines.RepositoryPathReference()
{
Name = "actions/no-such-action",
Ref = "master",
RepositoryType = "GitHub"
}
}
};
_configurationStore.Object.GetSettings().IsHostedServer = false;
var actionDirectory = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "actions/no-such-action", "master");
Directory.CreateDirectory(actionDirectory);
var watermarkFile = $"{actionDirectory}.completed";
File.WriteAllText(watermarkFile, DateTime.UtcNow.ToString());
var actionFile = Path.Combine(actionDirectory, "action.yml");
File.WriteAllText(actionFile, @"
name: ""no-such-action""
runs:
using: node12
main: no-such-action.js
");
var testFile = Path.Combine(actionDirectory, "test-file");
File.WriteAllText(testFile, "asdf");

// Act
await _actionManager.PrepareActionsAsync(_ec.Object, actions);

// Assert
Assert.True(File.Exists(testFile));
}
finally
{
Teardown();
}
}

[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Worker")]
Expand Down

0 comments on commit 388abc2

Please sign in to comment.