Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

cache actions on premises and download from .com #381

Merged
merged 1 commit into from
Mar 25, 2020
Merged

cache actions on premises and download from .com #381

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions src/Runner.Common/ConfigurationStore.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ namespace GitHub.Runner.Common
[DataContract]
public sealed class RunnerSettings
{
[DataMember(Name = "IsHostedServer", EmitDefaultValue = false)]
private bool? _isHostedServer;

[DataMember(EmitDefaultValue = false)]
public int AgentId { get; set; }

Expand Down Expand Up @@ -42,6 +45,21 @@ public sealed class RunnerSettings
[DataMember(EmitDefaultValue = false)]
public string MonitorSocketAddress { get; set; }

[IgnoreDataMember]
public bool IsHostedServer
{
get
{
// Old runners do not have this property. Hosted runners likely don't have this property either.
return _isHostedServer ?? true;
}

set
{
_isHostedServer = value;
}
}

/// <summary>
// Computed property for convenience. Can either return:
// 1. If runner was configured at the repo level, returns something like: "myorg/myrepo"
Expand Down Expand Up @@ -69,6 +87,15 @@ public string RepoOrOrgName
return repoOrOrgName;
}
}

[OnSerializing]
private void OnSerializing(StreamingContext context)
{
if (_isHostedServer.HasValue && _isHostedServer.Value)
{
_isHostedServer = null;
}
}
}

[ServiceLocator(Default = typeof(ConfigurationStore))]
Expand Down
6 changes: 2 additions & 4 deletions src/Runner.Listener/Configuration/ConfigurationManager.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,6 @@ public async Task ConfigureAsync(CommandSettings command)

RunnerSettings runnerSettings = new RunnerSettings();

bool isHostedServer = false;
// Loop getting url and creds until you can connect
ICredentialProvider credProvider = null;
VssCredentials creds = null;
Expand Down Expand Up @@ -117,7 +116,7 @@ public async Task ConfigureAsync(CommandSettings command)
try
{
// Determine the service deployment type based on connection data. (Hosted/OnPremises)
isHostedServer = await IsHostedServer(runnerSettings.ServerUrl, creds);
runnerSettings.IsHostedServer = await IsHostedServer(runnerSettings.ServerUrl, creds);

// Validate can connect.
await _runnerServer.ConnectAsync(new Uri(runnerSettings.ServerUrl), creds);
Expand Down Expand Up @@ -248,7 +247,7 @@ public async Task ConfigureAsync(CommandSettings command)
{
UriBuilder configServerUrl = new UriBuilder(runnerSettings.ServerUrl);
UriBuilder oauthEndpointUrlBuilder = new UriBuilder(agent.Authorization.AuthorizationUrl);
if (!isHostedServer && Uri.Compare(configServerUrl.Uri, oauthEndpointUrlBuilder.Uri, UriComponents.SchemeAndServer, UriFormat.Unescaped, StringComparison.OrdinalIgnoreCase) != 0)
if (!runnerSettings.IsHostedServer && Uri.Compare(configServerUrl.Uri, oauthEndpointUrlBuilder.Uri, UriComponents.SchemeAndServer, UriFormat.Unescaped, StringComparison.OrdinalIgnoreCase) != 0)
{
oauthEndpointUrlBuilder.Scheme = configServerUrl.Scheme;
oauthEndpointUrlBuilder.Host = configServerUrl.Host;
Expand Down Expand Up @@ -381,7 +380,6 @@ public async Task UnconfigureAsync(CommandSettings command)
}

// Determine the service deployment type based on connection data. (Hosted/OnPremises)
bool isHostedServer = await IsHostedServer(settings.ServerUrl, creds);
await _runnerServer.ConnectAsync(new Uri(settings.ServerUrl), creds);

var agents = await _runnerServer.GetAgentsAsync(settings.PoolId, settings.AgentName);
Expand Down
52 changes: 34 additions & 18 deletions src/Runner.Worker/ActionManager.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,14 @@ public async Task<List<JobExtensionRunner>> PrepareActionsAsync(IExecutionContex
executionContext.Warning("The 'PREVIEW_ACTION_TOKEN' secret is depreciated. Please remove it from the repository's secrets");
}

// Clear the cache (local runner)
IOUtil.DeleteDirectory(HostContext.GetDirectory(WellKnownDirectory.Actions), executionContext.CancellationToken);
// Clear the cache (for self-hosted runners)
// Note, temporarily avoid this step for the on-premises product, to avoid rate limiting.
var configurationStore = HostContext.GetService<IConfigurationStore>();
var isHostedServer = configurationStore.GetSettings().IsHostedServer;
if (isHostedServer)
ericsciple marked this conversation as resolved.
Show resolved Hide resolved
{
IOUtil.DeleteDirectory(HostContext.GetDirectory(WellKnownDirectory.Actions), executionContext.CancellationToken);
}

foreach (var action in actions)
{
Expand Down Expand Up @@ -448,7 +454,8 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont
ArgUtil.NotNullOrEmpty(repositoryReference.Ref, nameof(repositoryReference.Ref));

string destDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), repositoryReference.Name.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar), repositoryReference.Ref);
if (File.Exists(destDirectory + ".completed"))
string watermarkFile = destDirectory + ".completed";
if (File.Exists(watermarkFile))
{
executionContext.Debug($"Action '{repositoryReference.Name}@{repositoryReference.Ref}' already downloaded at '{destDirectory}'.");
return;
Expand Down Expand Up @@ -498,24 +505,33 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont
using (var httpClientHandler = HostContext.CreateHttpClientHandler())
using (var httpClient = new HttpClient(httpClientHandler))
{
var authToken = Environment.GetEnvironmentVariable("_GITHUB_ACTION_TOKEN");
if (string.IsNullOrEmpty(authToken))
{
// TODO: Depreciate the PREVIEW_ACTION_TOKEN
authToken = executionContext.Variables.Get("PREVIEW_ACTION_TOKEN");
}

if (!string.IsNullOrEmpty(authToken))
var configurationStore = HostContext.GetService<IConfigurationStore>();
var isHostedServer = configurationStore.GetSettings().IsHostedServer;
if (isHostedServer)
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the diff looks bad, but adding auth header just moved inside if block

{
HostContext.SecretMasker.AddValue(authToken);
var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"PAT:{authToken}"));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
var authToken = Environment.GetEnvironmentVariable("_GITHUB_ACTION_TOKEN");
if (string.IsNullOrEmpty(authToken))
{
// TODO: Depreciate the PREVIEW_ACTION_TOKEN
authToken = executionContext.Variables.Get("PREVIEW_ACTION_TOKEN");
}

if (!string.IsNullOrEmpty(authToken))
{
HostContext.SecretMasker.AddValue(authToken);
var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"PAT:{authToken}"));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
}
else
{
var accessToken = executionContext.GetGitHubContext("token");
var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{accessToken}"));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
}
}
else
{
var accessToken = executionContext.GetGitHubContext("token");
var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{accessToken}"));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
// Intentionally empty. Temporary for GHES alpha release, download from dotcom unauthenticated.
}

httpClient.DefaultRequestHeaders.UserAgent.Add(HostContext.UserAgent);
Expand Down Expand Up @@ -610,7 +626,7 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont
}

Trace.Verbose("Create watermark file indicate action download succeed.");
File.WriteAllText(destDirectory + ".completed", DateTime.UtcNow.ToString());
File.WriteAllText(watermarkFile, DateTime.UtcNow.ToString());

executionContext.Debug($"Archive '{archiveFile}' has been unzipped into '{destDirectory}'.");
Trace.Info("Finished getting action repository.");
Expand Down
51 changes: 51 additions & 0 deletions src/Test/L0/Worker/ActionManagerL0.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,57 @@ public async void PrepareActions_DownloadActionFromGraph()
}
}

[Fact]
ericsciple marked this conversation as resolved.
Show resolved Hide resolved
[Trait("Level", "L0")]
[Trait("Category", "Worker")]
public async void PrepareActions_SkipDownloadActionFromGraphWhenCached_OnPremises()
{
try
{
// Arrange
Setup();
var actionId = Guid.NewGuid();
var actions = new List<Pipelines.ActionStep>
{
new Pipelines.ActionStep()
{
Name = "action",
Id = actionId,
Reference = new Pipelines.RepositoryPathReference()
{
Name = "actions/no-such-action",
Ref = "master",
RepositoryType = "GitHub"
}
}
};
_configurationStore.Object.GetSettings().IsHostedServer = false;
var actionDirectory = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "actions/no-such-action", "master");
Directory.CreateDirectory(actionDirectory);
var watermarkFile = $"{actionDirectory}.completed";
File.WriteAllText(watermarkFile, DateTime.UtcNow.ToString());
var actionFile = Path.Combine(actionDirectory, "action.yml");
File.WriteAllText(actionFile, @"
name: ""no-such-action""
runs:
using: node12
main: no-such-action.js
");
var testFile = Path.Combine(actionDirectory, "test-file");
File.WriteAllText(testFile, "asdf");

// Act
await _actionManager.PrepareActionsAsync(_ec.Object, actions);

// Assert
Assert.True(File.Exists(testFile));
}
finally
{
Teardown();
}
}

[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Worker")]
Expand Down