Skip to content

Commit

Permalink
Add AllowWorkDirectoryRepositories knob (#4423)
Browse files Browse the repository at this point in the history 
* New knob AllowWorkDirectoryRepositories

* test changes

* Bring back the knob changes for CI testing

* Add tests for BuildDirectoryManager

* Add tests for RepositoryPlugin

* Remove redundant code

* Add missing Knob parameter that was accidentally deleted during merge conflict resolution

---------

Co-authored-by: Kirill Ivlev <102740624+kirill-ivlev@users.noreply.github.com>
  • Loading branch information
@aleksandrlevochkin @kirill-ivlev
aleksandrlevochkin and kirill-ivlev authored Oct 17, 2023
1 parent 34e2c30 commit a3067b7
Show file tree
Hide file tree
Showing 7 changed files with 229 additions and 23 deletions.
14 changes: 12 additions & 2 deletions src/Agent.Plugins/RepositoryPlugin.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,24 +140,34 @@ public override async Task RunAsync(AgentTaskPluginExecutionContext executionCon
MergeCheckoutOptions(executionContext, repo);

var currentRepoPath = repo.Properties.Get<string>(Pipelines.RepositoryPropertyNames.Path);
var workDirectory = executionContext.Variables.GetValueOrDefault("agent.workfolder")?.Value;
var buildDirectory = executionContext.Variables.GetValueOrDefault("agent.builddirectory")?.Value;
var tempDirectory = executionContext.Variables.GetValueOrDefault("agent.tempdirectory")?.Value;

ArgUtil.NotNullOrEmpty(currentRepoPath, nameof(currentRepoPath));
ArgUtil.NotNullOrEmpty(workDirectory, nameof(workDirectory));
ArgUtil.NotNullOrEmpty(buildDirectory, nameof(buildDirectory));
ArgUtil.NotNullOrEmpty(tempDirectory, nameof(tempDirectory));

// Determine the path that we should clone/move the repository into
const string sourcesDirectory = "s"; //Constants.Build.Path.SourcesDirectory
string expectRepoPath;
var path = executionContext.GetInput("path");
var maxRootDirectory = buildDirectory;

if (!string.IsNullOrEmpty(path))
{
// When the checkout task provides a path, always use that one
expectRepoPath = IOUtil.ResolvePath(buildDirectory, path);
if (!expectRepoPath.StartsWith(buildDirectory.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar))

if (AgentKnobs.AllowWorkDirectoryRepositories.GetValue(executionContext).AsBoolean())
{
maxRootDirectory = workDirectory;
}

if (!expectRepoPath.StartsWith(maxRootDirectory.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar))
{
throw new ArgumentException($"Input path '{path}' should resolve to a directory under '{buildDirectory}', current resolved path '{expectRepoPath}'.");
throw new ArgumentException($"Input path '{path}' should resolve to a directory under '{maxRootDirectory}', current resolved path '{expectRepoPath}'.");
}
}
else if (HasMultipleCheckouts(executionContext))
Expand Down
7 changes: 7 additions & 0 deletions src/Agent.Sdk/Knob/AgentKnobs.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -520,6 +520,13 @@ public class AgentKnobs
new EnvironmentKnobSource("AZP_AGENT_IGNORE_VSTSTASKLIB"),
new BuiltInDefaultKnobSource("false"));

public static readonly Knob AllowWorkDirectoryRepositories = new Knob(
nameof(AllowWorkDirectoryRepositories),
"Allows repositories to be checked out below work directory level on self hosted agents.",
new RuntimeKnobSource("AZP_AGENT_ALLOW_WORK_DIRECTORY_REPOSITORIES"),
new EnvironmentKnobSource("AZP_AGENT_ALLOW_WORK_DIRECTORY_REPOSITORIES"),
new BuiltInDefaultKnobSource("false"));

public static readonly Knob CheckForTaskDeprecation = new Knob(
nameof(CheckForTaskDeprecation),
"If true, the agent will check in the 'Initialize job' step each task used in the job for task deprecation.",
Expand Down
33 changes: 27 additions & 6 deletions src/Agent.Worker/Build/BuildDirectoryManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@ TrackingConfig UpdateDirectory(

string GetRelativeRepositoryPath(
string buildDirectory,
string repositoryPath);
string repositoryPath,
IExecutionContext executionContext);
}

public sealed class BuildDirectoryManager : AgentService, IBuildDirectoryManager
Expand Down Expand Up @@ -164,7 +165,7 @@ public TrackingConfig UpdateDirectory(

// Update the repositoryInfo on the config
string buildDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), trackingConfig.BuildDirectory);
string relativeRepoPath = GetRelativeRepositoryPath(buildDirectory, repoPath);
string relativeRepoPath = GetRelativeRepositoryPath(buildDirectory, repoPath, executionContext);
var effectedRepo = trackingConfig.RepositoryTrackingInfo.FirstOrDefault(r => string.Equals(r.Identifier, updatedRepository.Alias, StringComparison.OrdinalIgnoreCase));
if (effectedRepo != null)
{
Expand All @@ -188,19 +189,39 @@ public TrackingConfig UpdateDirectory(

public string GetRelativeRepositoryPath(
string buildDirectory,
string repositoryPath)
string repositoryPath,
IExecutionContext executionContext)
{
var maxRootDirectory = buildDirectory;
var workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
var allowWorkDirectoryRepositories = AgentKnobs.AllowWorkDirectoryRepositories.GetValue(executionContext).AsBoolean();

ArgUtil.NotNullOrEmpty(buildDirectory, nameof(buildDirectory));
ArgUtil.NotNullOrEmpty(repositoryPath, nameof(repositoryPath));

if (repositoryPath.StartsWith(buildDirectory + Path.DirectorySeparatorChar) || repositoryPath.StartsWith(buildDirectory + Path.AltDirectorySeparatorChar))
// resolve any potentially left over relative part of the path
repositoryPath = Path.GetFullPath(repositoryPath);

if (allowWorkDirectoryRepositories)
{
maxRootDirectory = workDirectory;
}

if (repositoryPath.StartsWith(maxRootDirectory + Path.DirectorySeparatorChar) || repositoryPath.StartsWith(maxRootDirectory + Path.AltDirectorySeparatorChar))
{
// The sourcesDirectory in tracking file is a relative path to agent's work folder.
return repositoryPath.Substring(HostContext.GetDirectory(WellKnownDirectory.Work).Length + 1).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
return repositoryPath.Substring(workDirectory.Length + 1).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
}
else
{
throw new ArgumentException($"Repository path '{repositoryPath}' should be located under agent's work directory '{buildDirectory}'.");
if (allowWorkDirectoryRepositories)
{
throw new ArgumentException($"Repository path '{repositoryPath}' should be located under agent's work directory '{workDirectory}'.");
}
else
{
throw new ArgumentException($"Repository path '{repositoryPath}' should be located under agent's build directory '{buildDirectory}'.");
}
}
}

Expand Down
3 changes: 1 addition & 2 deletions src/Agent.Worker/PluginInternalCommandExtension.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
using Microsoft.VisualStudio.Services.Agent.Worker.Build;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;

Expand Down Expand Up @@ -71,7 +70,7 @@ public void Execute(IExecutionContext context, Command command)
// In Multi-checkout, we don't want to reset sources dir or default working dir.
// So, we will just reset the repo local path
string buildDirectory = context.Variables.Get(Constants.Variables.Pipeline.Workspace);
string repoRelativePath = directoryManager.GetRelativeRepositoryPath(buildDirectory, repositoryPath);
string repoRelativePath = directoryManager.GetRelativeRepositoryPath(buildDirectory, repositoryPath, context);

string sourcesDirectory = context.Variables.Get(Constants.Variables.Build.SourcesDirectory);
string repoLocalPath = context.Variables.Get(Constants.Variables.Build.RepoLocalPath);
Expand Down
111 changes: 104 additions & 7 deletions src/Test/L0/Plugin/RepositoryPluginL0.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,8 @@

using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using Agent.Plugins.Repository;
Expand Down Expand Up @@ -338,10 +336,53 @@ public async Task RepositoryPlugin_NoPathInputMoveBackToDefault()
}
}

public async Task RepositoryPlugin_InvalidPathInputDirectlyToBuildDirectory_DontAllowWorkingDirectoryRepository()
{
using (TestHostContext tc = new TestHostContext(this))
{
var trace = tc.GetTrace();
Setup(tc);
var repository = _executionContext.Repositories.Single();
var currentPath = repository.Properties.Get<string>(Pipelines.RepositoryPropertyNames.Path);
Directory.CreateDirectory(currentPath);
_executionContext.Inputs["Path"] = $"..{Path.DirectorySeparatorChar}1";

var ex = await Assert.ThrowsAsync<ArgumentException>(async () => await _checkoutTask.RunAsync(_executionContext, CancellationToken.None));
Assert.True(ex.Message.Contains("should resolve to a directory under"));

var temp = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
File.Copy(tc.TraceFileName, temp);
Assert.False(File.ReadAllText(temp).Contains($"##vso[plugininternal.updaterepositorypath alias=myRepo;]"));
}
}

[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Plugin")]
public async Task RepositoryPlugin_InvalidPathInput()
public async Task RepositoryPlugin_InvalidPathInputDirectlyToWorkingDirectory_AllowWorkingDirectoryRepositorie()
{
using (TestHostContext tc = new TestHostContext(this))
{
var trace = tc.GetTrace();
Setup(tc, allowWorkDirectory: "true");
var repository = _executionContext.Repositories.Single();
var currentPath = repository.Properties.Get<string>(Pipelines.RepositoryPropertyNames.Path);
Directory.CreateDirectory(currentPath);
_executionContext.Inputs["Path"] = $"..";

var ex = await Assert.ThrowsAsync<ArgumentException>(async () => await _checkoutTask.RunAsync(_executionContext, CancellationToken.None));
Assert.True(ex.Message.Contains("should resolve to a directory under"));

var temp = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
File.Copy(tc.TraceFileName, temp);
Assert.False(File.ReadAllText(temp).Contains($"##vso[plugininternal.updaterepositorypath alias=myRepo;]"));
}
}

[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Plugin")]
public async Task RepositoryPlugin_InvalidPathInput_DontAllowWorkingDirectoryRepositorie()
{
using (TestHostContext tc = new TestHostContext(this))
{
Expand All @@ -350,7 +391,7 @@ public async Task RepositoryPlugin_InvalidPathInput()
var repository = _executionContext.Repositories.Single();
var currentPath = repository.Properties.Get<string>(Pipelines.RepositoryPropertyNames.Path);
Directory.CreateDirectory(currentPath);
_executionContext.Inputs["Path"] = "..";
_executionContext.Inputs["Path"] = $"..{Path.DirectorySeparatorChar}test{Path.DirectorySeparatorChar}foo";

var ex = await Assert.ThrowsAsync<ArgumentException>(async () => await _checkoutTask.RunAsync(_executionContext, CancellationToken.None));
Assert.True(ex.Message.Contains("should resolve to a directory under"));
Expand All @@ -361,6 +402,58 @@ public async Task RepositoryPlugin_InvalidPathInput()
}
}

[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Plugin")]
public async Task RepositoryPlugin_ValidPathInput_AllowWorkingDirectoryRepositorie()
{
using (TestHostContext tc = new TestHostContext(this))
{
var trace = tc.GetTrace();
Setup(tc, allowWorkDirectory: "true");
var repository = _executionContext.Repositories.Single();
var currentPath = repository.Properties.Get<string>(Pipelines.RepositoryPropertyNames.Path);
Directory.CreateDirectory(currentPath);
_executionContext.Inputs["Path"] = $"..{Path.DirectorySeparatorChar}test{Path.DirectorySeparatorChar}foo";


await _checkoutTask.RunAsync(_executionContext, CancellationToken.None);

var actualPath = repository.Properties.Get<string>(Pipelines.RepositoryPropertyNames.Path);

Assert.NotEqual(actualPath, currentPath);
Assert.Equal(actualPath, Path.Combine(tc.GetDirectory(WellKnownDirectory.Work), "test", "foo"));
Assert.True(Directory.Exists(actualPath));
Assert.False(Directory.Exists(currentPath));

var temp = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
File.Copy(tc.TraceFileName, temp);
Assert.True(File.ReadAllText(temp).Contains($"##vso[plugininternal.updaterepositorypath alias=myRepo;]{actualPath}"));
}
}

[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Plugin")]
public async Task RepositoryPlugin_InvalidPathInput_AllowWorkingDirectoryRepositorie()
{
using (TestHostContext tc = new TestHostContext(this))
{
var trace = tc.GetTrace();
Setup(tc, allowWorkDirectory: "true");
var repository = _executionContext.Repositories.Single();
var currentPath = repository.Properties.Get<string>(Pipelines.RepositoryPropertyNames.Path);
Directory.CreateDirectory(currentPath);
_executionContext.Inputs["Path"] = $"..{Path.DirectorySeparatorChar}..{Path.DirectorySeparatorChar}test{Path.DirectorySeparatorChar}foo";

var ex = await Assert.ThrowsAsync<ArgumentException>(async () => await _checkoutTask.RunAsync(_executionContext, CancellationToken.None));
Assert.True(ex.Message.Contains("should resolve to a directory under"));
var temp = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
File.Copy(tc.TraceFileName, temp);
Assert.False(File.ReadAllText(temp).Contains($"##vso[plugininternal.updaterepositorypath alias=myRepo;]"));
}
}

[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Plugin")]
Expand Down Expand Up @@ -440,12 +533,12 @@ private Pipelines.RepositoryResource GetRepository(TestHostContext hostContext,
return repo;
}

private void Setup(TestHostContext hostContext)
private void Setup(TestHostContext hostContext, string allowWorkDirectory = "false")
{
Setup(hostContext, new List<Pipelines.RepositoryResource>() { GetRepository(hostContext, "myRepo", "s") });
Setup(hostContext, new List<Pipelines.RepositoryResource>() { GetRepository(hostContext, "myRepo", "s") }, allowWorkDirectory);
}

private void Setup(TestHostContext hostContext, List<Pipelines.RepositoryResource> repos)
private void Setup(TestHostContext hostContext, List<Pipelines.RepositoryResource> repos, string allowWorkDirectory = "false")
{
_executionContext = new AgentTaskPluginExecutionContext(hostContext.GetTrace())
{
Expand All @@ -468,6 +561,10 @@ private void Setup(TestHostContext hostContext, List<Pipelines.RepositoryResourc
{
"agent.tempdirectory",
hostContext.GetDirectory(WellKnownDirectory.Temp)
},
{
"AZP_AGENT_ALLOW_WORK_DIRECTORY_REPOSITORIES",
allowWorkDirectory
}
},
JobSettings = new Dictionary<string, string>()
Expand Down
Loading

0 comments on commit a3067b7

Please sign in to comment.