Skip to content

Commit

Permalink
Add support to specify substitutions externally (#88230)
Browse files Browse the repository at this point in the history 
ILLinker allows specifying substitutions on the command line, same as descriptors. This can be useful for various workarounds. Add support for this for ILC as well.
  • Loading branch information
@MichalStrehovsky
MichalStrehovsky authored Jul 11, 2023
1 parent 17a5d43 commit 076f8c5
Show file tree
Hide file tree
Showing 15 changed files with 244 additions and 50 deletions.
11 changes: 9 additions & 2 deletions src/coreclr/tools/Common/Compiler/ProcessLinkerXmlBase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,7 @@
using System.Reflection.Metadata;
using System.Text;
using System.Text.RegularExpressions;
#if !READYTORUN
using System.Xml;
#endif
using System.Xml.Linq;
using System.Xml.XPath;

Expand Down Expand Up @@ -59,6 +57,15 @@ public abstract class ProcessLinkerXmlBase
private readonly IReadOnlyDictionary<string, bool> _featureSwitchValues;
protected readonly TypeSystemContext _context;

protected ProcessLinkerXmlBase(Logger logger, TypeSystemContext context, XmlReader xmlReader, string xmlDocumentLocation, IReadOnlyDictionary<string, bool> featureSwitchValues)
{
_context = context;
_logger = logger;
_document = XDocument.Load(xmlReader, LoadOptions.SetLineInfo).CreateNavigator();
_xmlDocumentLocation = xmlDocumentLocation;
_featureSwitchValues = featureSwitchValues;
}

protected ProcessLinkerXmlBase(Logger logger, TypeSystemContext context, Stream documentStream, string xmlDocumentLocation, IReadOnlyDictionary<string, bool> featureSwitchValues)
{
_context = context;
Expand Down
2 changes: 1 addition & 1 deletion src/coreclr/tools/aot/ILCompiler.Compiler.Tests/DependencyGraphTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ public void TestDependencyGraphInvariants(EcmaMethod method)
new FullyBlockedMetadataBlockingPolicy(), new FullyBlockedManifestResourceBlockingPolicy(),
null, new NoStackTraceEmissionPolicy(), new NoDynamicInvokeThunkGenerationPolicy(),
new ILLink.Shared.TrimAnalysis.FlowAnnotations(Logger.Null, ilProvider, compilerGeneratedState), UsageBasedMetadataGenerationOptions.None,
default, Logger.Null, Array.Empty<KeyValuePair<string, bool>>(), Array.Empty<string>(), Array.Empty<string>(), Array.Empty<string>(), Array.Empty<string>());
default, Logger.Null, new Dictionary<string, bool>(), Array.Empty<string>(), Array.Empty<string>(), Array.Empty<string>(), Array.Empty<string>());

CompilationBuilder builder = new RyuJitCompilationBuilder(context, compilationGroup)
.UseILProvider(ilProvider);
Expand Down
2 changes: 1 addition & 1 deletion src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/BodySubstitution.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

namespace ILCompiler
{
internal sealed class BodySubstitution
public sealed class BodySubstitution
{
private object _value;

Expand Down
54 changes: 51 additions & 3 deletions src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/BodySubstitutionParser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,15 @@
using System.IO;
using System.Reflection.Metadata;
using Internal.TypeSystem;
using System.Xml;
using System.Xml.XPath;
using System.Globalization;
using System.Linq;
using ILLink.Shared;

namespace ILCompiler
{
internal sealed class BodySubstitutionsParser : ProcessLinkerXmlBase
public sealed class BodySubstitutionsParser : ProcessLinkerXmlBase
{
private readonly Dictionary<MethodDesc, BodySubstitution> _methodSubstitutions;
private readonly Dictionary<FieldDesc, object> _fieldSubstitutions;
Expand All @@ -27,6 +28,13 @@ private BodySubstitutionsParser(Logger logger, TypeSystemContext context, Stream
_fieldSubstitutions = new Dictionary<FieldDesc, object>();
}

private BodySubstitutionsParser(Logger logger, TypeSystemContext context, XmlReader document, string xmlDocumentLocation, IReadOnlyDictionary<string, bool> featureSwitchValues)
: base(logger, context, document, xmlDocumentLocation, featureSwitchValues)
{
_methodSubstitutions = new Dictionary<MethodDesc, BodySubstitution>();
_fieldSubstitutions = new Dictionary<FieldDesc, object>();
}

protected override void ProcessAssembly(ModuleDesc assembly, XPathNavigator nav, bool warnOnUnresolvedTypes)
{
ProcessTypes(assembly, nav, warnOnUnresolvedTypes);
Expand Down Expand Up @@ -178,11 +186,51 @@ private static object TryCreateSubstitution(TypeDesc type, string value)
return null;
}

public static (Dictionary<MethodDesc, BodySubstitution>, Dictionary<FieldDesc, object>) GetSubstitutions(Logger logger, TypeSystemContext context, UnmanagedMemoryStream documentStream, ManifestResource resource, ModuleDesc resourceAssembly, string xmlDocumentLocation, IReadOnlyDictionary<string, bool> featureSwitchValues)
public static BodyAndFieldSubstitutions GetSubstitutions(Logger logger, TypeSystemContext context, Stream documentStream, ManifestResource resource, ModuleDesc resourceAssembly, string xmlDocumentLocation, IReadOnlyDictionary<string, bool> featureSwitchValues)
{
var rdr = new BodySubstitutionsParser(logger, context, documentStream, resource, resourceAssembly, xmlDocumentLocation, featureSwitchValues);
rdr.ProcessXml(false);
return (rdr._methodSubstitutions, rdr._fieldSubstitutions);
return new BodyAndFieldSubstitutions(rdr._methodSubstitutions, rdr._fieldSubstitutions);
}

public static BodyAndFieldSubstitutions GetSubstitutions(Logger logger, TypeSystemContext context, XmlReader reader, string xmlDocumentLocation, IReadOnlyDictionary<string, bool> featureSwitchValues)
{
var rdr = new BodySubstitutionsParser(logger, context, reader, xmlDocumentLocation, featureSwitchValues);
rdr.ProcessXml(false);
return new BodyAndFieldSubstitutions(rdr._methodSubstitutions, rdr._fieldSubstitutions);
}
}

public struct BodyAndFieldSubstitutions
{
private Dictionary<MethodDesc, BodySubstitution> _bodySubstitutions;
private Dictionary<FieldDesc, object> _fieldSubstitutions;

public IReadOnlyDictionary<MethodDesc, BodySubstitution> BodySubstitutions => _bodySubstitutions;
public IReadOnlyDictionary<FieldDesc, object> FieldSubstitutions => _fieldSubstitutions;

public BodyAndFieldSubstitutions(Dictionary<MethodDesc, BodySubstitution> bodySubstitutions, Dictionary<FieldDesc, object> fieldSubstitutions)
=> (_bodySubstitutions, _fieldSubstitutions) = (bodySubstitutions, fieldSubstitutions);

public void AppendFrom(BodyAndFieldSubstitutions other)
{
if (_bodySubstitutions == null)
{
_bodySubstitutions = other._bodySubstitutions;
_fieldSubstitutions = other._fieldSubstitutions;
}
else if (other._bodySubstitutions == null)
{
// Nothing to do
}
else
{
foreach (var kvp in other._bodySubstitutions)
_bodySubstitutions[kvp.Key] = kvp.Value;

foreach (var kvp in other._fieldSubstitutions)
_fieldSubstitutions[kvp.Key] = kvp.Value;
}
}
}
}
32 changes: 17 additions & 15 deletions src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/FeatureSwitchManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,10 @@ public class FeatureSwitchManager : ILProvider
private readonly FeatureSwitchHashtable _hashtable;
private readonly ILProvider _nestedILProvider;

public FeatureSwitchManager(ILProvider nestedILProvider, Logger logger, IEnumerable<KeyValuePair<string, bool>> switchValues)
public FeatureSwitchManager(ILProvider nestedILProvider, Logger logger, IReadOnlyDictionary<string, bool> switchValues, BodyAndFieldSubstitutions globalSubstitutions)
{
_nestedILProvider = nestedILProvider;

// Last setting wins
var dictionary = new Dictionary<string, bool>();
foreach ((string name, bool value) in switchValues)
dictionary[name] = value;

_hashtable = new FeatureSwitchHashtable(logger, dictionary);
_hashtable = new FeatureSwitchHashtable(logger, switchValues, globalSubstitutions);
}

private BodySubstitution GetSubstitution(MethodDesc method)
Expand Down Expand Up @@ -895,13 +889,15 @@ public SubstitutedDebugInformation(MethodDebugInformation originalDebugInformati

private sealed class FeatureSwitchHashtable : LockFreeReaderHashtable<EcmaModule, AssemblyFeatureInfo>
{
internal readonly Dictionary<string, bool> _switchValues;
internal readonly IReadOnlyDictionary<string, bool> _switchValues;
private readonly Logger _logger;
private readonly BodyAndFieldSubstitutions _globalSubstitutions;

public FeatureSwitchHashtable(Logger logger, Dictionary<string, bool> switchValues)
public FeatureSwitchHashtable(Logger logger, IReadOnlyDictionary<string, bool> switchValues, BodyAndFieldSubstitutions globalSubstitutions)
{
_logger = logger;
_switchValues = switchValues;
_globalSubstitutions = globalSubstitutions;
}

protected override bool CompareKeyToValue(EcmaModule key, AssemblyFeatureInfo value) => key == value.Module;
Expand All @@ -911,19 +907,19 @@ public FeatureSwitchHashtable(Logger logger, Dictionary<string, bool> switchValu

protected override AssemblyFeatureInfo CreateValueFromKey(EcmaModule key)
{
return new AssemblyFeatureInfo(key, _logger, _switchValues);
return new AssemblyFeatureInfo(key, _logger, _switchValues, _globalSubstitutions);
}
}

private sealed class AssemblyFeatureInfo
{
public EcmaModule Module { get; }

public Dictionary<MethodDesc, BodySubstitution> BodySubstitutions { get; }
public Dictionary<FieldDesc, object> FieldSubstitutions { get; }
public IReadOnlyDictionary<MethodDesc, BodySubstitution> BodySubstitutions { get; }
public IReadOnlyDictionary<FieldDesc, object> FieldSubstitutions { get; }
public Dictionary<string, string> InlineableResourceStrings { get; }

public AssemblyFeatureInfo(EcmaModule module, Logger logger, IReadOnlyDictionary<string, bool> featureSwitchValues)
public AssemblyFeatureInfo(EcmaModule module, Logger logger, IReadOnlyDictionary<string, bool> featureSwitchValues, BodyAndFieldSubstitutions globalSubstitutions)
{
Module = module;

Expand Down Expand Up @@ -951,7 +947,13 @@ public AssemblyFeatureInfo(EcmaModule module, Logger logger, IReadOnlyDictionary
ms = new UnmanagedMemoryStream(reader.CurrentPointer, length);
}

(BodySubstitutions, FieldSubstitutions) = BodySubstitutionsParser.GetSubstitutions(logger, module.Context, ms, resource, module, "name", featureSwitchValues);
BodyAndFieldSubstitutions substitutions = BodySubstitutionsParser.GetSubstitutions(logger, module.Context, ms, resource, module, "name", featureSwitchValues);

// Also apply any global substitutions
// Note we allow these to overwrite substitutions in the assembly
substitutions.AppendFrom(globalSubstitutions);

(BodySubstitutions, FieldSubstitutions) = (substitutions.BodySubstitutions, substitutions.FieldSubstitutions);
}
else if (InlineableStringsResourceNode.IsInlineableStringsResource(module, resourceName))
{
Expand Down
85 changes: 70 additions & 15 deletions src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/ManifestResourceBlockingPolicy.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.IO;
using System.Reflection.Metadata;
using System.Reflection.PortableExecutable;
using System.Xml;
using System.Xml.XPath;

using Internal.TypeSystem;
Expand All @@ -23,9 +24,9 @@ public class ManifestResourceBlockingPolicy

protected ManifestResourceBlockingPolicy() { }

public ManifestResourceBlockingPolicy(Logger logger, IEnumerable<KeyValuePair<string, bool>> switchValues)
public ManifestResourceBlockingPolicy(Logger logger, IReadOnlyDictionary<string, bool> switchValues, IReadOnlyDictionary<ModuleDesc, IReadOnlySet<string>> globalBlocks)
{
_hashtable = new FeatureSwitchHashtable(logger, new Dictionary<string, bool>(switchValues));
_hashtable = new FeatureSwitchHashtable(logger, switchValues, globalBlocks);
}

/// <summary>
Expand All @@ -39,15 +40,47 @@ public virtual bool IsManifestResourceBlocked(ModuleDesc module, string resource
|| (resourceName.StartsWith("ILLink.") && resourceName.EndsWith(".xml")));
}

public static IReadOnlyDictionary<ModuleDesc, IReadOnlySet<string>> UnionBlockings(IReadOnlyDictionary<ModuleDesc, IReadOnlySet<string>> left, IReadOnlyDictionary<ModuleDesc, IReadOnlySet<string>> right)
{
var result = new Dictionary<ModuleDesc, HashSet<string>>();
if (left != null)
AddAll(result, left);
if (right != null)
AddAll(result, right);

return AsReadOnlyDictionary(result);

static void AddAll(Dictionary<ModuleDesc, HashSet<string>> result, IReadOnlyDictionary<ModuleDesc, IReadOnlySet<string>> addend)
{
foreach (var item in addend)
{
if (!result.TryGetValue(item.Key, out HashSet<string> set))
result.Add(item.Key, set = new HashSet<string>());
set.UnionWith(item.Value);
}
}
}

private static Dictionary<ModuleDesc, IReadOnlySet<string>> AsReadOnlyDictionary(Dictionary<ModuleDesc, HashSet<string>> original)
{
// IReadOnlyDictionary is not variant, so we need to:
var copy = new Dictionary<ModuleDesc, IReadOnlySet<string>>();
foreach (KeyValuePair<ModuleDesc, HashSet<string>> moduleSet in original)
copy.Add(moduleSet.Key, moduleSet.Value);
return copy;
}

private sealed class FeatureSwitchHashtable : LockFreeReaderHashtable<EcmaModule, AssemblyFeatureInfo>
{
private readonly Dictionary<string, bool> _switchValues;
private readonly IReadOnlyDictionary<string, bool> _switchValues;
private readonly IReadOnlyDictionary<ModuleDesc, IReadOnlySet<string>> _globalBlocks;
private readonly Logger _logger;

public FeatureSwitchHashtable(Logger logger, Dictionary<string, bool> switchValues)
public FeatureSwitchHashtable(Logger logger, IReadOnlyDictionary<string, bool> switchValues, IReadOnlyDictionary<ModuleDesc, IReadOnlySet<string>> globalBlocks)
{
_logger = logger;
_switchValues = switchValues;
_globalBlocks = globalBlocks;
}

protected override bool CompareKeyToValue(EcmaModule key, AssemblyFeatureInfo value) => key == value.Module;
Expand All @@ -57,17 +90,17 @@ public FeatureSwitchHashtable(Logger logger, Dictionary<string, bool> switchValu

protected override AssemblyFeatureInfo CreateValueFromKey(EcmaModule key)
{
return new AssemblyFeatureInfo(key, _logger, _switchValues);
return new AssemblyFeatureInfo(key, _logger, _switchValues, _globalBlocks);
}
}

private sealed class AssemblyFeatureInfo
{
public EcmaModule Module { get; }

public HashSet<string> BlockedResources { get; }
public IReadOnlySet<string> BlockedResources { get; }

public AssemblyFeatureInfo(EcmaModule module, Logger logger, IReadOnlyDictionary<string, bool> featureSwitchValues)
public AssemblyFeatureInfo(EcmaModule module, Logger logger, IReadOnlyDictionary<string, bool> featureSwitchValues, IReadOnlyDictionary<ModuleDesc, IReadOnlySet<string>> globalBlocks)
{
Module = module;
BlockedResources = new HashSet<string>();
Expand Down Expand Up @@ -96,29 +129,49 @@ public AssemblyFeatureInfo(EcmaModule module, Logger logger, IReadOnlyDictionary
ms = new UnmanagedMemoryStream(reader.CurrentPointer, length);
}

BlockedResources = SubstitutionsReader.GetSubstitutions(logger, module.Context, ms, resource, module, "resource " + resourceName + " in " + module.ToString(), featureSwitchValues);
BlockedResources = SubstitutionsReader.GetSubstitutions(logger, module.Context, ms, resource, module, "resource " + resourceName + " in " + module.ToString(), featureSwitchValues)[module];
}
}

if (globalBlocks != null && globalBlocks.TryGetValue(module, out IReadOnlySet<string> fromGlobal))
{
var result = new HashSet<string>(fromGlobal);
result.UnionWith(BlockedResources);
BlockedResources = result;
}
}
}

private sealed class SubstitutionsReader : ProcessLinkerXmlBase
public sealed class SubstitutionsReader : ProcessLinkerXmlBase
{
private readonly HashSet<string> _substitutions = new();
private readonly Dictionary<ModuleDesc, HashSet<string>> _substitutions = new();

private SubstitutionsReader(Logger logger, TypeSystemContext context, Stream documentStream, ManifestResource resource, ModuleDesc resourceAssembly, string xmlDocumentLocation, IReadOnlyDictionary<string, bool> featureSwitchValues)
: base(logger, context, documentStream, resource, resourceAssembly, xmlDocumentLocation, featureSwitchValues)
{
_substitutions.Add(resourceAssembly, new HashSet<string>());
}

public static HashSet<string> GetSubstitutions(Logger logger, TypeSystemContext context, Stream documentStream, ManifestResource resource, ModuleDesc resourceAssembly, string xmlDocumentLocation, IReadOnlyDictionary<string, bool> featureSwitchValues)
private SubstitutionsReader(Logger logger, TypeSystemContext context, XmlReader document, string xmlDocumentLocation, IReadOnlyDictionary<string, bool> featureSwitchValues)
: base(logger, context, document, xmlDocumentLocation, featureSwitchValues)
{
}

public static IReadOnlyDictionary<ModuleDesc, IReadOnlySet<string>> GetSubstitutions(Logger logger, TypeSystemContext context, Stream documentStream, ManifestResource resource, ModuleDesc resourceAssembly, string xmlDocumentLocation, IReadOnlyDictionary<string, bool> featureSwitchValues)
{
var rdr = new SubstitutionsReader(logger, context, documentStream, resource, resourceAssembly, xmlDocumentLocation, featureSwitchValues);
rdr.ProcessXml(false);
return rdr._substitutions;
return AsReadOnlyDictionary(rdr._substitutions);
}

public static IReadOnlyDictionary<ModuleDesc, IReadOnlySet<string>> GetSubstitutions(Logger logger, TypeSystemContext context, XmlReader document, string xmlDocumentLocation, IReadOnlyDictionary<string, bool> featureSwitchValues)
{
var rdr = new SubstitutionsReader(logger, context, document, xmlDocumentLocation, featureSwitchValues);
rdr.ProcessXml(false);
return AsReadOnlyDictionary(rdr._substitutions);
}

private void ProcessResources(XPathNavigator nav)
private void ProcessResources(ModuleDesc assembly, XPathNavigator nav)
{
foreach (XPathNavigator resourceNav in nav.SelectChildren("resource", ""))
{
Expand All @@ -139,13 +192,15 @@ private void ProcessResources(XPathNavigator nav)
continue;
}

_substitutions.Add(name);
if (!_substitutions.TryGetValue(assembly, out HashSet<string> removed))
_substitutions.Add(assembly, removed = new HashSet<string>());
removed.Add(name);
}
}

protected override void ProcessAssembly(ModuleDesc assembly, XPathNavigator nav, bool warnOnUnresolvedTypes)
{
ProcessResources(nav);
ProcessResources(assembly, nav);
}

// Should not be resolving types. That's useless work.
Expand Down
Loading

0 comments on commit 076f8c5

Please sign in to comment.