Skip to content

[FMV] Allow mixing target_version with target_clones. #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

[FMV] Allow mixing target_version with target_clones. #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6a49501
[FMV] Emit the resolver along with the default version definition.
labrinea Mar 7, 2024
63585c5
[FMV] Allow multi versioning without default declaration.
labrinea Mar 15, 2024
cb4f5f9
[FMV] Allow mixing target_version with target_clones.
labrinea Mar 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions clang/include/clang/Basic/Attr.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3080,6 +3080,20 @@ def TargetClones : InheritableAttr {
StringRef getFeatureStr(unsigned Index) const {
return *(featuresStrs_begin() + Index);
}
bool isDefaultVersion(unsigned Index) const {
return getFeatureStr(Index) == "default";
}
void getFeatures(llvm::SmallVectorImpl<StringRef> &Out,
unsigned Index) const {
if (isDefaultVersion(Index)) return;
StringRef Features = getFeatureStr(Index);
SmallVector<StringRef, 8> AttrFeatures;
Features.split(AttrFeatures, "+");
for (auto &Feature : AttrFeatures) {
Feature = Feature.trim();
Out.push_back(Feature);
}
}
// Given an index into the 'featuresStrs' sequence, compute a unique
// ID to be used with function name mangling for the associated variant.
// This mapping is necessary due to a requirement that the mangling ID
Expand Down
15 changes: 6 additions & 9 deletions clang/lib/AST/ASTContext.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13621,22 +13621,19 @@ void ASTContext::getFunctionFeatureMap(llvm::StringMap<bool> &FeatureMap,
Target->initFeatureMap(FeatureMap, getDiagnostics(), TargetCPU, Features);
} else if (const auto *TC = FD->getAttr<TargetClonesAttr>()) {
std::vector<std::string> Features;
StringRef VersionStr = TC->getFeatureStr(GD.getMultiVersionIndex());
if (Target->getTriple().isAArch64()) {
// TargetClones for AArch64
if (VersionStr != "default") {
SmallVector<StringRef, 1> VersionFeatures;
VersionStr.split(VersionFeatures, "+");
for (auto &VFeature : VersionFeatures) {
VFeature = VFeature.trim();
llvm::SmallVector<StringRef, 8> Feats;
TC->getFeatures(Feats, GD.getMultiVersionIndex());
for (StringRef Feat : Feats)
if (Target->validateCpuSupports(Feat.str()))
// Use '?' to mark features that came from AArch64 TargetClones.
Features.push_back((StringRef{"?"} + VFeature).str());
}
}
Features.push_back("?" + Feat.str());
Features.insert(Features.begin(),
Target->getTargetOpts().FeaturesAsWritten.begin(),
Target->getTargetOpts().FeaturesAsWritten.end());
} else {
StringRef VersionStr = TC->getFeatureStr(GD.getMultiVersionIndex());
if (VersionStr.starts_with("arch="))
TargetCPU = VersionStr.drop_front(sizeof("arch=") - 1);
else if (VersionStr != "default")
Expand Down
214 changes: 122 additions & 92 deletions clang/lib/CodeGen/CodeGenModule.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3447,6 +3447,9 @@ bool CodeGenModule::MayBeEmittedEagerly(const ValueDecl *Global) {
// Implicit template instantiations may change linkage if they are later
// explicitly instantiated, so they should not be emitted eagerly.
return false;
// Defer until all versions have been semantically checked.
if (FD->hasAttr<TargetVersionAttr>() && !FD->isMultiVersion())
return false;
}
if (const auto *VD = dyn_cast<VarDecl>(Global)) {
if (Context.getInlineVariableDefinitionKind(VD) ==
Expand Down Expand Up @@ -3988,17 +3991,21 @@ void CodeGenModule::EmitMultiVersionFunctionDefinition(GlobalDecl GD,
auto *Spec = FD->getAttr<CPUSpecificAttr>();
for (unsigned I = 0; I < Spec->cpus_size(); ++I)
EmitGlobalFunctionDefinition(GD.getWithMultiVersionIndex(I), nullptr);
} else if (FD->isTargetClonesMultiVersion()) {
auto *Clone = FD->getAttr<TargetClonesAttr>();
for (unsigned I = 0; I < Clone->featuresStrs_size(); ++I)
if (Clone->isFirstOfVersion(I))
} else if (auto *TC = FD->getAttr<TargetClonesAttr>()) {
for (unsigned I = 0; I < TC->featuresStrs_size(); ++I)
// AArch64 favors the default target version over the clone if any.
if ((!TC->isDefaultVersion(I) || !getTarget().getTriple().isAArch64()) &&
TC->isFirstOfVersion(I))
EmitGlobalFunctionDefinition(GD.getWithMultiVersionIndex(I), nullptr);
// Ensure that the resolver function is also emitted.
GetOrCreateMultiVersionResolver(GD);
} else if (FD->hasAttr<TargetVersionAttr>()) {
GetOrCreateMultiVersionResolver(GD);
} else
EmitGlobalFunctionDefinition(GD, GV);

// Defer the resolver emission until we can reason whether the TU
// contains a default target version implementation.
if (FD->isTargetVersionMultiVersion())
AddDeferredMultiVersionResolverToEmit(GD);
}

void CodeGenModule::EmitGlobalDefinition(GlobalDecl GD, llvm::GlobalValue *GV) {
Expand Down Expand Up @@ -4084,113 +4091,122 @@ llvm::GlobalValue::LinkageTypes getMultiversionLinkage(CodeGenModule &CGM,
return llvm::GlobalValue::WeakODRLinkage;
}

static FunctionDecl *createDefaultTargetVersionFrom(const FunctionDecl *FD) {
DeclContext *DeclCtx = FD->getASTContext().getTranslationUnitDecl();
TypeSourceInfo *TInfo = FD->getTypeSourceInfo();
StorageClass SC = FD->getStorageClass();
DeclarationName Name = FD->getNameInfo().getName();

FunctionDecl *NewDecl =
FunctionDecl::Create(FD->getASTContext(), DeclCtx, FD->getBeginLoc(),
FD->getEndLoc(), Name, TInfo->getType(), TInfo, SC);

NewDecl->setIsMultiVersion();
NewDecl->addAttr(TargetVersionAttr::CreateImplicit(
NewDecl->getASTContext(), "default", NewDecl->getSourceRange()));

return NewDecl;
}

void CodeGenModule::emitMultiVersionFunctions() {
std::vector<GlobalDecl> MVFuncsToEmit;
MultiVersionFuncs.swap(MVFuncsToEmit);
for (GlobalDecl GD : MVFuncsToEmit) {
const auto *FD = cast<FunctionDecl>(GD.getDecl());
assert(FD && "Expected a FunctionDecl");

auto createFunction = [&](const FunctionDecl *Decl, unsigned MVIdx = 0) {
GlobalDecl CurGD{Decl->isDefined() ? Decl->getDefinition() : Decl, MVIdx};
StringRef MangledName = getMangledName(CurGD);
llvm::Constant *Func = GetGlobalValue(MangledName);
if (!Func) {
if (Decl->isDefined()) {
EmitGlobalFunctionDefinition(CurGD, nullptr);
Func = GetGlobalValue(MangledName);
} else {
const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(CurGD);
llvm::FunctionType *Ty = getTypes().GetFunctionType(FI);
Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false,
/*DontDefer=*/false, ForDefinition);
}
assert(Func && "This should have just been created");
}
return Func;
};

bool HasDefaultDecl = !FD->isTargetVersionMultiVersion();
bool ShouldEmitResolver = !FD->isTargetVersionMultiVersion();
SmallVector<CodeGenFunction::MultiVersionResolverOption, 10> Options;
if (FD->isTargetMultiVersion()) {
getContext().forEachMultiversionedFunctionVersion(
FD, [this, &GD, &Options](const FunctionDecl *CurFD) {
GlobalDecl CurGD{
(CurFD->isDefined() ? CurFD->getDefinition() : CurFD)};
StringRef MangledName = getMangledName(CurGD);
llvm::Constant *Func = GetGlobalValue(MangledName);
if (!Func) {
if (CurFD->isDefined()) {
EmitGlobalFunctionDefinition(CurGD, nullptr);
Func = GetGlobalValue(MangledName);
} else {
const CGFunctionInfo &FI =
getTypes().arrangeGlobalDeclaration(GD);
llvm::FunctionType *Ty = getTypes().GetFunctionType(FI);
Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false,
/*DontDefer=*/false, ForDefinition);
getContext().forEachMultiversionedFunctionVersion(
FD, [&](const FunctionDecl *CurFD) {
llvm::SmallVector<StringRef, 8> Feats;
llvm::Constant *Func = createFunction(CurFD);

if (const auto *TA = CurFD->getAttr<TargetAttr>()) {
TA->getAddedFeatures(Feats);
Options.emplace_back(cast<llvm::Function>(Func),
TA->getArchitecture(), Feats);
} else if (const auto *TVA = CurFD->getAttr<TargetVersionAttr>()) {
bool HasDefaultDef = TVA->isDefaultVersion() &&
CurFD->doesThisDeclarationHaveABody();
HasDefaultDecl |= TVA->isDefaultVersion();
ShouldEmitResolver |= (CurFD->isUsed() || HasDefaultDef);
TVA->getFeatures(Feats);
Options.emplace_back(cast<llvm::Function>(Func),
/*Architecture*/ "", Feats);
} else if (const auto *TC = CurFD->getAttr<TargetClonesAttr>()) {
for (unsigned I = 0; I < TC->featuresStrs_size(); ++I) {
if (!TC->isFirstOfVersion(I))
continue;

llvm::Constant *Func = createFunction(CurFD, I);
StringRef Architecture;
Feats.clear();
if (getTarget().getTriple().isAArch64())
TC->getFeatures(Feats, I);
else {
StringRef Version = TC->getFeatureStr(I);
if (Version.starts_with("arch="))
Architecture = Version.drop_front(sizeof("arch=") - 1);
else if (Version != "default")
Feats.push_back(Version);
}
assert(Func && "This should have just been created");
}
if (CurFD->getMultiVersionKind() == MultiVersionKind::Target) {
const auto *TA = CurFD->getAttr<TargetAttr>();
llvm::SmallVector<StringRef, 8> Feats;
TA->getAddedFeatures(Feats);
Options.emplace_back(cast<llvm::Function>(Func),
TA->getArchitecture(), Feats);
} else {
const auto *TVA = CurFD->getAttr<TargetVersionAttr>();
llvm::SmallVector<StringRef, 8> Feats;
TVA->getFeatures(Feats);
Options.emplace_back(cast<llvm::Function>(Func),
/*Architecture*/ "", Feats);
Architecture, Feats);
}
});
} else if (FD->isTargetClonesMultiVersion()) {
const auto *TC = FD->getAttr<TargetClonesAttr>();
for (unsigned VersionIndex = 0; VersionIndex < TC->featuresStrs_size();
++VersionIndex) {
if (!TC->isFirstOfVersion(VersionIndex))
continue;
GlobalDecl CurGD{(FD->isDefined() ? FD->getDefinition() : FD),
VersionIndex};
StringRef Version = TC->getFeatureStr(VersionIndex);
StringRef MangledName = getMangledName(CurGD);
llvm::Constant *Func = GetGlobalValue(MangledName);
if (!Func) {
if (FD->isDefined()) {
EmitGlobalFunctionDefinition(CurGD, nullptr);
Func = GetGlobalValue(MangledName);
} else {
const CGFunctionInfo &FI =
getTypes().arrangeGlobalDeclaration(CurGD);
llvm::FunctionType *Ty = getTypes().GetFunctionType(FI);
Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false,
/*DontDefer=*/false, ForDefinition);
}
assert(Func && "This should have just been created");
}

StringRef Architecture;
llvm::SmallVector<StringRef, 1> Feature;

if (getTarget().getTriple().isAArch64()) {
if (Version != "default") {
llvm::SmallVector<StringRef, 8> VerFeats;
Version.split(VerFeats, "+");
for (auto &CurFeat : VerFeats)
Feature.push_back(CurFeat.trim());
}
} else {
if (Version.starts_with("arch="))
Architecture = Version.drop_front(sizeof("arch=") - 1);
else if (Version != "default")
Feature.push_back(Version);
}
ShouldEmitResolver = true;
} else
llvm_unreachable("unexpected MultiVersionKind");
});

Options.emplace_back(cast<llvm::Function>(Func), Architecture, Feature);
}
} else {
assert(0 && "Expected a target or target_clones multiversion function");
if (!ShouldEmitResolver)
continue;

if (!HasDefaultDecl) {
FunctionDecl *NewFD = createDefaultTargetVersionFrom(FD);
llvm::Constant *Func = createFunction(NewFD);
llvm::SmallVector<StringRef, 1> Feats;
Options.emplace_back(cast<llvm::Function>(Func), "", Feats);
}

llvm::Constant *ResolverConstant = GetOrCreateMultiVersionResolver(GD);
if (auto *IFunc = dyn_cast<llvm::GlobalIFunc>(ResolverConstant)) {
ResolverConstant = IFunc->getResolver();
if (FD->isTargetClonesMultiVersion() ||
FD->isTargetVersionMultiVersion()) {
const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
llvm::FunctionType *DeclTy = getTypes().GetFunctionType(FI);
std::string MangledName = getMangledNameImpl(
*this, GD, FD, /*OmitMultiVersionMangling=*/true);
// In prior versions of Clang, the mangling for ifuncs incorrectly
// included an .ifunc suffix. This alias is generated for backward
// compatibility. It is deprecated, and may be removed in the future.
auto *Alias = llvm::GlobalAlias::create(
DeclTy, 0, getMultiversionLinkage(*this, GD),
MangledName + ".ifunc", IFunc, &getModule());
SetCommonAttributes(FD, Alias);
if (!GetGlobalValue(MangledName + ".ifunc")) {
const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
llvm::FunctionType *DeclTy = getTypes().GetFunctionType(FI);
// In prior versions of Clang, the mangling for ifuncs incorrectly
// included an .ifunc suffix. This alias is generated for backward
// compatibility. It is deprecated, and may be removed in the future.
auto *Alias = llvm::GlobalAlias::create(
DeclTy, 0, getMultiversionLinkage(*this, GD),
MangledName + ".ifunc", IFunc, &getModule());
SetCommonAttributes(FD, Alias);
}
}
}
llvm::Function *ResolverFunc = cast<llvm::Function>(ResolverConstant);
Expand Down Expand Up @@ -4347,6 +4363,20 @@ void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) {
}
}

/// Adds a declaration to the list of multi version functions if not present.
void CodeGenModule::AddDeferredMultiVersionResolverToEmit(GlobalDecl GD) {
const auto *FD = cast<FunctionDecl>(GD.getDecl());
assert(FD && "Not a FunctionDecl?");

if (FD->isTargetVersionMultiVersion() || FD->isTargetClonesMultiVersion()) {
std::string MangledName =
getMangledNameImpl(*this, GD, FD, /*OmitMultiVersionMangling=*/true);
if (!DeferredResolversToEmit.insert(MangledName).second)
return;
}
MultiVersionFuncs.push_back(GD);
}

/// If a dispatcher for the specified mangled name is not in the module, create
/// and return an llvm Function with the specified type.
llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver(GlobalDecl GD) {
Expand Down Expand Up @@ -4386,7 +4416,7 @@ llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver(GlobalDecl GD) {
// The resolver needs to be created. For target and target_clones, defer
// creation until the end of the TU.
if (FD->isTargetMultiVersion() || FD->isTargetClonesMultiVersion())
MultiVersionFuncs.push_back(GD);
AddDeferredMultiVersionResolverToEmit(GD);

// For cpu_specific, don't create an ifunc yet because we don't know if the
// cpu_dispatch will be emitted in this translation unit.
Expand Down
5 changes: 5 additions & 0 deletions clang/lib/CodeGen/CodeGenModule.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -348,6 +348,8 @@ class CodeGenModule : public CodeGenTypeCache {
/// yet.
llvm::DenseMap<StringRef, GlobalDecl> DeferredDecls;

llvm::StringSet<llvm::BumpPtrAllocator> DeferredResolversToEmit;

/// This is a list of deferred decls which we have seen that *are* actually
/// referenced. These get code generated when the module is done.
std::vector<GlobalDecl> DeferredDeclsToEmit;
Expand Down Expand Up @@ -1588,6 +1590,9 @@ class CodeGenModule : public CodeGenTypeCache {
llvm::AttributeList ExtraAttrs = llvm::AttributeList(),
ForDefinition_t IsForDefinition = NotForDefinition);

// Adds a declaration to the list of multi version functions if not present.
void AddDeferredMultiVersionResolverToEmit(GlobalDecl GD);

// References to multiversion functions are resolved through an implicitly
// defined resolver function. This function is responsible for creating
// the resolver symbol for the provided declaration. The value returned
Expand Down
Loading