[AMDGPU] Re-Re-apply: Implement vop3p complex pattern optmization for gisel #146984
+676
−81
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
…tmizatio…" This reverts commit 5b8304d.
|
@llvm/pr-subscribers-backend-amdgpu @llvm/pr-subscribers-llvm-globalisel Author: None (Shoreshen) ChangesReverts llvm/llvm-project#146982 Fix up reported building error for #136262 with: Patch is 39.35 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/146984.diff 7 Files Affected:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index b632b16f5c198..fd679a9933cf0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -4327,60 +4327,591 @@ AMDGPUInstructionSelector::selectVOP3NoMods(MachineOperand &Root) const {
}};
}
-std::pair<Register, unsigned>
-AMDGPUInstructionSelector::selectVOP3PModsImpl(
- Register Src, const MachineRegisterInfo &MRI, bool IsDOT) const {
+enum class SrcStatus {
+ IS_SAME,
+ IS_UPPER_HALF,
+ IS_LOWER_HALF,
+ IS_UPPER_HALF_NEG,
+ // This means current op = [op_upper, op_lower] and src = -op_lower.
+ IS_LOWER_HALF_NEG,
+ IS_HI_NEG,
+ // This means current op = [op_upper, op_lower] and src = [op_upper,
+ // -op_lower].
+ IS_LO_NEG,
+ IS_BOTH_NEG,
+ INVALID,
+ NEG_START = IS_UPPER_HALF_NEG,
+ NEG_END = IS_BOTH_NEG,
+ HALF_START = IS_UPPER_HALF,
+ HALF_END = IS_LOWER_HALF_NEG
+};
+/// Test if the MI is truncating to half, such as `%reg0:n = G_TRUNC %reg1:2n`
+static bool isTruncHalf(const MachineInstr *MI,
+ const MachineRegisterInfo &MRI) {
+ if (MI->getOpcode() != AMDGPU::G_TRUNC)
+ return false;
+
+ unsigned DstSize = MRI.getType(MI->getOperand(0).getReg()).getSizeInBits();
+ unsigned SrcSize = MRI.getType(MI->getOperand(1).getReg()).getSizeInBits();
+ return DstSize * 2 == SrcSize;
+}
+
+/// Test if the MI is logic shift right with half bits,
+/// such as `%reg0:2n =G_LSHR %reg1:2n, CONST(n)`
+static bool isLshrHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI) {
+ if (MI->getOpcode() != AMDGPU::G_LSHR)
+ return false;
+
+ Register ShiftSrc;
+ std::optional<ValueAndVReg> ShiftAmt;
+ if (mi_match(MI->getOperand(0).getReg(), MRI,
+ m_GLShr(m_Reg(ShiftSrc), m_GCst(ShiftAmt)))) {
+ unsigned SrcSize = MRI.getType(MI->getOperand(1).getReg()).getSizeInBits();
+ unsigned Shift = ShiftAmt->Value.getZExtValue();
+ return Shift * 2 == SrcSize;
+ }
+ return false;
+}
+
+/// Test if the MI is shift left with half bits,
+/// such as `%reg0:2n =G_SHL %reg1:2n, CONST(n)`
+static bool isShlHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI) {
+ if (MI->getOpcode() != AMDGPU::G_SHL)
+ return false;
+
+ Register ShiftSrc;
+ std::optional<ValueAndVReg> ShiftAmt;
+ if (mi_match(MI->getOperand(0).getReg(), MRI,
+ m_GShl(m_Reg(ShiftSrc), m_GCst(ShiftAmt)))) {
+ unsigned SrcSize = MRI.getType(MI->getOperand(1).getReg()).getSizeInBits();
+ unsigned Shift = ShiftAmt->Value.getZExtValue();
+ return Shift * 2 == SrcSize;
+ }
+ return false;
+}
+
+/// Test function, if the MI is `%reg0:n, %reg1:n = G_UNMERGE_VALUES %reg2:2n`
+static bool isUnmergeHalf(const MachineInstr *MI,
+ const MachineRegisterInfo &MRI) {
+ if (MI->getOpcode() != AMDGPU::G_UNMERGE_VALUES)
+ return false;
+ return MI->getNumOperands() == 3 && MI->getOperand(0).isDef() &&
+ MI->getOperand(1).isDef() && !MI->getOperand(2).isDef();
+}
+
+enum class TypeClass { VECTOR_OF_TWO, SCALAR, NONE_OF_LISTED };
+
+static TypeClass isVectorOfTwoOrScalar(Register Reg,
+ const MachineRegisterInfo &MRI) {
+ LLT OpTy = MRI.getType(Reg);
+ if (OpTy.isScalar())
+ return TypeClass::SCALAR;
+ if (OpTy.isVector() && OpTy.getNumElements() == 2)
+ return TypeClass::VECTOR_OF_TWO;
+ return TypeClass::NONE_OF_LISTED;
+}
+
+static SrcStatus getNegStatus(Register Reg, SrcStatus S,
+ const MachineRegisterInfo &MRI) {
+ TypeClass NegType = isVectorOfTwoOrScalar(Reg, MRI);
+ if (NegType != TypeClass::VECTOR_OF_TWO && NegType != TypeClass::SCALAR)
+ return SrcStatus::INVALID;
+
+ switch (S) {
+ case SrcStatus::IS_SAME:
+ if (NegType == TypeClass::VECTOR_OF_TWO) {
+ // Vector of 2:
+ // [SrcHi, SrcLo] = [CurrHi, CurrLo]
+ // [CurrHi, CurrLo] = neg [OpHi, OpLo](2 x Type)
+ // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
+ // [SrcHi, SrcLo] = [-OpHi, -OpLo]
+ return SrcStatus::IS_BOTH_NEG;
+ }
+ if (NegType == TypeClass::SCALAR) {
+ // Scalar:
+ // [SrcHi, SrcLo] = [CurrHi, CurrLo]
+ // [CurrHi, CurrLo] = neg [OpHi, OpLo](Type)
+ // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
+ // [SrcHi, SrcLo] = [-OpHi, OpLo]
+ return SrcStatus::IS_HI_NEG;
+ }
+ break;
+ case SrcStatus::IS_HI_NEG:
+ if (NegType == TypeClass::VECTOR_OF_TWO) {
+ // Vector of 2:
+ // [SrcHi, SrcLo] = [-CurrHi, CurrLo]
+ // [CurrHi, CurrLo] = neg [OpHi, OpLo](2 x Type)
+ // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
+ // [SrcHi, SrcLo] = [-(-OpHi), -OpLo] = [OpHi, -OpLo]
+ return SrcStatus::IS_LO_NEG;
+ }
+ if (NegType == TypeClass::SCALAR) {
+ // Scalar:
+ // [SrcHi, SrcLo] = [-CurrHi, CurrLo]
+ // [CurrHi, CurrLo] = neg [OpHi, OpLo](Type)
+ // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
+ // [SrcHi, SrcLo] = [-(-OpHi), OpLo] = [OpHi, OpLo]
+ return SrcStatus::IS_SAME;
+ }
+ break;
+ case SrcStatus::IS_LO_NEG:
+ if (NegType == TypeClass::VECTOR_OF_TWO) {
+ // Vector of 2:
+ // [SrcHi, SrcLo] = [CurrHi, -CurrLo]
+ // [CurrHi, CurrLo] = fneg [OpHi, OpLo](2 x Type)
+ // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
+ // [SrcHi, SrcLo] = [-OpHi, -(-OpLo)] = [-OpHi, OpLo]
+ return SrcStatus::IS_HI_NEG;
+ }
+ if (NegType == TypeClass::SCALAR) {
+ // Scalar:
+ // [SrcHi, SrcLo] = [CurrHi, -CurrLo]
+ // [CurrHi, CurrLo] = fneg [OpHi, OpLo](Type)
+ // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
+ // [SrcHi, SrcLo] = [-OpHi, -OpLo]
+ return SrcStatus::IS_BOTH_NEG;
+ }
+ break;
+ case SrcStatus::IS_BOTH_NEG:
+ if (NegType == TypeClass::VECTOR_OF_TWO) {
+ // Vector of 2:
+ // [SrcHi, SrcLo] = [-CurrHi, -CurrLo]
+ // [CurrHi, CurrLo] = fneg [OpHi, OpLo](2 x Type)
+ // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
+ // [SrcHi, SrcLo] = [OpHi, OpLo]
+ return SrcStatus::IS_SAME;
+ }
+ if (NegType == TypeClass::SCALAR) {
+ // Scalar:
+ // [SrcHi, SrcLo] = [-CurrHi, -CurrLo]
+ // [CurrHi, CurrLo] = fneg [OpHi, OpLo](Type)
+ // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
+ // [SrcHi, SrcLo] = [OpHi, -OpLo]
+ return SrcStatus::IS_LO_NEG;
+ }
+ break;
+ case SrcStatus::IS_UPPER_HALF:
+ // Vector of 2:
+ // Src = CurrUpper
+ // Curr = [CurrUpper, CurrLower]
+ // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
+ // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
+ // Src = -OpUpper
+ //
+ // Scalar:
+ // Src = CurrUpper
+ // Curr = [CurrUpper, CurrLower]
+ // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
+ // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
+ // Src = -OpUpper
+ return SrcStatus::IS_UPPER_HALF_NEG;
+ case SrcStatus::IS_LOWER_HALF:
+ if (NegType == TypeClass::VECTOR_OF_TWO) {
+ // Vector of 2:
+ // Src = CurrLower
+ // Curr = [CurrUpper, CurrLower]
+ // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
+ // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
+ // Src = -OpLower
+ return SrcStatus::IS_LOWER_HALF_NEG;
+ }
+ if (NegType == TypeClass::SCALAR) {
+ // Scalar:
+ // Src = CurrLower
+ // Curr = [CurrUpper, CurrLower]
+ // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
+ // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
+ // Src = OpLower
+ return SrcStatus::IS_LOWER_HALF;
+ }
+ break;
+ case SrcStatus::IS_UPPER_HALF_NEG:
+ // Vector of 2:
+ // Src = -CurrUpper
+ // Curr = [CurrUpper, CurrLower]
+ // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
+ // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
+ // Src = -(-OpUpper) = OpUpper
+ //
+ // Scalar:
+ // Src = -CurrUpper
+ // Curr = [CurrUpper, CurrLower]
+ // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
+ // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
+ // Src = -(-OpUpper) = OpUpper
+ return SrcStatus::IS_UPPER_HALF;
+ case SrcStatus::IS_LOWER_HALF_NEG:
+ if (NegType == TypeClass::VECTOR_OF_TWO) {
+ // Vector of 2:
+ // Src = -CurrLower
+ // Curr = [CurrUpper, CurrLower]
+ // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
+ // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
+ // Src = -(-OpLower) = OpLower
+ return SrcStatus::IS_LOWER_HALF;
+ }
+ if (NegType == TypeClass::SCALAR) {
+ // Scalar:
+ // Src = -CurrLower
+ // Curr = [CurrUpper, CurrLower]
+ // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
+ // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
+ // Src = -OpLower
+ return SrcStatus::IS_LOWER_HALF_NEG;
+ }
+ break;
+ default:
+ llvm_unreachable("unexpected SrcStatus");
+ }
+}
+
+static std::optional<std::pair<Register, SrcStatus>>
+calcNextStatus(std::pair<Register, SrcStatus> Curr,
+ const MachineRegisterInfo &MRI) {
+ const MachineInstr *MI = MRI.getVRegDef(Curr.first);
+
+ unsigned Opc = MI->getOpcode();
+
+ // Handle general Opc cases.
+ switch (Opc) {
+ case AMDGPU::G_BITCAST:
+ return std::optional<std::pair<Register, SrcStatus>>(
+ {MI->getOperand(1).getReg(), Curr.second});
+ case AMDGPU::COPY:
+ if (MI->getOperand(1).getReg().isPhysical())
+ return std::nullopt;
+ return std::optional<std::pair<Register, SrcStatus>>(
+ {MI->getOperand(1).getReg(), Curr.second});
+ case AMDGPU::G_FNEG: {
+ SrcStatus Stat = getNegStatus(Curr.first, Curr.second, MRI);
+ if (Stat == SrcStatus::INVALID)
+ return std::nullopt;
+ return std::optional<std::pair<Register, SrcStatus>>(
+ {MI->getOperand(1).getReg(), Stat});
+ }
+ default:
+ break;
+ }
+
+ // Calc next Stat from current Stat.
+ switch (Curr.second) {
+ case SrcStatus::IS_SAME:
+ if (isTruncHalf(MI, MRI))
+ return std::optional<std::pair<Register, SrcStatus>>(
+ {MI->getOperand(1).getReg(), SrcStatus::IS_LOWER_HALF});
+ else if (isUnmergeHalf(MI, MRI)) {
+ if (Curr.first == MI->getOperand(0).getReg())
+ return std::optional<std::pair<Register, SrcStatus>>(
+ {MI->getOperand(2).getReg(), SrcStatus::IS_LOWER_HALF});
+ return std::optional<std::pair<Register, SrcStatus>>(
+ {MI->getOperand(2).getReg(), SrcStatus::IS_UPPER_HALF});
+ }
+ break;
+ case SrcStatus::IS_HI_NEG:
+ if (isTruncHalf(MI, MRI)) {
+ // [SrcHi, SrcLo] = [-CurrHi, CurrLo]
+ // [CurrHi, CurrLo] = trunc [OpUpper, OpLower] = OpLower
+ // = [OpLowerHi, OpLowerLo]
+ // Src = [SrcHi, SrcLo] = [-CurrHi, CurrLo]
+ // = [-OpLowerHi, OpLowerLo]
+ // = -OpLower
+ return std::optional<std::pair<Register, SrcStatus>>(
+ {MI->getOperand(1).getReg(), SrcStatus::IS_LOWER_HALF_NEG});
+ }
+ if (isUnmergeHalf(MI, MRI)) {
+ if (Curr.first == MI->getOperand(0).getReg())
+ return std::optional<std::pair<Register, SrcStatus>>(
+ {MI->getOperand(2).getReg(), SrcStatus::IS_LOWER_HALF_NEG});
+ return std::optional<std::pair<Register, SrcStatus>>(
+ {MI->getOperand(2).getReg(), SrcStatus::IS_UPPER_HALF_NEG});
+ }
+ break;
+ case SrcStatus::IS_UPPER_HALF:
+ if (isShlHalf(MI, MRI))
+ return std::optional<std::pair<Register, SrcStatus>>(
+ {MI->getOperand(1).getReg(), SrcStatus::IS_LOWER_HALF});
+ break;
+ case SrcStatus::IS_LOWER_HALF:
+ if (isLshrHalf(MI, MRI))
+ return std::optional<std::pair<Register, SrcStatus>>(
+ {MI->getOperand(1).getReg(), SrcStatus::IS_UPPER_HALF});
+ break;
+ case SrcStatus::IS_UPPER_HALF_NEG:
+ if (isShlHalf(MI, MRI))
+ return std::optional<std::pair<Register, SrcStatus>>(
+ {MI->getOperand(1).getReg(), SrcStatus::IS_LOWER_HALF_NEG});
+ break;
+ case SrcStatus::IS_LOWER_HALF_NEG:
+ if (isLshrHalf(MI, MRI))
+ return std::optional<std::pair<Register, SrcStatus>>(
+ {MI->getOperand(1).getReg(), SrcStatus::IS_UPPER_HALF_NEG});
+ break;
+ default:
+ break;
+ }
+ return std::nullopt;
+}
+
+/// This is used to control valid status that current MI supports. For example,
+/// non floating point intrinsic such as @llvm.amdgcn.sdot2 does not support NEG
+/// bit on VOP3P.
+/// The class can be further extended to recognize support on SEL, NEG, ABS bit
+/// for different MI on different arch
+class SearchOptions {
+private:
+ bool HasNeg = false;
+ // Assume all complex pattern of VOP3P have opsel.
+ bool HasOpsel = true;
+
+public:
+ SearchOptions(Register Reg, const MachineRegisterInfo &MRI) {
+ const MachineInstr *MI = MRI.getVRegDef(Reg);
+ unsigned Opc = MI->getOpcode();
+
+ if (Opc < TargetOpcode::GENERIC_OP_END) {
+ // Keep same for generic op.
+ HasNeg = true;
+ } else if (Opc == TargetOpcode::G_INTRINSIC) {
+ Intrinsic::ID IntrinsicID = cast<GIntrinsic>(*MI).getIntrinsicID();
+ // Only float point intrinsic has neg & neg_hi bits.
+ if (IntrinsicID == Intrinsic::amdgcn_fdot2)
+ HasNeg = true;
+ }
+ }
+ bool checkOptions(SrcStatus Stat) const {
+ if (!HasNeg &&
+ (Stat >= SrcStatus::NEG_START && Stat <= SrcStatus::NEG_END)) {
+ return false;
+ }
+ if (!HasOpsel &&
+ (Stat >= SrcStatus::HALF_START && Stat <= SrcStatus::HALF_END)) {
+ return false;
+ }
+ return true;
+ }
+};
+
+static SmallVector<std::pair<Register, SrcStatus>>
+getSrcStats(Register Reg, const MachineRegisterInfo &MRI, SearchOptions SO,
+ int MaxDepth = 3) {
+ int Depth = 0;
+ auto Curr = calcNextStatus({Reg, SrcStatus::IS_SAME}, MRI);
+ SmallVector<std::pair<Register, SrcStatus>> Statlist;
+
+ while (Depth <= MaxDepth && Curr.has_value()) {
+ Depth++;
+ if (SO.checkOptions(Curr.value().second))
+ Statlist.push_back(Curr.value());
+ Curr = calcNextStatus(Curr.value(), MRI);
+ }
+
+ return Statlist;
+}
+
+static std::pair<Register, SrcStatus>
+getLastSameOrNeg(Register Reg, const MachineRegisterInfo &MRI, SearchOptions SO,
+ int MaxDepth = 3) {
+ int Depth = 0;
+ std::pair<Register, SrcStatus> LastSameOrNeg = {Reg, SrcStatus::IS_SAME};
+ auto Curr = calcNextStatus(LastSameOrNeg, MRI);
+
+ while (Depth <= MaxDepth && Curr.has_value()) {
+ Depth++;
+ SrcStatus Stat = Curr.value().second;
+ if (SO.checkOptions(Stat)) {
+ if (Stat == SrcStatus::IS_SAME || Stat == SrcStatus::IS_HI_NEG ||
+ Stat == SrcStatus::IS_LO_NEG || Stat == SrcStatus::IS_BOTH_NEG)
+ LastSameOrNeg = Curr.value();
+ }
+ Curr = calcNextStatus(Curr.value(), MRI);
+ }
+
+ return LastSameOrNeg;
+}
+
+static bool isSameBitWidth(Register Reg1, Register Reg2,
+ const MachineRegisterInfo &MRI) {
+ unsigned Width1 = MRI.getType(Reg1).getSizeInBits();
+ unsigned Width2 = MRI.getType(Reg2).getSizeInBits();
+ return Width1 == Width2;
+}
+
+static unsigned updateMods(SrcStatus HiStat, SrcStatus LoStat, unsigned Mods) {
+ // SrcStatus::IS_LOWER_HALF remain 0.
+ if (HiStat == SrcStatus::IS_UPPER_HALF_NEG) {
+ Mods ^= SISrcMods::NEG_HI;
+ Mods |= SISrcMods::OP_SEL_1;
+ } else if (HiStat == SrcStatus::IS_UPPER_HALF)
+ Mods |= SISrcMods::OP_SEL_1;
+ else if (HiStat == SrcStatus::IS_LOWER_HALF_NEG)
+ Mods ^= SISrcMods::NEG_HI;
+ else if (HiStat == SrcStatus::IS_HI_NEG)
+ Mods ^= SISrcMods::NEG_HI;
+
+ if (LoStat == SrcStatus::IS_UPPER_HALF_NEG) {
+ Mods ^= SISrcMods::NEG;
+ Mods |= SISrcMods::OP_SEL_0;
+ } else if (LoStat == SrcStatus::IS_UPPER_HALF)
+ Mods |= SISrcMods::OP_SEL_0;
+ else if (LoStat == SrcStatus::IS_LOWER_HALF_NEG)
+ Mods |= SISrcMods::NEG;
+ else if (LoStat == SrcStatus::IS_HI_NEG)
+ Mods ^= SISrcMods::NEG;
+
+ return Mods;
+}
+
+static bool isValidToPack(SrcStatus HiStat, SrcStatus LoStat, Register NewReg,
+ Register RootReg, const SIInstrInfo &TII,
+ const MachineRegisterInfo &MRI) {
+ auto IsHalfState = [](SrcStatus S) {
+ return S == SrcStatus::IS_UPPER_HALF || S == SrcStatus::IS_UPPER_HALF_NEG ||
+ S == SrcStatus::IS_LOWER_HALF || S == SrcStatus::IS_LOWER_HALF_NEG;
+ };
+ return isSameBitWidth(NewReg, RootReg, MRI) && IsHalfState(LoStat) &&
+ IsHalfState(HiStat);
+}
+
+std::pair<Register, unsigned> AMDGPUInstructionSelector::selectVOP3PModsImpl(
+ Register RootReg, const MachineRegisterInfo &MRI, bool IsDOT) const {
unsigned Mods = 0;
- MachineInstr *MI = MRI.getVRegDef(Src);
+ // No modification if Root type is not form of <2 x Type>.
+ if (isVectorOfTwoOrScalar(RootReg, MRI) != TypeClass::VECTOR_OF_TWO) {
+ Mods |= SISrcMods::OP_SEL_1;
+ return {RootReg, Mods};
+ }
+
+ SearchOptions SO(RootReg, MRI);
+
+ std::pair<Register, SrcStatus> Stat = getLastSameOrNeg(RootReg, MRI, SO);
- if (MI->getOpcode() == AMDGPU::G_FNEG &&
- // It's possible to see an f32 fneg here, but unlikely.
- // TODO: Treat f32 fneg as only high bit.
- MRI.getType(Src) == LLT::fixed_vector(2, 16)) {
+ if (Stat.second == SrcStatus::IS_BOTH_NEG)
Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI);
- Src = MI->getOperand(1).getReg();
- MI = MRI.getVRegDef(Src);
+ else if (Stat.second == SrcStatus::IS_HI_NEG)
+ Mods ^= SISrcMods::NEG_HI;
+ else if (Stat.second == SrcStatus::IS_LO_NEG)
+ Mods ^= SISrcMods::NEG;
+
+ MachineInstr *MI = MRI.getVRegDef(Stat.first);
+
+ if (MI->getOpcode() != AMDGPU::G_BUILD_VECTOR || MI->getNumOperands() != 3 ||
+ (IsDOT && Subtarget->hasDOTOpSelHazard())) {
+ Mods |= SISrcMods::OP_SEL_1;
+ return {Stat.first, Mods};
}
- // TODO: Handle G_FSUB 0 as fneg
+ SmallVector<std::pair<Register, SrcStatus>> StatlistHi =
+ getSrcStats(MI->getOperand(2).getReg(), MRI, SO);
- // TODO: Match op_sel through g_build_vector_trunc and g_shuffle_vector.
- (void)IsDOT; // DOTs do not use OPSEL on gfx942+, check ST.hasDOTOpSelHazard()
+ if (StatlistHi.empty()) {
+ Mods |= SISrcMods::OP_SEL_1;
+ return {Stat.first, Mods};
+ }
+ SmallVector<std::pair<Register, SrcStatus>> StatlistLo =
+ getSrcStats(MI->getOperand(1).getReg(), MRI, SO);
+
+ if (StatlistLo.empty()) {
+ Mods |= SISrcMods::OP_SEL_1;
+ return {Stat.first, Mods};
+ }
+
+ for (int I = StatlistHi.size() - 1; I >= 0; I--) {
+ for (int J = StatlistLo.size() - 1; J >= 0; J--) {
+ if (StatlistHi[I].first == StatlistLo[J].first &&
+ isValidToPack(StatlistHi[I].second, StatlistLo[J].second,
+ StatlistHi[I].first, RootReg, TII, MRI))
+ return {StatlistHi[I].first,
+ updateMods(StatlistHi[I].second, StatlistLo[J].second, Mods)};
+ }
+ }
// Packed instructions do not have abs modifiers.
Mods |= SISrcMods::OP_SEL_1;
- return std::pair(Src, Mods);
+ return {Stat.first, Mods};
}
-InstructionSelector::ComplexRendererFns
-AMDGPUInstructionSelector::selectVOP3PMods(MachineOperand &Root) const {
- MachineRegisterInfo &MRI
- = Root.getParent()->getParent()->getParent()->getRegInfo();
+int64_t getAllKindImm(const MachineOperand *Op) {
+ switch (Op->getType()) {
+ case MachineOperand::MachineOperandType::MO_Immediate:
+ return Op->getImm();
+ case MachineOperand::MachineOperandType::MO_CImmediate:
+ return Op->getCImm()->getSExtValue();
+ case MachineOperand::MachineOperandType::MO_FPImmediate:
+ return Op->getFPImm()->getValueAPF().bitcastToAPInt().getSExtValue();
+ default:
+ llvm_unreachable("not an imm type");
+ }
+}
- Register Src;
+static bool checkRB(Register Reg, unsigned int RBNo,
+ const AMDGPURegisterBankInfo &RBI,
+ const MachineRegisterInfo &MRI,
+ const TargetRegisterInfo &TRI) {
+ const RegisterBank *RB = RBI.getRegBank(Reg, MRI, TRI);
+ return RB->getID() == RBNo;
+}
+
+// This function is used to get the correct register bank for returned reg.
+// Assume:
+// 1. VOP3P is always legal for VGPR.
+// 2. RootOp's regbank is legal.
+// Thus
+// 1. If RootOp is SGPR, then NewOp can be SGPR or VGPR.
+// 2. If RootOp is VGPR, then NewOp mu...
[truncated]
|
Shoreshen
requested review from
jayfoad,
arsenm,
jmmartinez,
shiltian,
rovka and
cdevadas
There was a problem hiding this comment.
Pull Request Overview
Re-applies the VOP3P complex pattern optimization for the AMDGPU backend (Gisel) and fixes a build error in AMDGPUInstructionSelector.cpp by ensuring all control paths return a value.
- Restore and extend
selectVOP3PModsImplwith detailed neg/op_sel tracking and packing logic - Introduce helper routines (
getSrcStats,calcNextStatus,getLegalRegBank, etc.) to support the new pattern matching - Update tests to validate the new
neg_lo,neg_hi, andop_selmodifiers and switchread().decode("ascii")to UTF-8
Reviewed Changes
Copilot reviewed 8 out of 8 changed files in this pull request and generated 2 comments.
Show a summary per file
| File | Description |
|---|---|
| llvm/test/lit.cfg.py | Switched decode("ascii") to decode("utf-8") |
| llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll | Updated expected GISEL patterns to use neg_lo/neg_hi |
| llvm/test/CodeGen/AMDGPU/packed-fp32.ll | Consolidated XOR+PACK sequences into neg_lo/neg_hi |
| llvm/test/CodeGen/AMDGPU/GlobalISel/*.ll | Added op_sel/op_sel_hi modifiers in expected outputs |
| llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h | Renamed parameters and declared selectVOP3PRetHelper |
| llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp | Fully rewrote selectVOP3PModsImpl, added helper funcs |
| IsHalfState(HiStat); | ||
| } | ||
|
|
||
| std::pair<Register, unsigned> AMDGPUInstructionSelector::selectVOP3PModsImpl( |
There was a problem hiding this comment.
[nitpick] selectVOP3PModsImpl has grown very large and contains deeply nested logic; consider breaking it into smaller, well-named helper methods to improve readability and ease future maintenance.
Copilot uses AI. Check for mistakes.
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Reverts #146982
Fix up reported building error for #136262 with: