Skip to content

[AMDGPU] Merge two V_CNDMASK instructions into V_DUAL_CNDMASK #135007

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed

[AMDGPU] Merge two V_CNDMASK instructions into V_DUAL_CNDMASK #135007

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
51b12c3
precommit
mihajlovicana Apr 9, 2025
b2f1080
[AMDGPU] Merge V_CNDMASKS into V_DUAL_CNDMASK
mihajlovicana Apr 9, 2025
e7e3462
cover case for single v_cndmask to reduce code size
mihajlovicana Apr 9, 2025
3d1ae88
added float instructions
mihajlovicana Apr 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 108 additions & 4 deletions llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,9 +105,41 @@ class SIFoldOperandsImpl {
}
}

unsigned getInverseCompareOpcode(MachineInstr &MI) const {
switch (MI.getOpcode()) {
// unsigned 32
case AMDGPU::V_CMP_EQ_U32_e64:
return AMDGPU::V_CMP_NE_U32_e64;
case AMDGPU::V_CMP_NE_U32_e64:
return AMDGPU::V_CMP_EQ_U32_e64;
case AMDGPU::V_CMP_GE_U32_e64:
return AMDGPU::V_CMP_LT_U32_e64;
case AMDGPU::V_CMP_LE_U32_e64:
return AMDGPU::V_CMP_GT_U32_e64;
case AMDGPU::V_CMP_GT_U32_e64:
return AMDGPU::V_CMP_LE_U32_e64;
case AMDGPU::V_CMP_LT_U32_e64:
return AMDGPU::V_CMP_GE_U32_e64;
// float 32
case AMDGPU::V_CMP_EQ_F32_e64:
return AMDGPU::V_CMP_NEQ_F32_e64;
case AMDGPU::V_CMP_NEQ_F32_e64:
return AMDGPU::V_CMP_EQ_F32_e64;
case AMDGPU::V_CMP_GE_F32_e64:
return AMDGPU::V_CMP_LT_F32_e64;
case AMDGPU::V_CMP_LE_F32_e64:
return AMDGPU::V_CMP_GT_F32_e64;
case AMDGPU::V_CMP_GT_F32_e64:
return AMDGPU::V_CMP_LE_F32_e64;
case AMDGPU::V_CMP_LT_F32_e64:
return AMDGPU::V_CMP_GE_F32_e64;
default:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should also handle floating point comparisons?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes

return 0;
}
}

bool foldCopyToVGPROfScalarAddOfFrameIndex(Register DstReg, Register SrcReg,
MachineInstr &MI) const;

bool updateOperand(FoldCandidate &Fold) const;

bool canUseImmWithOpSel(FoldCandidate &Fold) const;
Expand All @@ -133,7 +165,8 @@ class SIFoldOperandsImpl {

std::optional<int64_t> getImmOrMaterializedImm(MachineOperand &Op) const;
bool tryConstantFoldOp(MachineInstr *MI) const;
bool tryFoldCndMask(MachineInstr &MI) const;
bool tryFoldCndMask(MachineInstr &MI, Register *RegVCC,
Register *newVCC) const;
bool tryFoldZeroHighBits(MachineInstr &MI) const;
bool foldInstOperand(MachineInstr &MI, MachineOperand &OpToFold) const;

Expand All @@ -152,6 +185,9 @@ class SIFoldOperandsImpl {

bool tryOptimizeAGPRPhis(MachineBasicBlock &MBB);

bool shouldSwitchOperands(MachineRegisterInfo &MRI, MachineInstr &MI,
const SIInstrInfo &TII) const;

public:
SIFoldOperandsImpl() = default;

Expand Down Expand Up @@ -1459,13 +1495,79 @@ bool SIFoldOperandsImpl::tryConstantFoldOp(MachineInstr *MI) const {
return false;
}

bool SIFoldOperandsImpl::shouldSwitchOperands(MachineRegisterInfo &MRI,
MachineInstr &MI,
const SIInstrInfo &TII) const {
auto allUses = MRI.use_nodbg_operands(MI.getOperand(5).getReg());
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

All variable names should start with an upper case letter.

unsigned count = 0;

for (auto &Use : allUses) {
if (Use.getParent()->getOpcode() != AMDGPU::V_CNDMASK_B32_e64)
return false;
MachineOperand *Src0 =
TII.getNamedOperand(*Use.getParent(), AMDGPU::OpName::src0);
MachineOperand *Src1 =
TII.getNamedOperand(*Use.getParent(), AMDGPU::OpName::src1);

auto src0Imm = getImmOrMaterializedImm(*Src0);
auto src1Imm = getImmOrMaterializedImm(*Src1);

if (!src1Imm && src0Imm)
return false;
if (src1Imm && !src0Imm)
count++;
Comment on lines +1517 to +1518
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If src0 has source modifiers then swapping operands will not help with turning this instruction into vop2.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should that be added to check ?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes

}
return (count >= 1);
}

// Try to fold an instruction into a simpler one
bool SIFoldOperandsImpl::tryFoldCndMask(MachineInstr &MI) const {
bool SIFoldOperandsImpl::tryFoldCndMask(MachineInstr &MI, Register *RegVCC,
Register *NewVCC) const {
Comment on lines +1524 to +1525
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't understand how the RegVCC and NewVCC arguments are used. Please add a comment to explain.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

used to remember vcc and new inverted vcc so that shouldSwitchOperands check is called only once.

unsigned Opc = MI.getOpcode();
if (Opc != AMDGPU::V_CNDMASK_B32_e32 && Opc != AMDGPU::V_CNDMASK_B32_e64 &&
Opc != AMDGPU::V_CNDMASK_B64_PSEUDO)
return false;

if (Opc == AMDGPU::V_CNDMASK_B32_e64) {
const DebugLoc &DL = MI.getDebugLoc();
auto Reg = MI.getOperand(5).getReg();

if (*RegVCC != Reg) {
MachineInstr *DefMI = MRI->getVRegDef(Reg);
if (DefMI) {
unsigned Opcode = getInverseCompareOpcode(*DefMI);
if (Opcode &&
SIFoldOperandsImpl::shouldSwitchOperands(*MRI, MI, *TII)) {
auto cmpDL = DefMI->getDebugLoc();
*NewVCC = MRI->createVirtualRegister(MRI->getRegClass(Reg));
*RegVCC = Reg;
MachineInstrBuilder InverseCompare = BuildMI(
*DefMI->getParent(), DefMI, cmpDL, TII->get(Opcode), *NewVCC);
InverseCompare->setFlags(DefMI->getFlags());

unsigned OpNum = DefMI->getNumExplicitOperands();
for (unsigned i = 1; i < OpNum; i++) {
MachineOperand Op = DefMI->getOperand(i);
InverseCompare.add(Op);
if (Op.isReg() && Op.isKill())
InverseCompare->getOperand(i).setIsKill(false);
}
}
}
}
if (*RegVCC == Reg) {
BuildMI(*MI.getParent(), MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64),
MI.getOperand(0).getReg())
.add(MI.getOperand(3))
.add(MI.getOperand(4))
.add(MI.getOperand(1))
.add(MI.getOperand(2))
.addReg(*NewVCC);
MI.eraseFromParent();
return true;
}
}

MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
if (!Src1->isIdenticalTo(*Src0)) {
Expand Down Expand Up @@ -2533,10 +2635,12 @@ bool SIFoldOperandsImpl::run(MachineFunction &MF) {
bool HasNSZ = MFI->hasNoSignedZerosFPMath();

bool Changed = false;
Register Reg = 0;
Register newVCC = 0;
for (MachineBasicBlock *MBB : depth_first(&MF)) {
MachineOperand *CurrentKnownM0Val = nullptr;
for (auto &MI : make_early_inc_range(*MBB)) {
Changed |= tryFoldCndMask(MI);
Changed |= tryFoldCndMask(MI, &Reg, &newVCC);

if (tryFoldZeroHighBits(MI)) {
Changed = true;
Expand Down
Loading
Loading