-
Notifications
You must be signed in to change notification settings - Fork 13.5k
[PowerPC] Add load/store support for v2048i1 and DMF cryptography instructions #136145
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
0945db6
8447ffe
59486b1
25fc59a
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change | ||||||
---|---|---|---|---|---|---|---|---|
|
@@ -1361,8 +1361,11 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, | |||||||
if (Subtarget.isISAFuture()) { | ||||||||
addRegisterClass(MVT::v512i1, &PPC::WACCRCRegClass); | ||||||||
addRegisterClass(MVT::v1024i1, &PPC::DMRRCRegClass); | ||||||||
addRegisterClass(MVT::v2048i1, &PPC::DMRpRCRegClass); | ||||||||
setOperationAction(ISD::LOAD, MVT::v1024i1, Custom); | ||||||||
setOperationAction(ISD::STORE, MVT::v1024i1, Custom); | ||||||||
setOperationAction(ISD::LOAD, MVT::v2048i1, Custom); | ||||||||
setOperationAction(ISD::STORE, MVT::v2048i1, Custom); | ||||||||
} else { | ||||||||
addRegisterClass(MVT::v512i1, &PPC::UACCRCRegClass); | ||||||||
} | ||||||||
|
@@ -11780,15 +11783,19 @@ SDValue PPCTargetLowering::LowerDMFVectorLoad(SDValue Op, | |||||||
SDValue LoadChain = LN->getChain(); | ||||||||
SDValue BasePtr = LN->getBasePtr(); | ||||||||
EVT VT = Op.getValueType(); | ||||||||
bool IsV1024i1 = VT == MVT::v1024i1; | ||||||||
bool IsV2048i1 = VT == MVT::v2048i1; | ||||||||
|
||||||||
// Type v1024i1 is used for Dense Math dmr registers. | ||||||||
assert(VT == MVT::v1024i1 && "Unsupported type."); | ||||||||
// The types v1024i1 and v2048i1 are used for Dense Math dmr registers and | ||||||||
// Dense Math dmr pair registers, respectively. | ||||||||
assert((IsV1024i1 || IsV2048i1) && "Unsupported type."); | ||||||||
assert((Subtarget.hasMMA() && Subtarget.isISAFuture()) && | ||||||||
"Dense Math support required."); | ||||||||
assert(Subtarget.pairedVectorMemops() && "Vector pair support required."); | ||||||||
|
||||||||
SmallVector<SDValue, 4> Loads; | ||||||||
SmallVector<SDValue, 4> LoadChains; | ||||||||
SmallVector<SDValue, 8> Loads; | ||||||||
SmallVector<SDValue, 8> LoadChains; | ||||||||
|
||||||||
SDValue IntrinID = DAG.getConstant(Intrinsic::ppc_vsx_lxvp, dl, MVT::i32); | ||||||||
SDValue LoadOps[] = {LoadChain, IntrinID, BasePtr}; | ||||||||
MachineMemOperand *MMO = LN->getMemOperand(); | ||||||||
|
@@ -11824,10 +11831,37 @@ SDValue PPCTargetLowering::LowerDMFVectorLoad(SDValue Op, | |||||||
SDValue HiSub = DAG.getTargetConstant(PPC::sub_wacc_hi, dl, MVT::i32); | ||||||||
SDValue RC = DAG.getTargetConstant(PPC::DMRRCRegClassID, dl, MVT::i32); | ||||||||
const SDValue Ops[] = {RC, Lo, LoSub, Hi, HiSub}; | ||||||||
|
||||||||
SDValue Value = | ||||||||
SDValue(DAG.getMachineNode(PPC::REG_SEQUENCE, dl, MVT::v1024i1, Ops), 0); | ||||||||
|
||||||||
SDValue RetOps[] = {Value, TF}; | ||||||||
if (IsV1024i1) { | ||||||||
SDValue RetOps[] = {Value, TF}; | ||||||||
return DAG.getMergeValues(RetOps, dl); | ||||||||
} | ||||||||
|
||||||||
// Handle Loads for V2048i1 which represents a dmr pair. | ||||||||
SDValue DmrPValue; | ||||||||
SDValue Dmr1Lo(DAG.getMachineNode(PPC::DMXXINSTDMR512, dl, MVT::v512i1, | ||||||||
Loads[4], Loads[5]), | ||||||||
0); | ||||||||
SDValue Dmr1Hi(DAG.getMachineNode(PPC::DMXXINSTDMR512_HI, dl, MVT::v512i1, | ||||||||
Loads[6], Loads[7]), | ||||||||
0); | ||||||||
const SDValue Dmr1Ops[] = {RC, Dmr1Lo, LoSub, Dmr1Hi, HiSub}; | ||||||||
SDValue Dmr1Value = SDValue( | ||||||||
DAG.getMachineNode(PPC::REG_SEQUENCE, dl, MVT::v1024i1, Dmr1Ops), 0); | ||||||||
|
||||||||
SDValue Dmr0Sub = DAG.getTargetConstant(PPC::sub_dmr0, dl, MVT::i32); | ||||||||
SDValue Dmr1Sub = DAG.getTargetConstant(PPC::sub_dmr1, dl, MVT::i32); | ||||||||
|
||||||||
SDValue DmrPRC = DAG.getTargetConstant(PPC::DMRpRCRegClassID, dl, MVT::i32); | ||||||||
const SDValue DmrPOps[] = {DmrPRC, Value, Dmr0Sub, Dmr1Value, Dmr1Sub}; | ||||||||
|
||||||||
DmrPValue = SDValue( | ||||||||
DAG.getMachineNode(PPC::REG_SEQUENCE, dl, MVT::v2048i1, DmrPOps), 0); | ||||||||
|
||||||||
SDValue RetOps[] = {DmrPValue, TF}; | ||||||||
return DAG.getMergeValues(RetOps, dl); | ||||||||
Comment on lines
+11864
to
11865
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. similar to above
Suggested change
|
||||||||
} | ||||||||
|
||||||||
|
@@ -11839,7 +11873,7 @@ SDValue PPCTargetLowering::LowerVectorLoad(SDValue Op, | |||||||
SDValue BasePtr = LN->getBasePtr(); | ||||||||
EVT VT = Op.getValueType(); | ||||||||
|
||||||||
if (VT == MVT::v1024i1) | ||||||||
if (VT == MVT::v1024i1 || VT == MVT::v2048i1) | ||||||||
return LowerDMFVectorLoad(Op, DAG); | ||||||||
|
||||||||
if (VT != MVT::v256i1 && VT != MVT::v512i1) | ||||||||
|
@@ -11886,34 +11920,88 @@ SDValue PPCTargetLowering::LowerDMFVectorStore(SDValue Op, | |||||||
StoreSDNode *SN = cast<StoreSDNode>(Op.getNode()); | ||||||||
SDValue StoreChain = SN->getChain(); | ||||||||
SDValue BasePtr = SN->getBasePtr(); | ||||||||
SmallVector<SDValue, 4> Values; | ||||||||
SmallVector<SDValue, 4> Stores; | ||||||||
SmallVector<SDValue, 8> Values; | ||||||||
SmallVector<SDValue, 8> Stores; | ||||||||
EVT VT = SN->getValue().getValueType(); | ||||||||
bool IsV1024i1 = VT == MVT::v1024i1; | ||||||||
bool IsV2048i1 = VT == MVT::v2048i1; | ||||||||
|
||||||||
// Type v1024i1 is used for Dense Math dmr registers. | ||||||||
assert(VT == MVT::v1024i1 && "Unsupported type."); | ||||||||
// The types v1024i1 and v2048i1 are used for Dense Math dmr registers and | ||||||||
// Dense Math dmr pair registers, respectively. | ||||||||
assert((IsV1024i1 || IsV2048i1) && "Unsupported type."); | ||||||||
assert((Subtarget.hasMMA() && Subtarget.isISAFuture()) && | ||||||||
"Dense Math support required."); | ||||||||
assert(Subtarget.pairedVectorMemops() && "Vector pair support required."); | ||||||||
|
||||||||
SDValue Lo( | ||||||||
DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, MVT::v512i1, | ||||||||
Op.getOperand(1), | ||||||||
DAG.getTargetConstant(PPC::sub_wacc_lo, dl, MVT::i32)), | ||||||||
0); | ||||||||
SDValue Hi( | ||||||||
DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, MVT::v512i1, | ||||||||
Op.getOperand(1), | ||||||||
DAG.getTargetConstant(PPC::sub_wacc_hi, dl, MVT::i32)), | ||||||||
0); | ||||||||
EVT ReturnTypes[] = {MVT::v256i1, MVT::v256i1}; | ||||||||
MachineSDNode *ExtNode = | ||||||||
DAG.getMachineNode(PPC::DMXXEXTFDMR512, dl, ReturnTypes, Lo); | ||||||||
Values.push_back(SDValue(ExtNode, 0)); | ||||||||
Values.push_back(SDValue(ExtNode, 1)); | ||||||||
ExtNode = DAG.getMachineNode(PPC::DMXXEXTFDMR512_HI, dl, ReturnTypes, Hi); | ||||||||
Values.push_back(SDValue(ExtNode, 0)); | ||||||||
Values.push_back(SDValue(ExtNode, 1)); | ||||||||
if (IsV1024i1) { | ||||||||
SDValue Lo(DAG.getMachineNode( | ||||||||
TargetOpcode::EXTRACT_SUBREG, dl, MVT::v512i1, | ||||||||
Op.getOperand(1), | ||||||||
DAG.getTargetConstant(PPC::sub_wacc_lo, dl, MVT::i32)), | ||||||||
0); | ||||||||
SDValue Hi(DAG.getMachineNode( | ||||||||
TargetOpcode::EXTRACT_SUBREG, dl, MVT::v512i1, | ||||||||
Op.getOperand(1), | ||||||||
DAG.getTargetConstant(PPC::sub_wacc_hi, dl, MVT::i32)), | ||||||||
0); | ||||||||
MachineSDNode *ExtNode = | ||||||||
DAG.getMachineNode(PPC::DMXXEXTFDMR512, dl, ReturnTypes, Lo); | ||||||||
Values.push_back(SDValue(ExtNode, 0)); | ||||||||
Values.push_back(SDValue(ExtNode, 1)); | ||||||||
ExtNode = DAG.getMachineNode(PPC::DMXXEXTFDMR512_HI, dl, ReturnTypes, Hi); | ||||||||
Values.push_back(SDValue(ExtNode, 0)); | ||||||||
Values.push_back(SDValue(ExtNode, 1)); | ||||||||
} else { | ||||||||
// This corresponds to v2048i1 which represents a dmr pair. | ||||||||
SDValue Dmr0( | ||||||||
DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, MVT::v1024i1, | ||||||||
Op.getOperand(1), | ||||||||
DAG.getTargetConstant(PPC::sub_dmr0, dl, MVT::i32)), | ||||||||
0); | ||||||||
|
||||||||
SDValue Dmr1( | ||||||||
DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, MVT::v1024i1, | ||||||||
Op.getOperand(1), | ||||||||
DAG.getTargetConstant(PPC::sub_dmr1, dl, MVT::i32)), | ||||||||
0); | ||||||||
|
||||||||
SDValue Dmr0Lo(DAG.getMachineNode( | ||||||||
TargetOpcode::EXTRACT_SUBREG, dl, MVT::v512i1, Dmr0, | ||||||||
DAG.getTargetConstant(PPC::sub_wacc_lo, dl, MVT::i32)), | ||||||||
0); | ||||||||
|
||||||||
SDValue Dmr0Hi(DAG.getMachineNode( | ||||||||
TargetOpcode::EXTRACT_SUBREG, dl, MVT::v512i1, Dmr0, | ||||||||
DAG.getTargetConstant(PPC::sub_wacc_hi, dl, MVT::i32)), | ||||||||
0); | ||||||||
|
||||||||
SDValue Dmr1Lo(DAG.getMachineNode( | ||||||||
TargetOpcode::EXTRACT_SUBREG, dl, MVT::v512i1, Dmr1, | ||||||||
DAG.getTargetConstant(PPC::sub_wacc_lo, dl, MVT::i32)), | ||||||||
0); | ||||||||
|
||||||||
SDValue Dmr1Hi(DAG.getMachineNode( | ||||||||
TargetOpcode::EXTRACT_SUBREG, dl, MVT::v512i1, Dmr1, | ||||||||
DAG.getTargetConstant(PPC::sub_wacc_hi, dl, MVT::i32)), | ||||||||
0); | ||||||||
|
||||||||
MachineSDNode *ExtNode = | ||||||||
DAG.getMachineNode(PPC::DMXXEXTFDMR512, dl, ReturnTypes, Dmr0Lo); | ||||||||
Values.push_back(SDValue(ExtNode, 0)); | ||||||||
Values.push_back(SDValue(ExtNode, 1)); | ||||||||
ExtNode = | ||||||||
DAG.getMachineNode(PPC::DMXXEXTFDMR512_HI, dl, ReturnTypes, Dmr0Hi); | ||||||||
Values.push_back(SDValue(ExtNode, 0)); | ||||||||
Values.push_back(SDValue(ExtNode, 1)); | ||||||||
ExtNode = DAG.getMachineNode(PPC::DMXXEXTFDMR512, dl, ReturnTypes, Dmr1Lo); | ||||||||
Values.push_back(SDValue(ExtNode, 0)); | ||||||||
Values.push_back(SDValue(ExtNode, 1)); | ||||||||
ExtNode = | ||||||||
DAG.getMachineNode(PPC::DMXXEXTFDMR512_HI, dl, ReturnTypes, Dmr1Hi); | ||||||||
Values.push_back(SDValue(ExtNode, 0)); | ||||||||
Values.push_back(SDValue(ExtNode, 1)); | ||||||||
} | ||||||||
|
||||||||
if (Subtarget.isLittleEndian()) | ||||||||
std::reverse(Values.begin(), Values.end()); | ||||||||
|
@@ -11952,7 +12040,7 @@ SDValue PPCTargetLowering::LowerVectorStore(SDValue Op, | |||||||
SDValue Value2 = SN->getValue(); | ||||||||
EVT StoreVT = Value.getValueType(); | ||||||||
|
||||||||
if (StoreVT == MVT::v1024i1) | ||||||||
if (StoreVT == MVT::v1024i1 || StoreVT == MVT::v2048i1) | ||||||||
return LowerDMFVectorStore(Op, DAG); | ||||||||
|
||||||||
if (StoreVT != MVT::v256i1 && StoreVT != MVT::v512i1) | ||||||||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -1140,6 +1140,6 @@ def PPCRegDMRpRCAsmOperand : AsmOperandClass { | |
let PredicateMethod = "isDMRpRegNumber"; | ||
} | ||
|
||
def dmrp : RegisterOperand<DMRpRC> { | ||
def dmrprc : RegisterOperand<DMRpRC> { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why are we renaming this? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It is dmr pair reg class, similar to def vsrc : RegisterOperand { There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Do we want to change all the dmr register classes then? Cause it seems the acc and dmr register def no longer follow that since I see |
||
let ParserMatchClass = PPCRegDMRpRCAsmOperand; | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
nit: can we get rid of tmp variables when possible?