Skip to content

Refactor udt intrinsic arg copy to before SROA, flatten RayDesc #7440

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 16 commits into from
May 16, 2025
Merged

Refactor udt intrinsic arg copy to before SROA, flatten RayDesc #7440

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
39f8067
Refactor udt intrinsic arg copy to before SROA
tex3d May 8, 2025
39f76a3
Copy RayDesc args in too, copyIntrinsicUDTArgs -> copyIntrinsicAggArgs
tex3d May 8, 2025
291b36a
Flatten RayDesc, update tests
tex3d May 9, 2025
8ded3e0
clang-format
tex3d May 9, 2025
ec41344
Remove/Update PIX tests checking for old RayDesc IR pattern
tex3d May 13, 2025
3720504
Remove leftover TODOs in SROA
tex3d May 13, 2025
c1a2b12
Address feedback.
tex3d May 13, 2025
c197679
Add Trace and Invoke SROA pass test
tex3d May 13, 2025
b5bdcc7
Add TraceRay SROA pass test
tex3d May 13, 2025
e6b8799
Add TraceRayInline tests, including for RayDesc from cbuffer
tex3d May 14, 2025
1a3064f
Fix a couple missing variable substitutions
tex3d May 14, 2025
23b49a0
Remove accidental add of bad test file
tex3d May 14, 2025
b986d79
Remove extra braces
tex3d May 14, 2025
28d0100
test robustness, fix missing value captures, exclude comma
tex3d May 14, 2025
fc2b54f
Fix bad CHECK expressions in tests
tex3d May 14, 2025
a307db7
feedback: interleave CHECKs and remove 'COM:'
tex3d May 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions include/dxc/DXIL/DxilConstants.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1583,6 +1583,10 @@ const unsigned kMSStoreOutputColOpIdx = 3;
const unsigned kMSStoreOutputVIdxOpIdx = 4;
const unsigned kMSStoreOutputValOpIdx = 5;

// HitObject::MakeMiss
const unsigned kHitObjectMakeMiss_RayDescOpIdx = 3;
const unsigned kHitObjectMakeMiss_NumOp = 11;

// HitObject::TraceRay
const unsigned kHitObjectTraceRay_RayDescOpIdx = 7;
const unsigned kHitObjectTraceRay_PayloadOpIdx = 15;
Expand Down
23 changes: 20 additions & 3 deletions include/dxc/HLSL/HLOperations.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -396,7 +396,12 @@ const unsigned kAnnotateHandleResourceTypeOpIdx = 3;

// TraceRay.
const unsigned kTraceRayRayDescOpIdx = 7;
const unsigned kTraceRayPayLoadOpIdx = 8;
// kTraceRayPayloadPreOpIdx is before flattening the RayDesc
const unsigned kTraceRayPayloadPreOpIdx = 8;
// kTraceRayPayloadOpIdx is after flattening the RayDesc
const unsigned kTraceRayPayloadOpIdx = 11;
const unsigned kTraceRay_PreNumOp = 9;
const unsigned kTraceRay_NumOp = 12;

// AllocateRayQuery
const unsigned kAllocateRayQueryRayFlagsIdx = 1;
Expand All @@ -407,6 +412,10 @@ const unsigned kCallShaderPayloadOpIdx = 2;

// TraceRayInline.
const unsigned kTraceRayInlineRayDescOpIdx = 5;
// kTraceRayInlinePayloadPreOpIdx is before flattening the RayDesc
const unsigned kTraceRayInlinePayloadPreOpIdx = 6;
// kTraceRayInlinePayloadOpIdx is after flattening the RayDesc
const unsigned kTraceRayInlinePayloadOpIdx = 9;

// ReportIntersection.
const unsigned kReportIntersectionAttributeOpIdx = 3;
Expand Down Expand Up @@ -435,11 +444,19 @@ const unsigned kAnnotateNodeRecordHandleNodeRecordPropIdx = 2;

// HitObject::MakeMiss
const unsigned kHitObjectMakeMiss_NumOp = 8;
const unsigned kHitObjectMakeMissRayDescOpIdx = 4;
const unsigned kHitObjectMakeMiss_RayDescOpIdx = 4;

// HitObject::TraceRay
const unsigned kHitObjectTraceRay_RayDescOpIdx = 8;
const unsigned kHitObjectTraceRay_NumOp = 10;
// kHitObjectTraceRay_PayloadPreOpIdx is before flattening the RayDesc
const unsigned kHitObjectTraceRay_PayloadPreOpIdx = 9;
// kHitObjectTraceRay_PayloadOpIdx is after flattening the RayDesc
const unsigned kHitObjectTraceRay_PayloadOpIdx = 12;
const unsigned kHitObjectTraceRay_PreNumOp = 10;
const unsigned kHitObjectTraceRay_NumOp = 13;

// HitObject::Invoke
const unsigned kHitObjectInvoke_PayloadOpIdx = 2;

// HitObject::FromRayQuery
const unsigned kHitObjectFromRayQuery_WithAttrs_AttributeOpIdx = 4;
Expand Down
200 changes: 86 additions & 114 deletions lib/HLSL/HLOperationLower.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5720,37 +5720,24 @@ Value *TranslateCallShader(CallInst *CI, IntrinsicOp IOP, OP::OpCode opcode,
return Builder.CreateCall(F, {opArg, ShaderIndex, Parameter});
}

static unsigned LoadRayDescElementsIntoArgs(Value **Args, hlsl::OP *OP,
IRBuilder<> &Builder,
Value *RayDescPtr, unsigned Index) {
// struct RayDesc
//{
// float3 Origin;
// float TMin;
// float3 Direction;
// float TMax;
//};
Value *ZeroIdx = OP->GetU32Const(0);
Value *Origin = Builder.CreateGEP(RayDescPtr, {ZeroIdx, ZeroIdx});
Origin = Builder.CreateLoad(Origin);
Args[Index++] = Builder.CreateExtractElement(Origin, (uint64_t)0);
Args[Index++] = Builder.CreateExtractElement(Origin, 1);
Args[Index++] = Builder.CreateExtractElement(Origin, 2);

Value *TMinPtr = Builder.CreateGEP(RayDescPtr, {ZeroIdx, OP->GetU32Const(1)});
Args[Index++] = Builder.CreateLoad(TMinPtr);

Value *DirectionPtr =
Builder.CreateGEP(RayDescPtr, {ZeroIdx, OP->GetU32Const(2)});
Value *Direction = Builder.CreateLoad(DirectionPtr);

Args[Index++] = Builder.CreateExtractElement(Direction, (uint64_t)0);
Args[Index++] = Builder.CreateExtractElement(Direction, 1);
Args[Index++] = Builder.CreateExtractElement(Direction, 2);

Value *TMaxPtr = Builder.CreateGEP(RayDescPtr, {ZeroIdx, OP->GetU32Const(3)});
Args[Index++] = Builder.CreateLoad(TMaxPtr);
return Index;
static void TransferRayDescArgs(Value **Args, hlsl::OP *OP,
IRBuilder<> &Builder, CallInst *CI,
unsigned &Index, unsigned &HLIndex) {
// Extract elements from flattened ray desc arguments in HL op.
// float3 Origin;
Value *origin = CI->getArgOperand(HLIndex++);
Args[Index++] = Builder.CreateExtractElement(origin, (uint64_t)0);
Args[Index++] = Builder.CreateExtractElement(origin, 1);
Args[Index++] = Builder.CreateExtractElement(origin, 2);
// float TMin;
Args[Index++] = CI->getArgOperand(HLIndex++);
// float3 Direction;
Value *direction = CI->getArgOperand(HLIndex++);
Args[Index++] = Builder.CreateExtractElement(direction, (uint64_t)0);
Args[Index++] = Builder.CreateExtractElement(direction, 1);
Args[Index++] = Builder.CreateExtractElement(direction, 2);
// float TMax;
Args[Index++] = CI->getArgOperand(HLIndex++);
}

Value *TranslateTraceRay(CallInst *CI, IntrinsicOp IOP, OP::OpCode OpCode,
Expand All @@ -5759,21 +5746,24 @@ Value *TranslateTraceRay(CallInst *CI, IntrinsicOp IOP, OP::OpCode OpCode,
bool &Translated) {
hlsl::OP *OP = &Helper.hlslOP;

Value *RayDesc = CI->getArgOperand(HLOperandIndex::kTraceRayRayDescOpIdx);
Value *PayLoad = CI->getArgOperand(HLOperandIndex::kTraceRayPayLoadOpIdx);

Value *Args[DXIL::OperandIndex::kTraceRayNumOp];
Args[0] = OP->GetU32Const(static_cast<unsigned>(OpCode));
for (unsigned i = 1; i < HLOperandIndex::kTraceRayRayDescOpIdx; i++)
Args[i] = CI->getArgOperand(i);
unsigned Index = 1, HLIndex = 1;
while (HLIndex < HLOperandIndex::kTraceRayRayDescOpIdx)
Args[Index++] = CI->getArgOperand(HLIndex++);

IRBuilder<> Builder(CI);
LoadRayDescElementsIntoArgs(Args, OP, Builder, RayDesc,
DXIL::OperandIndex::kTraceRayRayDescOpIdx);
TransferRayDescArgs(Args, OP, Builder, CI, Index, HLIndex);
DXASSERT_NOMSG(HLIndex == CI->getNumArgOperands() - 1);
DXASSERT_NOMSG(Index == DXIL::OperandIndex::kTraceRayPayloadOpIdx);

Value *Payload = CI->getArgOperand(HLIndex++);
Args[Index++] = Payload;

Args[DXIL::OperandIndex::kTraceRayPayloadOpIdx] = PayLoad;
DXASSERT_NOMSG(HLIndex == CI->getNumArgOperands());
DXASSERT_NOMSG(Index == DXIL::OperandIndex::kTraceRayNumOp);

Type *Ty = PayLoad->getType();
Type *Ty = Payload->getType();
Function *F = OP->GetOpFunc(OpCode, Ty);

return Builder.CreateCall(F, Args);
Expand Down Expand Up @@ -5817,33 +5807,16 @@ Value *TranslateTraceRayInline(CallInst *CI, IntrinsicOp IOP, OP::OpCode opcode,

Value *Args[DXIL::OperandIndex::kTraceRayInlineNumOp];
Args[0] = opArg;
for (unsigned i = 1; i < HLOperandIndex::kTraceRayInlineRayDescOpIdx; i++) {
Args[i] = CI->getArgOperand(i);
}
unsigned Index = 1, HLIndex = 1;
while (HLIndex < HLOperandIndex::kTraceRayInlineRayDescOpIdx)
Args[Index++] = CI->getArgOperand(HLIndex++);

IRBuilder<> Builder(CI);
unsigned hlIndex = HLOperandIndex::kTraceRayInlineRayDescOpIdx;
unsigned index = DXIL::OperandIndex::kTraceRayInlineRayDescOpIdx;

// struct RayDesc
//{
// float3 Origin;
Value *origin = CI->getArgOperand(hlIndex++);
Args[index++] = Builder.CreateExtractElement(origin, (uint64_t)0);
Args[index++] = Builder.CreateExtractElement(origin, 1);
Args[index++] = Builder.CreateExtractElement(origin, 2);
// float TMin;
Args[index++] = CI->getArgOperand(hlIndex++);
// float3 Direction;
Value *direction = CI->getArgOperand(hlIndex++);
Args[index++] = Builder.CreateExtractElement(direction, (uint64_t)0);
Args[index++] = Builder.CreateExtractElement(direction, 1);
Args[index++] = Builder.CreateExtractElement(direction, 2);
// float TMax;
Args[index++] = CI->getArgOperand(hlIndex++);
//};

DXASSERT_NOMSG(index == DXIL::OperandIndex::kTraceRayInlineNumOp);
DXASSERT_NOMSG(HLIndex == HLOperandIndex::kTraceRayInlineRayDescOpIdx);
DXASSERT_NOMSG(Index == DXIL::OperandIndex::kTraceRayInlineRayDescOpIdx);
TransferRayDescArgs(Args, hlslOP, Builder, CI, Index, HLIndex);
DXASSERT_NOMSG(HLIndex == CI->getNumArgOperands());
DXASSERT_NOMSG(Index == DXIL::OperandIndex::kTraceRayInlineNumOp);

Function *F = hlslOP->GetOpFunc(opcode, Builder.getVoidTy());

Expand Down Expand Up @@ -6197,55 +6170,49 @@ Value *TranslateUnpack(CallInst *CI, IntrinsicOp IOP, OP::OpCode opcode,

// Shader Execution Reordering.
namespace {
Value *TranslateHitObjectMake(CallInst *CI, IntrinsicOp IOP, OP::OpCode Opcode,
HLOperationLowerHelper &Helper,
HLObjectOperationLowerHelper *ObjHelper,
bool &Translated) {
Value *TranslateHitObjectMakeNop(CallInst *CI, IntrinsicOp IOP,
OP::OpCode Opcode,
HLOperationLowerHelper &Helper,
HLObjectOperationLowerHelper *ObjHelper,
bool &Translated) {
hlsl::OP *HlslOP = &Helper.hlslOP;
IRBuilder<> Builder(CI);
unsigned SrcIdx = 1;
Value *HitObjectPtr = CI->getArgOperand(SrcIdx++);
if (Opcode == OP::OpCode::HitObject_MakeNop) {
Value *HitObject = TrivialDxilOperation(
Opcode, {nullptr}, Type::getVoidTy(CI->getContext()), CI, HlslOP);
Builder.CreateStore(HitObject, HitObjectPtr);
DXASSERT(
CI->use_empty(),
"Default ctor return type is a Clang artifact. Value must not be used");
return nullptr;
}
Value *HitObjectPtr = CI->getArgOperand(1);
Value *HitObject = TrivialDxilOperation(
Opcode, {nullptr}, Type::getVoidTy(CI->getContext()), CI, HlslOP);
Builder.CreateStore(HitObject, HitObjectPtr);
DXASSERT(
CI->use_empty(),
"Default ctor return type is a Clang artifact. Value must not be used");
return nullptr;
}

Value *TranslateHitObjectMakeMiss(CallInst *CI, IntrinsicOp IOP,
OP::OpCode Opcode,
HLOperationLowerHelper &Helper,
HLObjectOperationLowerHelper *ObjHelper,
bool &Translated) {
DXASSERT_NOMSG(CI->getNumArgOperands() ==
HLOperandIndex::kHitObjectMakeMiss_NumOp);
Value *RayFlags = CI->getArgOperand(SrcIdx++);
Value *MissShaderIdx = CI->getArgOperand(SrcIdx++);
DXASSERT_NOMSG(SrcIdx == HLOperandIndex::kHitObjectMakeMissRayDescOpIdx);
Value *RayDescOrigin = CI->getArgOperand(SrcIdx++);
Value *RayDescOriginX =
Builder.CreateExtractElement(RayDescOrigin, (uint64_t)0);
Value *RayDescOriginY =
Builder.CreateExtractElement(RayDescOrigin, (uint64_t)1);
Value *RayDescOriginZ =
Builder.CreateExtractElement(RayDescOrigin, (uint64_t)2);

Value *RayDescTMin = CI->getArgOperand(SrcIdx++);
Value *RayDescDirection = CI->getArgOperand(SrcIdx++);
Value *RayDescDirectionX =
Builder.CreateExtractElement(RayDescDirection, (uint64_t)0);
Value *RayDescDirectionY =
Builder.CreateExtractElement(RayDescDirection, (uint64_t)1);
Value *RayDescDirectionZ =
Builder.CreateExtractElement(RayDescDirection, (uint64_t)2);

Value *RayDescTMax = CI->getArgOperand(SrcIdx++);
hlsl::OP *OP = &Helper.hlslOP;
IRBuilder<> Builder(CI);
Value *Args[DXIL::OperandIndex::kHitObjectMakeMiss_NumOp];
Args[0] = nullptr; // Filled in by TrivialDxilOperation

unsigned DestIdx = 1, SrcIdx = 1;
Value *HitObjectPtr = CI->getArgOperand(SrcIdx++);
Args[DestIdx++] = CI->getArgOperand(SrcIdx++); // RayFlags
Args[DestIdx++] = CI->getArgOperand(SrcIdx++); // MissShaderIdx

DXASSERT_NOMSG(SrcIdx == HLOperandIndex::kHitObjectMakeMiss_RayDescOpIdx);
DXASSERT_NOMSG(DestIdx ==
DXIL::OperandIndex::kHitObjectMakeMiss_RayDescOpIdx);
TransferRayDescArgs(Args, OP, Builder, CI, DestIdx, SrcIdx);
DXASSERT_NOMSG(SrcIdx == CI->getNumArgOperands());
DXASSERT_NOMSG(DestIdx == DXIL::OperandIndex::kHitObjectMakeMiss_NumOp);

Value *OutHitObject = TrivialDxilOperation(
Opcode,
{nullptr, RayFlags, MissShaderIdx, RayDescOriginX, RayDescOriginY,
RayDescOriginZ, RayDescTMin, RayDescDirectionX, RayDescDirectionY,
RayDescDirectionZ, RayDescTMax},
Helper.voidTy, CI, HlslOP);
Value *OutHitObject =
TrivialDxilOperation(Opcode, Args, Helper.voidTy, CI, OP);
Builder.CreateStore(OutHitObject, HitObjectPtr);
return nullptr;
}
Expand Down Expand Up @@ -6348,10 +6315,9 @@ Value *TranslateHitObjectTraceRay(CallInst *CI, IntrinsicOp IOP,
hlsl::OP *OP = &Helper.hlslOP;
IRBuilder<> Builder(CI);

const unsigned DxilNumArgs = DxilInst_HitObject_TraceRay::arg_payload + 1;
DXASSERT_NOMSG(CI->getNumArgOperands() ==
HLOperandIndex::kHitObjectTraceRay_NumOp);
Value *Args[DxilNumArgs];
Value *Args[DXIL::OperandIndex::kHitObjectTraceRay_NumOp];
Value *OpArg = OP->GetU32Const(static_cast<unsigned>(OpCode));
Args[0] = OpArg;

Expand All @@ -6363,13 +6329,19 @@ Value *TranslateHitObjectTraceRay(CallInst *CI, IntrinsicOp IOP,
Args[DestIdx] = CI->getArgOperand(SrcIdx);
}

Value *RayDescPtr = CI->getArgOperand(SrcIdx++);
DestIdx = LoadRayDescElementsIntoArgs(Args, OP, Builder, RayDescPtr, DestIdx);
DXASSERT_NOMSG(SrcIdx == HLOperandIndex::kHitObjectTraceRay_RayDescOpIdx);
DXASSERT_NOMSG(DestIdx ==
DXIL::OperandIndex::kHitObjectTraceRay_RayDescOpIdx);
TransferRayDescArgs(Args, OP, Builder, CI, DestIdx, SrcIdx);
DXASSERT_NOMSG(SrcIdx == CI->getNumArgOperands() - 1);
DXASSERT_NOMSG(DestIdx ==
DXIL::OperandIndex::kHitObjectTraceRay_PayloadOpIdx);

Value *Payload = CI->getArgOperand(SrcIdx++);
Args[DestIdx++] = Payload;

DXASSERT_NOMSG(SrcIdx == CI->getNumArgOperands());
DXASSERT_NOMSG(DestIdx == DxilNumArgs);
DXASSERT_NOMSG(DestIdx == DXIL::OperandIndex::kHitObjectTraceRay_NumOp);

Function *F = OP->GetOpFunc(OpCode, Payload->getType());

Expand Down Expand Up @@ -7402,7 +7374,7 @@ IntrinsicLower gLowerTable[] = {
DXIL::OpCode::NumOpCodes},
{IntrinsicOp::MOP_InterlockedUMin, TranslateMopAtomicBinaryOperation,
DXIL::OpCode::NumOpCodes},
{IntrinsicOp::MOP_DxHitObject_MakeNop, TranslateHitObjectMake,
{IntrinsicOp::MOP_DxHitObject_MakeNop, TranslateHitObjectMakeNop,
DXIL::OpCode::HitObject_MakeNop},
{IntrinsicOp::IOP_DxMaybeReorderThread, TranslateMaybeReorderThread,
DXIL::OpCode::MaybeReorderThread},
Expand Down Expand Up @@ -7462,7 +7434,7 @@ IntrinsicLower gLowerTable[] = {
{IntrinsicOp::MOP_DxHitObject_LoadLocalRootTableConstant,
TranslateHitObjectLoadLocalRootTableConstant,
DXIL::OpCode::HitObject_LoadLocalRootTableConstant},
{IntrinsicOp::MOP_DxHitObject_MakeMiss, TranslateHitObjectMake,
{IntrinsicOp::MOP_DxHitObject_MakeMiss, TranslateHitObjectMakeMiss,
DXIL::OpCode::HitObject_MakeMiss},
{IntrinsicOp::MOP_DxHitObject_SetShaderTableIndex,
TranslateHitObjectSetShaderTableIndex,
Expand Down
Loading