Revert "First batch of fixes that reduce the stack enough to run on t…

…he ProceduralGeometry sample"

This reverts commit 419f019.

Libraries/D3D12RaytracingFallback/src/DxilShaderPatcher.cpp

@@ -11,7 +11,7 @@
 #include "pch.h"
 
 using namespace hlsl;
-#define SPEW_SHADERS 1
+#define SPEW_SHADERS 0
 namespace FallbackLayer
 {
     void DxilShaderPatcher::VerifyResult(IDxcOperationResult *pResult)
@@ -189,7 +189,6 @@ namespace FallbackLayer
 
         shaderIdentifiers.resize(exportNames.size());
         CComPtr<IDxcOperationResult> pResult;
-        pFallbackCompiler->SetDebugOutput(3);
         pFallbackCompiler->Compile(L"main", pLibBlobPtrs.data(), (UINT32)pLibBlobPtrs.size(), exportNames.data(), (int*)shaderIdentifiers.data(), (UINT32)exportNames.size(), stackSize, &pResult);
 
         VerifyResult(pResult);

Libraries/D3D12RaytracingFallback/src/TraverseFunction.hlsli

@@ -472,25 +472,6 @@ void dump(BoundingBox box, uint2 flags) {}
 
 Declare_Fallback_SetPendingAttr(BuiltInTriangleIntersectionAttributes);
 
-#define EndSearch 0x1
-
-void SetBoolFlag(inout uint flagContainer, uint flag, bool enable)
-{
-    if (enable)
-    {
-        flagContainer |= flag;
-    }
-    else
-    {
-        flagContainer &= ~flag;
-    }
-}
-
-bool GetBoolFlag(uint flagContainer, uint flag)
-{
-    return flagContainer & flag;
-}
-
 bool Traverse(
     uint InstanceInclusionMask,
     uint RayContributionToHitGroupIndex,
@@ -500,9 +481,12 @@ bool Traverse(
     uint GI = Fallback_GroupIndex();
     const GpuVA nullptr = GpuVA(0, 0);
 
-    RayData currentRayData = GetRayData(WorldRayOrigin(), WorldRayDirection());
+    RayData worldRayData = GetRayData(WorldRayOrigin(), WorldRayDirection());
+    RayData currentRayData = worldRayData;
 
-    uint flagContainer = 0;
+    float3x4 CurrentObjectToWorld = 0;
+    float3x4 CurrentWorldToObject = 0;
+    bool ResetMatrices = true;
 
     uint nodesToProcess[NUM_BVH_LEVELS];
     uint currentBVHIndex = TOP_LEVEL_INDEX;
@@ -589,15 +573,13 @@ bool Traverse(
                             currentGpuVA = instanceDesc.AccelerationStructure;
                             instanceFlags = GetInstanceFlags(instanceDesc);
 
-                            float3x4 CurrentWorldToObject = CreateMatrix(instanceDesc.Transform);
-                            float3x4 CurrentObjectToWorld = CreateMatrix(metadata.ObjectToWorld);
+                            CurrentWorldToObject = CreateMatrix(instanceDesc.Transform);
+                            CurrentObjectToWorld = CreateMatrix(metadata.ObjectToWorld);
 
                             currentRayData = GetRayData(
                                 mul(CurrentWorldToObject, float4(WorldRayOrigin(), 1)),
                                 mul(CurrentWorldToObject, float4(WorldRayDirection(), 0)));
 
-                            UpdateObjectSpaceProperties(currentRayData.Origin, currentRayData.Direction, CurrentWorldToObject, CurrentObjectToWorld);
-
                             nodesToProcess[BOTTOM_LEVEL_INDEX] = 1;
                         }
                     }
@@ -634,10 +616,16 @@ bool Traverse(
                             uint intersectionStateId, anyHitStateId;
                             GetAnyHitAndIntersectionStateId(HitGroupShaderTable, hitGroupRecordOffset, anyHitStateId, intersectionStateId);
 
+                            if (ResetMatrices)
+                            {
+                                UpdateObjectSpaceProperties(currentRayData.Origin, currentRayData.Direction, CurrentWorldToObject, CurrentObjectToWorld);
+                                ResetMatrices = false;
+                            }
+
                             Fallback_SetAnyHitStateId(anyHitStateId);
                             Fallback_SetAnyHitResult(ACCEPT);
                             Fallback_CallIndirect(intersectionStateId);
-                            SetBoolFlag(flagContainer, EndSearch, Fallback_AnyHitResult() == END_SEARCH);
+                            endSearch = Fallback_AnyHitResult() == END_SEARCH;
                         }
                         else if (!culled && TestLeafNodeIntersections( // TODO: We need to break out this function so we can run anyhit on each triangle
                             currentBVH,
@@ -667,26 +655,33 @@ bool Traverse(
 #endif
                             closestBoxT = min(closestBoxT, resultT);
 
+                            if (ResetMatrices)
+                            {
+                                UpdateObjectSpaceProperties(currentRayData.Origin, currentRayData.Direction, CurrentWorldToObject, CurrentObjectToWorld);
+                                ResetMatrices = false;
+                            }
+
+                            bool endSearch = false;
                             if (opaque)
                             {
                                 MARK(8, 1);
                                 Fallback_CommitHit();
-                                SetBoolFlag(flagContainer, EndSearch, RayFlags() & RAY_FLAG_ACCEPT_FIRST_HIT_AND_END_SEARCH);
+                                endSearch = RayFlags() & RAY_FLAG_ACCEPT_FIRST_HIT_AND_END_SEARCH;
                             }
                             else
                             {
                                 MARK(8, 2);
+
                                 uint anyhitStateId = GetAnyHitStateId(HitGroupShaderTable, hitGroupRecordOffset);
                                 int ret = ACCEPT;
                                 if (anyhitStateId)
                                     ret = InvokeAnyHit(anyhitStateId);
                                 if (ret != IGNORE)
                                     Fallback_CommitHit();
-
-                                SetBoolFlag(flagContainer, EndSearch, (ret == END_SEARCH) || (RayFlags() & RAY_FLAG_ACCEPT_FIRST_HIT_AND_END_SEARCH));
+                                endSearch = (ret == END_SEARCH) || (RayFlags() & RAY_FLAG_ACCEPT_FIRST_HIT_AND_END_SEARCH);
                             }
                         }
-                        if (GetBoolFlag(flagContainer, EndSearch))
+                        if (endSearch)
                         {
                             nodesToProcess[BOTTOM_LEVEL_INDEX] = 0;
                             nodesToProcess[TOP_LEVEL_INDEX] = 0;
@@ -745,8 +740,9 @@ bool Traverse(
             }
         } while (nodesToProcess[currentBVHIndex] != 0);
         currentBVHIndex--;
-        currentRayData = GetRayData(WorldRayOrigin(), WorldRayDirection());
+        currentRayData = worldRayData;
         currentGpuVA = TopLevelAccelerationStructureGpuVA;
+        ResetMatrices = true;
     } 
     MARK(10,0);
     bool isHit = Fallback_InstanceIndex() != NO_HIT_SENTINEL;

Samples/Desktop/D3D12Raytracing/src/D3D12RaytracingProceduralGeometry/D3D12RaytracingProceduralGeometry.cpp

@@ -473,7 +473,6 @@ void D3D12RaytracingProceduralGeometry::CreateRaytracingPipelineStateObject()
     if (m_raytracingAPI == RaytracingAPI::FallbackLayer)
     {
         ThrowIfFailed(m_fallbackDevice->CreateStateObject(raytracingPipeline, IID_PPV_ARGS(&m_fallbackStateObject)), L"Couldn't create DirectX Raytracing state object.\n");
-        m_fallbackStateObject->SetPipelineStackSize(10);
     }
     else // DirectX Raytracing
     {
@@ -512,7 +511,7 @@ void D3D12RaytracingProceduralGeometry::CreateDescriptorHeap()
     // 2 - vertex and index buffer SRVs
     // 1 - raytracing output texture SRV
     // 2 - bottom and top level acceleration structure fallback wrapped pointer UAVs
-    descriptorHeapDesc.NumDescriptors = 6;
+    descriptorHeapDesc.NumDescriptors = 5;
     descriptorHeapDesc.Type = D3D12_DESCRIPTOR_HEAP_TYPE_CBV_SRV_UAV;
     descriptorHeapDesc.Flags = D3D12_DESCRIPTOR_HEAP_FLAG_SHADER_VISIBLE;
     descriptorHeapDesc.NodeMask = 0;
@@ -760,7 +759,7 @@ void D3D12RaytracingProceduralGeometry::BuildBotomLevelASInstanceDescs(BLASPtrTy
             -((NUM_INSTANCE_Z - 1) * (fWidth.z + c_aabbDistance) / 2.0f)));
 
         InstanceDescType instanceDescTemplate = {};
-        instanceDescTemplate.InstanceMask = 2;
+        instanceDescTemplate.InstanceMask = 1;
         instanceDescTemplate.InstanceContributionToHitGroupIndex = BottomLevelASType::AABB;
         instanceDescTemplate.AccelerationStructure = bottomLevelASaddresses[BottomLevelASType::AABB];