[Bugfix] Shade each XR eye from its own position

Sources/UntoldEngine/Renderer/RenderExtensionModelSurface.swift

@@ -479,7 +479,7 @@ private func drawModelSurfaceMeshes(
         uniforms.normalMatrix = normalMatrix
         uniforms.viewMatrix = viewMatrix
         uniforms.modelMatrix = modelMatrix
-        uniforms.cameraPosition = SceneRootTransform.shared.effectiveCameraPosition(cameraComponent.localPosition)
+        uniforms.cameraPosition = shadingCameraPosition(cameraComponent)
         uniforms.projectionMatrix = renderInfo.perspectiveSpace
 
         renderEncoder.setVertexBytes(

Sources/UntoldEngine/Renderer/RenderPasses.swift

@@ -291,7 +291,7 @@ public enum RenderPasses {
 
         init(cameraComponent: CameraComponent) {
             viewMatrix = SceneRootTransform.shared.effectiveViewMatrix(cameraComponent.viewSpace)
-            cameraPosition = SceneRootTransform.shared.effectiveCameraPosition(cameraComponent.localPosition)
+            cameraPosition = shadingCameraPosition(cameraComponent)
         }
 
         /// The entity's LOD component while LOD fades are on; nil otherwise, and for an
@@ -1119,7 +1119,7 @@ public enum RenderPasses {
         let viewMatrix = SceneRootTransform.shared.effectiveViewMatrix(cameraComponent.viewSpace)
         skyUniforms.invViewMatrix = viewMatrix.inverse
         skyUniforms.invProjectionMatrix = renderInfo.perspectiveSpace.inverse
-        skyUniforms.cameraPosition = SceneRootTransform.shared.effectiveCameraPosition(cameraComponent.localPosition)
+        skyUniforms.cameraPosition = shadingCameraPosition(cameraComponent)
 
         let sunParameters = getDirectionalLightParameters()
         skyUniforms.sunDirection = sunParameters.direction
@@ -1320,7 +1320,7 @@ public enum RenderPasses {
         let viewMatrix = SceneRootTransform.shared.effectiveViewMatrix(cameraComponent.viewSpace)
         // What every draw of the pass shares, read once.
         let sceneSnapshot = RenderSceneSnapshot()
-        let cameraPosition = SceneRootTransform.shared.effectiveCameraPosition(cameraComponent.localPosition)
+        let cameraPosition = shadingCameraPosition(cameraComponent)
         let projectionMatrix = renderInfo.perspectiveSpace
 
         // Render each cascade into its own depth array slice.
@@ -1433,7 +1433,7 @@ public enum RenderPasses {
         batchUniforms.viewMatrix = viewMatrix
         batchUniforms.modelViewMatrix = viewMatrix
         batchUniforms.normalMatrix = matrix_identity_float3x3
-        batchUniforms.cameraPosition = SceneRootTransform.shared.effectiveCameraPosition(cameraComponent.localPosition)
+        batchUniforms.cameraPosition = shadingCameraPosition(cameraComponent)
         batchUniforms.projectionMatrix = renderInfo.perspectiveSpace
 
         for cascadeIdx in 0 ..< csmCascadeCount {
@@ -1558,7 +1558,7 @@ public enum RenderPasses {
         let viewMatrix = SceneRootTransform.shared.effectiveViewMatrix(cameraComponent.viewSpace)
         // What every draw of the pass shares, read once.
         let sceneSnapshot = RenderSceneSnapshot()
-        let cameraPosition = SceneRootTransform.shared.effectiveCameraPosition(cameraComponent.localPosition)
+        let cameraPosition = shadingCameraPosition(cameraComponent)
         let projectionMatrix = renderInfo.perspectiveSpace
         for caster in spotShadowCasters(in: sceneSnapshot) {
             let transformComponent = caster.components.world
@@ -1598,7 +1598,7 @@ public enum RenderPasses {
             batchUniforms.viewMatrix = viewMatrix
             batchUniforms.modelViewMatrix = viewMatrix
             batchUniforms.normalMatrix = matrix_identity_float3x3
-            batchUniforms.cameraPosition = SceneRootTransform.shared.effectiveCameraPosition(cameraComponent.localPosition)
+            batchUniforms.cameraPosition = shadingCameraPosition(cameraComponent)
             batchUniforms.projectionMatrix = renderInfo.perspectiveSpace
 
             for batchGroup in spotShadowCasterBatchGroups() {
@@ -1658,7 +1658,7 @@ public enum RenderPasses {
         let viewMatrix = SceneRootTransform.shared.effectiveViewMatrix(cameraComponent.viewSpace)
         // What every draw of the pass shares, read once.
         let sceneSnapshot = RenderSceneSnapshot()
-        let cameraPosition = SceneRootTransform.shared.effectiveCameraPosition(cameraComponent.localPosition)
+        let cameraPosition = shadingCameraPosition(cameraComponent)
         let projectionMatrix = renderInfo.perspectiveSpace
         let casters = pointShadowCasters(in: sceneSnapshot)
         let batchGroups = BatchingSystem.shared.isEnabled() ? pointShadowCasterBatchGroups() : []
@@ -1732,7 +1732,7 @@ public enum RenderPasses {
                 batchUniforms.viewMatrix = viewMatrix
                 batchUniforms.modelViewMatrix = viewMatrix
                 batchUniforms.normalMatrix = matrix_identity_float3x3
-                batchUniforms.cameraPosition = SceneRootTransform.shared.effectiveCameraPosition(cameraComponent.localPosition)
+                batchUniforms.cameraPosition = shadingCameraPosition(cameraComponent)
                 batchUniforms.projectionMatrix = renderInfo.perspectiveSpace
 
                 for batchGroup in batchGroups {
@@ -2130,7 +2130,7 @@ public enum RenderPasses {
         batchUniforms.viewMatrix = viewMatrix
         batchUniforms.modelViewMatrix = modelViewMatrix
         batchUniforms.normalMatrix = normalMatrix
-        batchUniforms.cameraPosition = SceneRootTransform.shared.effectiveCameraPosition(cameraComponent.localPosition)
+        batchUniforms.cameraPosition = shadingCameraPosition(cameraComponent)
         batchUniforms.projectionMatrix = renderInfo.perspectiveSpace
 
         // Render only batch groups that contain at least one visible entity.
@@ -2473,7 +2473,7 @@ public enum RenderPasses {
                 batchUniforms.viewMatrix = viewMatrix
                 batchUniforms.modelViewMatrix = simd_mul(viewMatrix, modelMatrix)
                 batchUniforms.normalMatrix = matrix3x3_upper_left(modelMatrix).inverse.transpose
-                batchUniforms.cameraPosition = SceneRootTransform.shared.effectiveCameraPosition(cameraComponent.localPosition)
+                batchUniforms.cameraPosition = shadingCameraPosition(cameraComponent)
                 batchUniforms.projectionMatrix = renderInfo.perspectiveSpace
 
                 for batchGroup in visibleBatchGroups {
@@ -2618,7 +2618,7 @@ public enum RenderPasses {
         lightQuadEncoder.setVertexBuffer(bufferResources.quadVerticesBuffer, offset: 0, index: 0)
         lightQuadEncoder.setVertexBuffer(bufferResources.quadTexCoordsBuffer, offset: 0, index: 1)
 
-        var effectiveCamPos = SceneRootTransform.shared.effectiveCameraPosition(cameraComponent.localPosition)
+        var effectiveCamPos = shadingCameraPosition(cameraComponent)
         lightQuadEncoder.setFragmentBytes(&effectiveCamPos, length: MemoryLayout<simd_float3>.stride, index: Int(lightPassCameraPositionIndex.rawValue))
 
         var csmUniforms = shadowSystem.makeUniforms()
@@ -3279,7 +3279,7 @@ public enum RenderPasses {
         renderEncoder.setVertexBuffer(bufferResources.quadVerticesBuffer, offset: 0, index: 0)
         renderEncoder.setVertexBuffer(bufferResources.quadTexCoordsBuffer, offset: 0, index: 1)
 
-        var effectiveCamPos = SceneRootTransform.shared.effectiveCameraPosition(cameraComponent.localPosition)
+        var effectiveCamPos = shadingCameraPosition(cameraComponent)
         renderEncoder.setFragmentBytes(&effectiveCamPos, length: MemoryLayout<simd_float3>.stride, index: Int(lightPassCameraPositionIndex.rawValue))
 
         // CSM uniforms: pack all cascade matrices + split distances into one struct.
@@ -3638,7 +3638,7 @@ public enum RenderPasses {
             index: Int(transparencyPassLightOrthoViewMatrixIndex.rawValue)
         )
 
-        var effectiveCamPos = SceneRootTransform.shared.effectiveCameraPosition(cameraComponent.localPosition)
+        var effectiveCamPos = shadingCameraPosition(cameraComponent)
         renderEncoder.setFragmentBytes(
             &effectiveCamPos,
             length: MemoryLayout<simd_float3>.stride,
@@ -3725,7 +3725,7 @@ public enum RenderPasses {
             index: Int(transparencyPassIBLRotationAngleIndex.rawValue)
         )
 
-        let effectiveCameraPosition = SceneRootTransform.shared.effectiveCameraPosition(cameraComponent.localPosition)
+        let effectiveCameraPosition = shadingCameraPosition(cameraComponent)
 
         // What every draw of the pass shares, read once.
         let sceneSnapshot = RenderSceneSnapshot()
@@ -3996,7 +3996,7 @@ public enum RenderPasses {
         guard !shellEntityIds.isEmpty else { return }
 
         let viewMatrix = SceneRootTransform.shared.effectiveViewMatrix(cameraComponent.viewSpace)
-        let effectiveCameraPosition = SceneRootTransform.shared.effectiveCameraPosition(cameraComponent.localPosition)
+        let effectiveCameraPosition = shadingCameraPosition(cameraComponent)
 
         let descriptor = MTLRenderPassDescriptor()
         descriptor.renderTargetWidth = depthTexture.width
@@ -4089,7 +4089,7 @@ public enum RenderPasses {
             return
         }
         let viewMatrix = SceneRootTransform.shared.effectiveViewMatrix(cameraComponent.viewSpace)
-        let effectiveCameraPosition = SceneRootTransform.shared.effectiveCameraPosition(cameraComponent.localPosition)
+        let effectiveCameraPosition = shadingCameraPosition(cameraComponent)
         wireframeRenderStateLock.lock()
         let wireframeSettings = wireframeRenderState
         wireframeRenderStateLock.unlock()
@@ -4278,7 +4278,7 @@ public enum RenderPasses {
             return
         }
         let viewMatrix = SceneRootTransform.shared.effectiveViewMatrix(cameraComponent.viewSpace)
-        let effectiveCameraPosition = SceneRootTransform.shared.effectiveCameraPosition(cameraComponent.localPosition)
+        let effectiveCameraPosition = shadingCameraPosition(cameraComponent)
 
         let sceneSnapshot = RenderSceneSnapshot()
         let batchingEnabled = BatchingSystem.shared.isEnabled()

Sources/UntoldEngine/Renderer/RenderResources.swift

@@ -82,6 +82,26 @@ public struct RenderInfo {
     public var xrEye0Projection: simd_float4x4 = matrix_identity_float4x4
     public var xrEye1View: simd_float4x4 = matrix_identity_float4x4
     public var xrEye1Projection: simd_float4x4 = matrix_identity_float4x4
+    /// The position of the eye `renderXR` is drawing, in the same space as
+    /// CameraComponent.localPosition; nil outside XR. Shading reads it through
+    /// shadingCameraPosition so each eye gets its own view vector (specular highlights,
+    /// Fresnel), while streaming, LOD and culling keep the head-centre camera position.
+    public var xrEyeCameraPosition: simd_float3?
+}
+
+/// Where an eye is, from its view matrix (the inverse of the eye's camera transform).
+@inline(__always)
+func eyePosition(fromViewMatrix viewMatrix: simd_float4x4) -> simd_float3 {
+    let eyeTransform = simd_inverse(viewMatrix)
+    return simd_float3(eyeTransform.columns.3.x, eyeTransform.columns.3.y, eyeTransform.columns.3.z)
+}
+
+/// The camera position shading uses: in XR, the eye being drawn, so highlights and
+/// Fresnel sit at the right depth for each eye; otherwise the camera's own position.
+/// Already folded through SceneRootTransform like every other shading camera position.
+@inline(__always)
+func shadingCameraPosition(_ cameraComponent: CameraComponent) -> simd_float3 {
+    SceneRootTransform.shared.effectiveCameraPosition(renderInfo.xrEyeCameraPosition ?? cameraComponent.localPosition)
 }
 
 @inline(__always)

Sources/UntoldEngine/Renderer/UntoldEngine.swift

@@ -693,6 +693,8 @@ public class UntoldRenderer: NSObject, MTKViewDelegate {
     }
 
     public func draw(in view: MTKView) {
+        // Not an XR eye: shading uses the camera's own position.
+        renderInfo.xrEyeCameraPosition = nil
         if pendingResize {
             initSizeableResources()
             pendingResize = false
@@ -835,6 +837,10 @@ public class UntoldRenderer: NSObject, MTKViewDelegate {
         }
 
         cameraComponent.viewSpace = viewMatrix
+        // The eye's own position: the view matrix is the inverse of the eye's camera
+        // transform. CameraComponent.localPosition stays at the head centre (see
+        // setXRCameraWorldPosition), which streaming wants and shading does not.
+        renderInfo.xrEyeCameraPosition = eyePosition(fromViewMatrix: viewMatrix)
 
         // Save this eye's view-projection for next frame's per-eye HZB culling, and the raw
         // view and projection it was built from for the Gaussian chunk cull, which folds in

Tests/UntoldEngineTests/ShadingCameraPositionTests.swift

@@ -0,0 +1,82 @@
+//
+//  ShadingCameraPositionTests.swift
+//
+//
+// Copyright (C) Untold Engine Studios
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+import simd
+@testable import UntoldEngine
+import XCTest
+
+@MainActor
+final class ShadingCameraPositionTests: XCTestCase {
+    let srt = SceneRootTransform.shared
+    let eps: Float = 1e-4
+
+    override func setUp() async throws {
+        srt.position = .zero
+        srt.rotation = simd_quatf()
+        srt.scale = .one
+        srt.updateIfNeeded()
+        renderInfo.xrEyeCameraPosition = nil
+    }
+
+    override func tearDown() async throws {
+        srt.position = .zero
+        srt.updateIfNeeded()
+        renderInfo.xrEyeCameraPosition = nil
+    }
+
+    func testEyePositionIsTheTranslationOfTheInverseViewMatrix() {
+        // An eye 3.2 cm left of a head at (0, 1.6, 2), turned 30 degrees about Y.
+        let rotation = simd_float4x4(simd_quatf(angle: .pi / 6, axis: simd_float3(0, 1, 0)))
+        var eyeTransform = rotation
+        eyeTransform.columns.3 = simd_float4(-0.032, 1.6, 2, 1)
+        let viewMatrix = simd_inverse(eyeTransform)
+
+        let position = eyePosition(fromViewMatrix: viewMatrix)
+
+        XCTAssertEqual(position.x, -0.032, accuracy: eps)
+        XCTAssertEqual(position.y, 1.6, accuracy: eps)
+        XCTAssertEqual(position.z, 2, accuracy: eps)
+    }
+
+    func testOutsideXRShadingUsesTheCameraPosition() {
+        let camera = CameraComponent()
+        camera.localPosition = simd_float3(1, 2, 3)
+
+        let position = shadingCameraPosition(camera)
+
+        XCTAssertEqual(position.x, 1, accuracy: eps)
+        XCTAssertEqual(position.y, 2, accuracy: eps)
+        XCTAssertEqual(position.z, 3, accuracy: eps)
+    }
+
+    func testInXRShadingUsesTheEyeBeingDrawnNotTheHeadCentre() {
+        let camera = CameraComponent()
+        camera.localPosition = simd_float3(0, 1.6, 0)
+        renderInfo.xrEyeCameraPosition = simd_float3(0.032, 1.6, 0)
+
+        let position = shadingCameraPosition(camera)
+
+        XCTAssertEqual(position.x, 0.032, accuracy: eps)
+    }
+
+    func testTheEyePositionGoesThroughTheSceneRootLikeTheCameraDid() {
+        let camera = CameraComponent()
+        renderInfo.xrEyeCameraPosition = simd_float3(0.032, 1.6, 0)
+        srt.position = simd_float3(5, 0, 0)
+        srt.updateIfNeeded()
+
+        let expected = srt.effectiveCameraPosition(simd_float3(0.032, 1.6, 0))
+        let position = shadingCameraPosition(camera)
+
+        XCTAssertEqual(position.x, expected.x, accuracy: eps)
+        XCTAssertEqual(position.y, expected.y, accuracy: eps)
+        XCTAssertEqual(position.z, expected.z, accuracy: eps)
+    }
+}