[Backport 8.x.x] Fixed decal duplication when editing a prefab in context

com.unity.render-pipelines.high-definition/CHANGELOG.md

@@ -73,6 +73,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Fixed issue with depth pyramid generation and dynamic resolution.
 - Fixed a serialization issue, preventing quality level parameters to undo/redo and update scene view on change.
 - Fixed an issue where opening the look dev window with the light theme would make the window blink and eventually crash unity.
+- Fixed culling for decals when used in prefabs and edited in context.
 
 ### Changed
 - Shadowmask and realtime reflection probe property are hide in Quality settings

com.unity.render-pipelines.high-definition/Editor/Material/Decal/DecalProjectorEditor.cs

@@ -259,7 +259,7 @@ void DrawHandles()
                     {
                         // Smoothly update the decal image projected
                         Matrix4x4 sizeOffset = Matrix4x4.Translate(decalProjector.decalOffset) * Matrix4x4.Scale(decalProjector.decalSize);
-                        DecalSystem.instance.UpdateCachedData(decalProjector.position, decalProjector.rotation, sizeOffset, decalProjector.drawDistance, decalProjector.fadeScale, decalProjector.uvScaleBias, decalProjector.affectsTransparency, decalProjector.Handle, decalProjector.gameObject.layer, HDUtils.GetSceneCullingMaskFromGameObject(decalProjector.gameObject), decalProjector.fadeFactor);
+                        DecalSystem.instance.UpdateCachedData(decalProjector.position, decalProjector.rotation, sizeOffset, decalProjector.drawDistance, decalProjector.fadeScale, decalProjector.uvScaleBias, decalProjector.affectsTransparency, decalProjector.Handle, decalProjector.gameObject.layer, decalProjector.gameObject.sceneCullingMask, decalProjector.fadeFactor);
                     }
                 }
             }

com.unity.render-pipelines.high-definition/Runtime/Material/Decal/DecalProjector.Migration.cs

@@ -37,7 +37,7 @@ enum Version
                 if (decal.m_Handle != null)
                     DecalSystem.instance.RemoveDecal(decal.m_Handle);
                 Matrix4x4 sizeOffset = Matrix4x4.Translate(decal.decalOffset) * Matrix4x4.Scale(decal.decalSize);
-                decal.m_Handle = DecalSystem.instance.AddDecal(decal.position, decal.rotation, Vector3.one, sizeOffset, decal.m_DrawDistance, decal.m_FadeScale, decal.uvScaleBias, decal.m_AffectsTransparency, decal.m_Material, decal.gameObject.layer, HDUtils.GetSceneCullingMaskFromGameObject(decal.gameObject), decal.m_FadeFactor);
+                decal.m_Handle = DecalSystem.instance.AddDecal(decal.position, decal.rotation, Vector3.one, sizeOffset, decal.m_DrawDistance, decal.m_FadeScale, decal.uvScaleBias, decal.m_AffectsTransparency, decal.m_Material, decal.gameObject.layer, decal.gameObject.sceneCullingMask, decal.m_FadeFactor);
             }),
             MigrationStep.New(Version.FixPivotPosition, (DecalProjector decal) =>
             {
@@ -69,7 +69,7 @@ enum Version
                 if (decal.m_Handle != null)
                     DecalSystem.instance.RemoveDecal(decal.m_Handle);
                 Matrix4x4 sizeOffset = Matrix4x4.Translate(decal.decalOffset) * Matrix4x4.Scale(decal.decalSize);
-                decal.m_Handle = DecalSystem.instance.AddDecal(decal.position, decal.rotation, Vector3.one, sizeOffset, decal.m_DrawDistance, decal.m_FadeScale, decal.uvScaleBias, decal.m_AffectsTransparency, decal.m_Material, decal.gameObject.layer, HDUtils.GetSceneCullingMaskFromGameObject(decal.gameObject), decal.m_FadeFactor);
+                decal.m_Handle = DecalSystem.instance.AddDecal(decal.position, decal.rotation, Vector3.one, sizeOffset, decal.m_DrawDistance, decal.m_FadeScale, decal.uvScaleBias, decal.m_AffectsTransparency, decal.m_Material, decal.gameObject.layer, decal.gameObject.sceneCullingMask, decal.m_FadeFactor);
             })
         );
 

com.unity.render-pipelines.high-definition/Runtime/Material/Decal/DecalProjector.cs

@@ -238,7 +238,7 @@ void OnEnable()
             }
 
             Matrix4x4 sizeOffset = Matrix4x4.Translate(decalOffset) * Matrix4x4.Scale(decalSize);
-            m_Handle = DecalSystem.instance.AddDecal(position, rotation, Vector3.one, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Material, gameObject.layer, HDUtils.GetSceneCullingMaskFromGameObject(gameObject), m_FadeFactor);
+            m_Handle = DecalSystem.instance.AddDecal(position, rotation, Vector3.one, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Material, gameObject.layer, gameObject.sceneCullingMask, m_FadeFactor);
             m_OldMaterial = m_Material;
 
 #if UNITY_EDITOR
@@ -260,7 +260,13 @@ void UpdateDecalVisibility()
             else if (m_Handle == null)
             {
                 Matrix4x4 sizeOffset = Matrix4x4.Translate(decalOffset) * Matrix4x4.Scale(decalSize);
-                m_Handle = DecalSystem.instance.AddDecal(position, rotation, Vector3.one, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Material, gameObject.layer, HDUtils.GetSceneCullingMaskFromGameObject(gameObject), m_FadeFactor);
+                m_Handle = DecalSystem.instance.AddDecal(position, rotation, Vector3.one, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Material, gameObject.layer, gameObject.sceneCullingMask, m_FadeFactor);
+            }
+            else
+            {
+                // Scene culling mask may have changed.
+                Matrix4x4 sizeOffset = Matrix4x4.Translate(decalOffset) * Matrix4x4.Scale(decalSize);
+                DecalSystem.instance.UpdateCachedData(position, rotation, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Handle, gameObject.layer, gameObject.sceneCullingMask, m_FadeFactor);
             }
         }
 #endif
@@ -299,7 +305,7 @@ internal void OnValidate()
 
                     if (m_Material != null)
                     {
-                        m_Handle = DecalSystem.instance.AddDecal(position, rotation, Vector3.one, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Material, gameObject.layer, HDUtils.GetSceneCullingMaskFromGameObject(gameObject), m_FadeFactor);
+                        m_Handle = DecalSystem.instance.AddDecal(position, rotation, Vector3.one, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Material, gameObject.layer, gameObject.sceneCullingMask, m_FadeFactor);
 
                         if (!DecalSystem.IsHDRenderPipelineDecal(m_Material.shader)) // non HDRP/decal shaders such as shader graph decal do not affect transparency
                         {
@@ -317,7 +323,7 @@ internal void OnValidate()
                 }
                 else // no material change, just update whatever else changed
                 {
-                    DecalSystem.instance.UpdateCachedData(position, rotation, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Handle, gameObject.layer, HDUtils.GetSceneCullingMaskFromGameObject(gameObject), m_FadeFactor);
+                    DecalSystem.instance.UpdateCachedData(position, rotation, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Handle, gameObject.layer, gameObject.sceneCullingMask, m_FadeFactor);
                 }
             }
         }
@@ -329,7 +335,7 @@ void Update() // only run in editor
             {
                 Matrix4x4 sizeOffset = Matrix4x4.Translate(decalOffset) * Matrix4x4.Scale(decalSize);
                 m_Layer = gameObject.layer;
-                DecalSystem.instance.UpdateCachedData(position, rotation, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Handle, gameObject.layer, HDUtils.GetSceneCullingMaskFromGameObject(gameObject), m_FadeFactor);
+                DecalSystem.instance.UpdateCachedData(position, rotation, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Handle, gameObject.layer, gameObject.sceneCullingMask, m_FadeFactor);
             }
         }
 #endif
@@ -341,7 +347,7 @@ void LateUpdate()
                 if (transform.hasChanged == true)
                 {
                     Matrix4x4 sizeOffset = Matrix4x4.Translate(decalOffset) * Matrix4x4.Scale(decalSize);
-                    DecalSystem.instance.UpdateCachedData(position, rotation, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Handle, gameObject.layer, HDUtils.GetSceneCullingMaskFromGameObject(gameObject), m_FadeFactor);
+                    DecalSystem.instance.UpdateCachedData(position, rotation, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Handle, gameObject.layer, gameObject.sceneCullingMask, m_FadeFactor);
                     transform.hasChanged = false;
                 }
             }

com.unity.render-pipelines.high-definition/Runtime/Material/Decal/DecalSystem.cs

@@ -496,6 +496,7 @@ public DecalHandle AddDecal(Matrix4x4 localToWorld, Quaternion rotation, Matrix4
                     m_CachedDrawDistances = newCachedDrawDistances;
                     m_CachedUVScaleBias = newCachedUVScaleBias;
                     m_CachedAffectsTransparency = newCachedAffectsTransparency;
+                    m_CachedLayerMask = newCachedLayerMask;
                     m_CachedSceneLayerMask = newCachedSceneLayerMask;
                     m_CachedFadeFactor = newCachedFadeFactor;
                 }

com.unity.render-pipelines.high-definition/Runtime/RenderPipeline/Utility/HDUtils.cs

@@ -942,15 +942,6 @@ internal static UInt64 GetSceneCullingMaskFromCamera(Camera camera)
 
         }
 
-        internal static ulong GetSceneCullingMaskFromGameObject(GameObject go)
-        {
-#if UNITY_EDITOR
-            return (go.scene == null) ? ~(ulong)0 : EditorSceneManager.GetSceneCullingMask(go.scene);
-#else
-            return 0;
-#endif
-        }
-
         internal static HDAdditionalCameraData TryGetAdditionalCameraDataOrDefault(Camera camera)
         {
             if (camera == null || camera.Equals(null))