Specular aliasing after using envAtlas

[scarletsky]

Here is my project's screenshot, it is built on top of v1.50.2(the environment lighting is using prefilteredCubemaps)

20260203120545_rec_-convert.mp4

But after I upgrade it to v1.51.7, v1.52.6, v1,53.4(the environment lighting migrated to envAtlas), the specular aliasing is really terrible:

20260203121233_rec_.mp4

---

I also try playcanvas online editor(v2.15.2 now), the specular aliasing is still terrible...
Here is the public project: https://playcanvas.com/editor/scene/2418205

20260203115835_rec_.mp4

v1.51.0 is released at 2021.12. Almost 4 years later, this bug is still here... 😭

[MAG-AdrianMeredith]

I've noticed this also, I wonder if its sampling too high quality

[Maksims]

This might have been related to toksvig normal mapping technique, I remember it was implemented to fight normals aliasing, but then there were some changes for it. Which might have affected our AA with normals.

[scarletsky]

@Maksims
Do you mean #4260 Toksvig specular doesn't work ?
But my project does not use normal maps. It is geometric normal.

[mvaligursky]

It's definitely worth reviewing, it seems significantly worse.
TAA is not a bad solution for these cases (when suitable): https://playcanvas.vercel.app/#/graphics/taa

[Maksims]

But my project does not use normal maps. It is geometric normal.

Oh, I've assumed it was normal map based, I can see in a project that it is geometry indeed.

[scarletsky]

Hi, guys. I did some research on v1.51.7.

Based on Algorithms for the Detection and Elimination of Specular Aliasing, ChatGPT told me to change reflectionEnvPS:

     vec2 dy2 = dFdy(uv2);
 
     // calculate min of both sets of dF to handle discontinuity at the azim edge
-    float maxd = min(max(dot(dx, dx), dot(dy, dy)), max(dot(dx2, dx2), dot(dy2, dy2)));
+    float maxd = max(max(dot(dx, dx), dot(dy, dy)), max(dot(dx2, dx2), dot(dy2, dy2)));
 
     return clamp(0.5 * log2(maxd) - 1.0, 0.0, 6.0);
 }
@@ -24,34 +25,38 @@ vec3 calcReflection(vec3 tReflDirW, float tGlossiness) {
     vec3 dir = cubeMapProject(tReflDirW) * vec3(-1.0, 1.0, 1.0);
     vec2 uv = toSphericalUv(dir);
 
-    // calculate roughness level
-    float level = saturate(1.0 - tGlossiness) * 5.0;
-    float ilevel = floor(level);
+    float levelBrdf = saturate(1.0 - tGlossiness) * 5.0;
+    float levelMin  = shinyMipLevel(uv * atlasSize);
+    float levelFinal = max(levelBrdf, levelMin);
 
-    // accessing the shiny (top level) reflection - perform manual mipmap lookup
-    float level2 = shinyMipLevel(uv * atlasSize);
-    float ilevel2 = floor(level2);
+    float ilevel = floor(levelFinal);
 
     vec2 uv0, uv1;
     float weight;
-    if (ilevel == 0.0) {
+
+    if (levelFinal < 1.0) {
+        float ilevel2 = floor(levelFinal);
         uv0 = mapMip(uv, ilevel2);
         uv1 = mapMip(uv, ilevel2 + 1.0);
-        weight = level2 - ilevel2;
+        weight = levelFinal - ilevel2;
+
+        vec3 linearA = $DECODE(texture2D(texture_envAtlas, uv0));
+        vec3 linearB = $DECODE(texture2D(texture_envAtlas, uv1));
+        return processEnvironment(mix(linearA, linearB, weight));
     } else {
-        // accessing rough reflection - just sample the same part twice
-        uv0 = uv1 = mapRoughnessUv(uv, ilevel);
-        weight = 0.0;
+        vec2 uva = mapRoughnessUv(uv, ilevel);
+        vec2 uvb = mapRoughnessUv(uv, ilevel + 1.0);
+        weight = levelFinal - ilevel;
+
+        vec3 linearA = $DECODE(texture2D(texture_envAtlas, uva));
+        vec3 linearB = $DECODE(texture2D(texture_envAtlas, uvb));
+        return processEnvironment(mix(linearA, linearB, weight));
     }
-
-    vec3 linearA = $DECODE(texture2D(texture_envAtlas, uv0));
-    vec3 linearB = $DECODE(texture2D(texture_envAtlas, uv1));
-    vec3 linear0 = mix(linearA, linearB, weight);
-    vec3 linear1 = $DECODE(texture2D(texture_envAtlas, mapRoughnessUv(uv, ilevel + 1.0)));
-
-    return processEnvironment(mix(linear0, linear1, level - ilevel));
 }

For convenience, I updated my code with this (I am not familar with the new shader chunks system, so I use the old system):

pc.shaderChunks.reflectionEnvPS = `
#ifndef ENV_ATLAS
#define ENV_ATLAS
uniform sampler2D texture_envAtlas;
#endif
uniform float material_reflectivity;

// calculate mip level for shiny reflection given equirect coords uv.
float shinyMipLevel(vec2 uv) {
    vec2 dx = dFdx(uv);
    vec2 dy = dFdy(uv);

    // calculate second dF at 180 degrees
    vec2 uv2 = vec2(fract(uv.x + 0.5), uv.y);
    vec2 dx2 = dFdx(uv2);
    vec2 dy2 = dFdy(uv2);

    // calculate min of both sets of dF to handle discontinuity at the azim edge
    float maxd = max(max(dot(dx, dx), dot(dy, dy)), max(dot(dx2, dx2), dot(dy2, dy2)));

    return clamp(0.5 * log2(maxd) - 1.0, 0.0, 5.0);
}

vec3 calcReflection(vec3 tReflDirW, float tGlossiness) {
    vec3 dir = cubeMapProject(tReflDirW) * vec3(-1.0, 1.0, 1.0);
    vec2 uv = toSphericalUv(dir);

    float levelBrdf = saturate(1.0 - tGlossiness) * 5.0;
    float levelMin  = shinyMipLevel(uv * atlasSize);
    float levelFinal = max(levelBrdf, levelMin);

    float ilevel = floor(levelFinal);

    vec2 uv0, uv1;
    float weight;

    if (levelFinal < 1.0) {
        float ilevel2 = floor(levelFinal);
        uv0 = mapMip(uv, ilevel2);
        uv1 = mapMip(uv, ilevel2 + 1.0);
        weight = levelFinal - ilevel2;

        vec3 linearA = $DECODE(texture2D(texture_envAtlas, uv0));
        vec3 linearB = $DECODE(texture2D(texture_envAtlas, uv1));
        return processEnvironment(mix(linearA, linearB, weight));
    } else {
        vec2 uva = mapRoughnessUv(uv, ilevel);
        vec2 uvb = mapRoughnessUv(uv, ilevel + 1.0);
        weight = levelFinal - ilevel;

        vec3 linearA = $DECODE(texture2D(texture_envAtlas, uva));
        vec3 linearB = $DECODE(texture2D(texture_envAtlas, uvb));
        return processEnvironment(mix(linearA, linearB, weight));
    }
}

void addReflection() {   
    dReflection += vec4(calcReflection(dReflDirW, dGlossiness), material_reflectivity);
}
`;

The results are great, but there are still a few artifacts:

20260204172919_rec_-convert.mp4

20260204173606_rec_-convert.mp4