renderer: compute normalmap from heightmap when missing normalmap, aka bumpmap

illwieckz · illwieckz · commit 7b714753e9ef · 2019-12-29T09:06:52.000+01:00
compute normalmap from heightmap when missing normalmap, also known as bumpmapping

two algorithms available:

- one naive that does the job, uses 3 samples
- one advanced that does better job, uses 8 samples (sobel operation)

a new cvar is added: r_sobelFiltering, which would be used
to disable or enable various sobel filtering operation including this one

the engine is likely to implement other sobel filters in the future,
for example DarkPlaces has a postprocessing effect that does such kind of compute,
maybe Dæmon based games will want such kind of effect, in this case
a common cvar to enable or disable various sobel compute does not look bad

the r_sobelFiltering cvar is enabled by default
diff --git a/src/engine/renderer/gl_shader.cpp b/src/engine/renderer/gl_shader.cpp
@@ -562,6 +562,11 @@ static std::string GenEngineConstants() {
 		AddConst( str, "MAX_GLSL_BONES", 4 );
 	}
 
+	if ( r_sobelFiltering->integer )
+	{
+		AddDefine( str, "r_sobelFiltering", 1 );
+	}
+
 	if ( r_wrapAroundLighting->value )
 		AddConst( str, "r_WrapAroundLighting", r_wrapAroundLighting->value );
 
@@ -1472,6 +1477,7 @@ GLShader_lightMapping::GLShader_lightMapping( GLShaderManager *manager ) :
 	GLDeformStage( this ),
 	GLCompileMacro_USE_DELUXE_MAPPING( this ),
 	GLCompileMacro_USE_HEIGHTMAP_IN_NORMALMAP( this ),
+	GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP( this ),
 	GLCompileMacro_USE_PARALLAX_MAPPING( this ),
 	GLCompileMacro_USE_PHYSICAL_SHADING( this )
 {
@@ -1528,6 +1534,7 @@ GLShader_vertexLighting_DBS_entity::GLShader_vertexLighting_DBS_entity( GLShader
 	GLCompileMacro_USE_VERTEX_SKINNING( this ),
 	GLCompileMacro_USE_VERTEX_ANIMATION( this ),
 	GLCompileMacro_USE_HEIGHTMAP_IN_NORMALMAP( this ),
+	GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP( this ),
 	GLCompileMacro_USE_PARALLAX_MAPPING( this ),
 	GLCompileMacro_USE_REFLECTIVE_SPECULAR( this ),
 	GLCompileMacro_USE_PHYSICAL_SHADING( this )
@@ -1588,6 +1595,7 @@ GLShader_vertexLighting_DBS_world::GLShader_vertexLighting_DBS_world( GLShaderMa
 	u_Lights( this ),
 	GLDeformStage( this ),
 	GLCompileMacro_USE_HEIGHTMAP_IN_NORMALMAP( this ),
+	GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP( this ),
 	GLCompileMacro_USE_PARALLAX_MAPPING( this ),
 	GLCompileMacro_USE_PHYSICAL_SHADING( this )
 {
@@ -1648,6 +1656,7 @@ GLShader_forwardLighting_omniXYZ::GLShader_forwardLighting_omniXYZ( GLShaderMana
 	GLCompileMacro_USE_VERTEX_SKINNING( this ),
 	GLCompileMacro_USE_VERTEX_ANIMATION( this ),
 	GLCompileMacro_USE_HEIGHTMAP_IN_NORMALMAP( this ),
+	GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP( this ),
 	GLCompileMacro_USE_PARALLAX_MAPPING( this ),
 	GLCompileMacro_USE_SHADOWING( this )
 {
@@ -1708,6 +1717,7 @@ GLShader_forwardLighting_projXYZ::GLShader_forwardLighting_projXYZ( GLShaderMana
 	GLCompileMacro_USE_VERTEX_SKINNING( this ),
 	GLCompileMacro_USE_VERTEX_ANIMATION( this ),
 	GLCompileMacro_USE_HEIGHTMAP_IN_NORMALMAP( this ),
+	GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP( this ),
 	GLCompileMacro_USE_PARALLAX_MAPPING( this ),
 	GLCompileMacro_USE_SHADOWING( this )
 {
@@ -1771,6 +1781,7 @@ GLShader_forwardLighting_directionalSun::GLShader_forwardLighting_directionalSun
 	GLCompileMacro_USE_VERTEX_SKINNING( this ),
 	GLCompileMacro_USE_VERTEX_ANIMATION( this ),
 	GLCompileMacro_USE_HEIGHTMAP_IN_NORMALMAP( this ),
+	GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP( this ),
 	GLCompileMacro_USE_PARALLAX_MAPPING( this ),
 	GLCompileMacro_USE_SHADOWING( this )
 {
@@ -1855,6 +1866,7 @@ GLShader_reflection::GLShader_reflection( GLShaderManager *manager ):
 	GLCompileMacro_USE_VERTEX_SKINNING( this ),
 	GLCompileMacro_USE_VERTEX_ANIMATION( this ),
 	GLCompileMacro_USE_HEIGHTMAP_IN_NORMALMAP( this ),
+	GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP( this ),
 	GLCompileMacro_USE_PARALLAX_MAPPING( this )
 {
 }
diff --git a/src/engine/renderer/gl_shader.h b/src/engine/renderer/gl_shader.h
@@ -785,6 +785,7 @@ class GLCompileMacro
 	  USE_TCGEN_LIGHTMAP,
 	  USE_DELUXE_MAPPING,
 	  USE_HEIGHTMAP_IN_NORMALMAP,
+	  USE_NORMALMAP_FROM_HEIGHTMAP,
 	  USE_PARALLAX_MAPPING,
 	  USE_REFLECTIVE_SPECULAR,
 	  USE_SHADOWING,
@@ -1035,6 +1036,31 @@ class GLCompileMacro_USE_HEIGHTMAP_IN_NORMALMAP :
 	}
 };
 
+class GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP :
+	GLCompileMacro
+{
+public:
+	GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP( GLShader *shader ) :
+		GLCompileMacro( shader )
+	{
+	}
+
+	const char *GetName() const
+	{
+		return "USE_NORMALMAP_FROM_HEIGHTMAP";
+	}
+
+	EGLCompileMacro GetType() const
+	{
+		return EGLCompileMacro::USE_NORMALMAP_FROM_HEIGHTMAP;
+	}
+
+	void SetNormalMapFromHeightMap( bool enable )
+	{
+		SetMacro( enable );
+	}
+};
+
 class GLCompileMacro_USE_PARALLAX_MAPPING :
 	GLCompileMacro
 {
@@ -2163,6 +2189,7 @@ class GLShader_lightMapping :
 	public GLDeformStage,
 	public GLCompileMacro_USE_DELUXE_MAPPING,
 	public GLCompileMacro_USE_HEIGHTMAP_IN_NORMALMAP,
+	public GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP,
 	public GLCompileMacro_USE_PARALLAX_MAPPING,
 	public GLCompileMacro_USE_PHYSICAL_SHADING
 {
@@ -2196,6 +2223,7 @@ class GLShader_vertexLighting_DBS_entity :
 	public GLCompileMacro_USE_VERTEX_SKINNING,
 	public GLCompileMacro_USE_VERTEX_ANIMATION,
 	public GLCompileMacro_USE_HEIGHTMAP_IN_NORMALMAP,
+	public GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP,
 	public GLCompileMacro_USE_PARALLAX_MAPPING,
 	public GLCompileMacro_USE_REFLECTIVE_SPECULAR,
 	public GLCompileMacro_USE_PHYSICAL_SHADING
@@ -2228,6 +2256,7 @@ class GLShader_vertexLighting_DBS_world :
 	public u_Lights,
 	public GLDeformStage,
 	public GLCompileMacro_USE_HEIGHTMAP_IN_NORMALMAP,
+	public GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP,
 	public GLCompileMacro_USE_PARALLAX_MAPPING,
 	public GLCompileMacro_USE_PHYSICAL_SHADING
 {
@@ -2266,6 +2295,7 @@ class GLShader_forwardLighting_omniXYZ :
 	public GLCompileMacro_USE_VERTEX_SKINNING,
 	public GLCompileMacro_USE_VERTEX_ANIMATION,
 	public GLCompileMacro_USE_HEIGHTMAP_IN_NORMALMAP,
+	public GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP,
 	public GLCompileMacro_USE_PARALLAX_MAPPING,
 	public GLCompileMacro_USE_SHADOWING //,
 {
@@ -2305,6 +2335,7 @@ class GLShader_forwardLighting_projXYZ :
 	public GLCompileMacro_USE_VERTEX_SKINNING,
 	public GLCompileMacro_USE_VERTEX_ANIMATION,
 	public GLCompileMacro_USE_HEIGHTMAP_IN_NORMALMAP,
+	public GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP,
 	public GLCompileMacro_USE_PARALLAX_MAPPING,
 	public GLCompileMacro_USE_SHADOWING //,
 {
@@ -2346,6 +2377,7 @@ class GLShader_forwardLighting_directionalSun :
 	public GLCompileMacro_USE_VERTEX_SKINNING,
 	public GLCompileMacro_USE_VERTEX_ANIMATION,
 	public GLCompileMacro_USE_HEIGHTMAP_IN_NORMALMAP,
+	public GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP,
 	public GLCompileMacro_USE_PARALLAX_MAPPING,
 	public GLCompileMacro_USE_SHADOWING //,
 {
@@ -2395,6 +2427,7 @@ class GLShader_reflection :
 	public GLCompileMacro_USE_VERTEX_SKINNING,
 	public GLCompileMacro_USE_VERTEX_ANIMATION,
 	public GLCompileMacro_USE_HEIGHTMAP_IN_NORMALMAP,
+	public GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP,
 	public GLCompileMacro_USE_PARALLAX_MAPPING
 {
 public:
diff --git a/src/engine/renderer/glsl_source/reliefMapping_fp.glsl b/src/engine/renderer/glsl_source/reliefMapping_fp.glsl
@@ -29,10 +29,11 @@ uniform sampler2D	u_NormalMap;
 uniform vec3        u_NormalScale;
 #endif // r_normalMapping
 
-#if defined(USE_PARALLAX_MAPPING)
-#if !defined(USE_HEIGHTMAP_IN_NORMALMAP)
+#if (defined(USE_PARALLAX_MAPPING) && !defined(USE_HEIGHTMAP_IN_NORMALMAP)) || defined(USE_NORMALMAP_FROM_HEIGHTMAP)
 uniform sampler2D	u_HeightMap;
-#endif // !USE_HEIGHTMAP_IN_NORMALMAP
+#endif // (USE_PARALLAX_MAPPING && !USE_HEIGHTMAP_IN_NORMALMAP) || USE_NORMALMAP_FROM_HEIGHTMAP
+
+#if defined(USE_PARALLAX_MAPPING)
 uniform float       u_ParallaxDepthScale;
 uniform float       u_ParallaxOffsetBias;
 #endif // USE_PARALLAX_MAPPING
@@ -43,7 +44,142 @@ vec3 NormalInTangentSpace(vec2 texNormal)
 	vec3 normal;
 
 #if defined(r_normalMapping)
-#if defined(USE_HEIGHTMAP_IN_NORMALMAP)
+#if defined(USE_NORMALMAP_FROM_HEIGHTMAP)
+	// inspired from various examples read on many places, in order to make the most complete code possible.
+	//
+	// major inspiration was this 2007 post by jollyjeffers:
+	//   https://www.gamedev.net/forums/topic/475213-generate-normal-map-from-heightmap-algorithm/4117038/
+	//
+	// various pages like this one recommends doing abs() on sample read in case of float image format:
+	//   http://www.catalinzima.com/2008/01/converting-displacement-maps-into-normal-maps/
+	// other people does the same there:
+	//   https://community.khronos.org/t/heightmap-to-normalmap/58862
+	// and one tells he had issues by not doing it:
+	//   https://gamedev.stackexchange.com/questions/165575²/calculating-normal-map-from-height-map-using-sobel-operator
+	// other people say texture2D clamps it but it's safer to do this, unfortunately no one GLSL code was read for comparison
+	// since all the examples found were using other languages: C++, HLSL, etc.
+
+	vec2 pixelSize = 1.0 / vec2(textureSize(u_HeightMap, 0));
+
+#if defined(r_sobelFiltering)
+	// useful things to know:
+	//
+	// * it's required to get height samples (S) surrounding the current pixel (P) to do a sobel filter:
+	//
+	//   S S S
+	//   S P S
+	//   S S S
+	//
+	// * sobel X kernel is:
+	//
+	//   [ 1  0  -1 ]
+	//   [ 2  0  -2 ]
+	//   [ 1  0  -1 ]
+	//
+	//   see https://en.wikipedia.org/wiki/Sobel_operator#Formulation
+	//
+	// * sobel Y kernel is:
+	//
+	//   [  1  2  1 ]
+	//   [  0  0  0 ]
+	//   [ -1 -2 -1 ]
+	//
+	//   which is the rotated sobel X kernel
+	//
+	// * tangent space normals are +Z
+
+	ivec3 offsets;
+	mat3 xCoords;
+	mat3 yCoords;
+	mat3 heights;
+	mat3 sobel;
+
+	// components will be computed by accumylating values
+	normal = vec3(0.0, 0.0, 0.0);
+
+	// set offsets
+	offsets = ivec3(-1, 0, 1);
+
+	// set sobel X kernel
+	// beware, X is line and Y is column in this notation
+	sobel[0] = vec3( 1.0,  2.0,  1.0);
+	sobel[1] = vec3( 0.0,  0.0,  0.0);
+	sobel[2] = vec3(-1.0, -2.0, -1.0);
+
+	for (int i = 0; i < 3; i++)
+	{
+		for (int j = 0; j < 3; j++)
+		{
+			if (i != 1 && j != 1)
+			{
+				// compute coords
+				xCoords[i][j] = texNormal.x + pixelSize.x * offsets[i];
+				yCoords[i][j] = texNormal.y + pixelSize.y * offsets[j];
+
+				// get surrounding samples
+				heights[i][j] = abs(texture2D(u_HeightMap, vec2(xCoords[i][j], yCoords[i][j])).r);
+
+				if (i != 1)
+				{
+					// sobel computation for X component (use the X sobel kernel)
+					normal.x += heights[i][j] * sobel[i][j];
+				}
+
+				if (j != 1)
+				{
+					// sobel computation for Y component (use the rotated X sobel kernel as Y one)
+					normal.y += heights[i][j] * sobel[j][i];
+				}
+			}
+		}
+	}
+
+	// reconstruct Z component
+	normal.z = sqrt(1.0 - dot(normal.xy, normal.xy));
+
+	// HACK: prevent artifacts that may produce black normalmap pixels
+	// Xonotic's textures/map_glowplant/sand_bump triggers this bug
+	normal.z = clamp(normal.z, 0, 1);
+
+	normal.xy = 2.0 * normal.xy;
+#else // !r_sobelFiltering
+	// useful thing to know:
+	//
+	// * only three samples are required to determine two vectors
+	//   they would be used to generate the normal at this pixel:
+	//
+	//   P S
+	//   S
+	//
+	//   the pixel itself is used as sample
+
+	vec2 hOffsets;
+	vec2 vOffsets;
+	vec2 hCoords;
+	vec2 vCoords;
+	vec3 heights;
+	vec3 xVector;
+	vec3 yVector;
+
+	// set horizontal and vertical offsets
+	hOffsets = vec2(pixelSize.x, 0.0);
+	vOffsets = vec2(0.0, pixelSize.y);
+
+	// compute coords
+	hCoords = vec2(texNormal.x + hOffsets.x, texNormal.y + hOffsets.y);
+	vCoords = vec2(texNormal.x + vOffsets.x, texNormal.y + vOffsets.y);
+
+	// get samples
+	heights.x = texture2D(u_HeightMap, texNormal).r; // no offset
+	heights.y = texture2D(u_HeightMap, hCoords).r;
+	heights.z = texture2D(u_HeightMap, vCoords).r;
+
+	xVector = vec3(hOffsets.x, vOffsets.x, heights.y - heights.x);
+	yVector = vec3(hOffsets.y, vOffsets.y, heights.z - heights.x);
+
+	normal = cross(xVector, yVector);
+#endif // !r_sobelFiltering
+#elif defined(USE_HEIGHTMAP_IN_NORMALMAP)
 	// alpha channel contains the height map so do not try to reconstruct normal map from it
 	normal = texture2D(u_NormalMap, texNormal).rgb;
 	normal = 2.0 * normal - 1.0;
diff --git a/src/engine/renderer/tr_init.cpp b/src/engine/renderer/tr_init.cpp
@@ -172,6 +172,7 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 	cvar_t      *r_glowMapping;
 	cvar_t      *r_reflectionMapping;
 
+	cvar_t      *r_sobelFiltering;
 	cvar_t      *r_wrapAroundLighting;
 	cvar_t      *r_halfLambertLighting;
 	cvar_t      *r_rimLighting;
@@ -1204,6 +1205,7 @@ ScreenshotCmd screenshotPNGRegistration("screenshotPNG", ssFormat_t::SSF_PNG, "p
 		r_glowMapping = ri.Cvar_Get( "r_glowMapping", "1", CVAR_LATCH );
 		r_reflectionMapping = ri.Cvar_Get( "r_reflectionMapping", "0", CVAR_CHEAT );
 
+		r_sobelFiltering = ri.Cvar_Get( "r_sobelFiltering", "1", CVAR_LATCH );
 		r_wrapAroundLighting = ri.Cvar_Get( "r_wrapAroundLighting", "0.7", CVAR_CHEAT | CVAR_LATCH );
 		r_halfLambertLighting = ri.Cvar_Get( "r_halfLambertLighting", "1", CVAR_CHEAT | CVAR_LATCH );
 		r_rimLighting = ri.Cvar_Get( "r_rimLighting", "0",  CVAR_LATCH | CVAR_ARCHIVE );
diff --git a/src/engine/renderer/tr_local.h b/src/engine/renderer/tr_local.h
@@ -2862,6 +2862,7 @@ static inline void halfToFloat( const f16vec4_t in, vec4_t out )
 	extern cvar_t *r_glowMapping;
 	extern cvar_t *r_reflectionMapping;
 
+	extern cvar_t *r_sobelFiltering;
 	extern cvar_t *r_wrapAroundLighting;
 	extern cvar_t *r_halfLambertLighting;
 	extern cvar_t *r_rimLighting;
diff --git a/src/engine/renderer/tr_shade.cpp b/src/engine/renderer/tr_shade.cpp
@@ -1238,6 +1238,8 @@ static void Render_lightMapping( int stage )
 
 	gl_lightMappingShader->SetHeightMapInNormalMap( hasHeightMapInNormalMap );
 
+	gl_lightMappingShader->SetNormalMapFromHeightMap( hasHeightMap && !hasNormalMap );
+
 	gl_lightMappingShader->SetParallaxMapping( parallaxMapping );
 
 	tess.vboVertexSprite = false;
