renderer: compute normalmap from heightmap when missing normalmap, aka bumpmap

compute normalmap from heightmap when missing normalmap, also known as bumpmapping

two algorithms available:

- one naive that does the job, uses 3 samples
- one advanced that does better job, uses 8 samples (sobel operation)

a new cvar is added: r_sobelFiltering, which would be used
to disable or enable various sobel filtering operation including this one

the engine is likely to implement other sobel filters in the future,
for example DarkPlaces has a postprocessing effect that does such kind of compute,
maybe Dæmon based games will want such kind of effect, in this case
a common cvar to enable or disable various sobel compute does not look bad

the r_sobelFiltering cvar is enabled by default

Author: illwieckz

src/engine/renderer/gl_shader.cpp

@@ -573,6 +573,11 @@ static std::string GenEngineConstants() {
 		AddConst( str, "MAX_GLSL_BONES", 4 );
 	}
 
+	if ( r_sobelFiltering->integer )
+	{
+		AddDefine( str, "r_sobelFiltering", 1 );
+	}
+
 	if ( r_wrapAroundLighting->value )
 		AddConst( str, "r_WrapAroundLighting", r_wrapAroundLighting->value );
 
@@ -1496,6 +1501,7 @@ GLShader_lightMapping::GLShader_lightMapping( GLShaderManager *manager ) :
 	GLCompileMacro_USE_LIGHT_MAPPING( this ),
 	GLCompileMacro_USE_DELUXE_MAPPING( this ),
 	GLCompileMacro_USE_HEIGHTMAP_IN_NORMALMAP( this ),
+	GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP( this ),
 	GLCompileMacro_USE_PARALLAX_MAPPING( this ),
 	GLCompileMacro_USE_REFLECTIVE_SPECULAR( this ),
 	GLCompileMacro_USE_PHYSICAL_SHADING( this )
@@ -1560,6 +1566,7 @@ GLShader_forwardLighting_omniXYZ::GLShader_forwardLighting_omniXYZ( GLShaderMana
 	GLCompileMacro_USE_VERTEX_SKINNING( this ),
 	GLCompileMacro_USE_VERTEX_ANIMATION( this ),
 	GLCompileMacro_USE_HEIGHTMAP_IN_NORMALMAP( this ),
+	GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP( this ),
 	GLCompileMacro_USE_PARALLAX_MAPPING( this ),
 	GLCompileMacro_USE_SHADOWING( this )
 {
@@ -1620,6 +1627,7 @@ GLShader_forwardLighting_projXYZ::GLShader_forwardLighting_projXYZ( GLShaderMana
 	GLCompileMacro_USE_VERTEX_SKINNING( this ),
 	GLCompileMacro_USE_VERTEX_ANIMATION( this ),
 	GLCompileMacro_USE_HEIGHTMAP_IN_NORMALMAP( this ),
+	GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP( this ),
 	GLCompileMacro_USE_PARALLAX_MAPPING( this ),
 	GLCompileMacro_USE_SHADOWING( this )
 {
@@ -1683,6 +1691,7 @@ GLShader_forwardLighting_directionalSun::GLShader_forwardLighting_directionalSun
 	GLCompileMacro_USE_VERTEX_SKINNING( this ),
 	GLCompileMacro_USE_VERTEX_ANIMATION( this ),
 	GLCompileMacro_USE_HEIGHTMAP_IN_NORMALMAP( this ),
+	GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP( this ),
 	GLCompileMacro_USE_PARALLAX_MAPPING( this ),
 	GLCompileMacro_USE_SHADOWING( this )
 {
@@ -1767,6 +1776,7 @@ GLShader_reflection::GLShader_reflection( GLShaderManager *manager ):
 	GLCompileMacro_USE_VERTEX_SKINNING( this ),
 	GLCompileMacro_USE_VERTEX_ANIMATION( this ),
 	GLCompileMacro_USE_HEIGHTMAP_IN_NORMALMAP( this ),
+	GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP( this ),
 	GLCompileMacro_USE_PARALLAX_MAPPING( this )
 {
 }
@@ -2035,6 +2045,7 @@ GLShader_liquid::GLShader_liquid( GLShaderManager *manager ) :
 	u_LightGridOrigin( this ),
 	u_LightGridScale( this ),
 	GLCompileMacro_USE_HEIGHTMAP_IN_NORMALMAP( this ),
+	GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP( this ),
 	GLCompileMacro_USE_PARALLAX_MAPPING( this )
 {
 }

src/engine/renderer/gl_shader.h

@@ -30,7 +30,7 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 #define USE_UNIFORM_FIREWALL 1
 
 // *INDENT-OFF*
-static const unsigned int MAX_SHADER_MACROS = 9;
+static const unsigned int MAX_SHADER_MACROS = 10;
 static const unsigned int GL_SHADER_VERSION = 3;
 
 class ShaderException : public std::runtime_error
@@ -787,6 +787,7 @@ class GLCompileMacro
 	  USE_LIGHT_MAPPING,
 	  USE_DELUXE_MAPPING,
 	  USE_HEIGHTMAP_IN_NORMALMAP,
+	  USE_NORMALMAP_FROM_HEIGHTMAP,
 	  USE_PARALLAX_MAPPING,
 	  USE_REFLECTIVE_SPECULAR,
 	  USE_SHADOWING,
@@ -1087,6 +1088,31 @@ class GLCompileMacro_USE_HEIGHTMAP_IN_NORMALMAP :
 	}
 };
 
+class GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP :
+	GLCompileMacro
+{
+public:
+	GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP( GLShader *shader ) :
+		GLCompileMacro( shader )
+	{
+	}
+
+	const char *GetName() const
+	{
+		return "USE_NORMALMAP_FROM_HEIGHTMAP";
+	}
+
+	EGLCompileMacro GetType() const
+	{
+		return EGLCompileMacro::USE_NORMALMAP_FROM_HEIGHTMAP;
+	}
+
+	void SetNormalMapFromHeightMap( bool enable )
+	{
+		SetMacro( enable );
+	}
+};
+
 class GLCompileMacro_USE_PARALLAX_MAPPING :
 	GLCompileMacro
 {
@@ -2245,6 +2271,7 @@ class GLShader_lightMapping :
 	public GLCompileMacro_USE_LIGHT_MAPPING,
 	public GLCompileMacro_USE_DELUXE_MAPPING,
 	public GLCompileMacro_USE_HEIGHTMAP_IN_NORMALMAP,
+	public GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP,
 	public GLCompileMacro_USE_PARALLAX_MAPPING,
 	public GLCompileMacro_USE_REFLECTIVE_SPECULAR,
 	public GLCompileMacro_USE_PHYSICAL_SHADING
@@ -2284,6 +2311,7 @@ class GLShader_forwardLighting_omniXYZ :
 	public GLCompileMacro_USE_VERTEX_SKINNING,
 	public GLCompileMacro_USE_VERTEX_ANIMATION,
 	public GLCompileMacro_USE_HEIGHTMAP_IN_NORMALMAP,
+	public GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP,
 	public GLCompileMacro_USE_PARALLAX_MAPPING,
 	public GLCompileMacro_USE_SHADOWING //,
 {
@@ -2323,6 +2351,7 @@ class GLShader_forwardLighting_projXYZ :
 	public GLCompileMacro_USE_VERTEX_SKINNING,
 	public GLCompileMacro_USE_VERTEX_ANIMATION,
 	public GLCompileMacro_USE_HEIGHTMAP_IN_NORMALMAP,
+	public GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP,
 	public GLCompileMacro_USE_PARALLAX_MAPPING,
 	public GLCompileMacro_USE_SHADOWING //,
 {
@@ -2364,6 +2393,7 @@ class GLShader_forwardLighting_directionalSun :
 	public GLCompileMacro_USE_VERTEX_SKINNING,
 	public GLCompileMacro_USE_VERTEX_ANIMATION,
 	public GLCompileMacro_USE_HEIGHTMAP_IN_NORMALMAP,
+	public GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP,
 	public GLCompileMacro_USE_PARALLAX_MAPPING,
 	public GLCompileMacro_USE_SHADOWING //,
 {
@@ -2413,6 +2443,7 @@ class GLShader_reflection :
 	public GLCompileMacro_USE_VERTEX_SKINNING,
 	public GLCompileMacro_USE_VERTEX_ANIMATION,
 	public GLCompileMacro_USE_HEIGHTMAP_IN_NORMALMAP,
+	public GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP,
 	public GLCompileMacro_USE_PARALLAX_MAPPING
 {
 public:
@@ -2620,6 +2651,7 @@ class GLShader_liquid :
 	public u_LightGridOrigin,
 	public u_LightGridScale,
 	public GLCompileMacro_USE_HEIGHTMAP_IN_NORMALMAP,
+	public GLCompileMacro_USE_NORMALMAP_FROM_HEIGHTMAP,
 	public GLCompileMacro_USE_PARALLAX_MAPPING
 {
 public:

src/engine/renderer/glsl_source/reliefMapping_fp.glsl

@@ -29,24 +29,181 @@ uniform sampler2D	u_NormalMap;
 uniform vec3        u_NormalScale;
 #endif // r_normalMapping
 
-#if defined(USE_PARALLAX_MAPPING)
-#if !defined(USE_HEIGHTMAP_IN_NORMALMAP)
+#if (defined(USE_PARALLAX_MAPPING) && !defined(USE_HEIGHTMAP_IN_NORMALMAP)) || defined(USE_NORMALMAP_FROM_HEIGHTMAP)
 uniform sampler2D	u_HeightMap;
-#endif // !USE_HEIGHTMAP_IN_NORMALMAP
+#endif // (USE_PARALLAX_MAPPING && !USE_HEIGHTMAP_IN_NORMALMAP) || USE_NORMALMAP_FROM_HEIGHTMAP
+
+#if defined(USE_PARALLAX_MAPPING)
 uniform float       u_ParallaxDepthScale;
 uniform float       u_ParallaxOffsetBias;
 #endif // USE_PARALLAX_MAPPING
 
+#if defined(USE_NORMALMAP_FROM_HEIGHTMAP)
+vec3 NormalFromHeightMap(vec2 texNormal)
+{
+	vec3 normal;
+
+	/* Major inspiration was this post by jollyjeffers:
+	https://www.gamedev.net/forums/topic/475213-generate-normal-map-from-heightmap-algorithm/#post_4117038
+
+	and this Wikipedia article:
+	https://en.wikipedia.org/wiki/Sobel_operator
+
+	Various people recommends doing abs() on sample read in case of float image format:
+	http://www.catalinzima.com/2008/01/converting-displacement-maps-into-normal-maps/
+	https://community.khronos.org/t/heightmap-to-normalmap/58862
+
+	Or report issues by not doing it:
+	https://gamedev.stackexchange.com/questions/165575/calculating-normal-map-from-height-map-using-sobel-operator
+
+	Other people say texture2D clamps it but it's safer to do this.
+	*/
+
+	// Get texel size
+	vec2 texelSize = 1.0 / vec2(textureSize(u_HeightMap, 0));
+
+#if defined(r_sobelFiltering)
+	/* Useful things to know:
+	- It's required to get height samples (S) surrounding the current texel (T) to do a sobel filter:
+
+	  S S S
+	  S T S
+	  S S S
+
+	- Sobel X kernel is:
+
+	  [ 1  0  -1 ]
+	  [ 2  0  -2 ]
+	  [ 1  0  -1 ]
+
+	  See https://en.wikipedia.org/wiki/Sobel_operator#Formulation
+
+	- Sobel Y kernel is:
+
+	  [  1  2  1 ]
+	  [  0  0  0 ]
+	  [ -1 -2 -1 ]
+
+	  Which is the rotated sobel X kernel.
+
+	- Tangent space normals are +Z.
+	*/
+
+	ivec3 offsets;
+	mat3 xCoords;
+	mat3 yCoords;
+	mat3 heights;
+	mat3 sobel;
+
+	// Components will be computed by accumulating values.
+	normal = vec3(0.0, 0.0, 0.0);
+
+	// Set offsets:
+	offsets = ivec3(-1, 0, 1);
+
+	// Set sobel X kernel (beware, line is column with this notation):
+	sobel[0] = vec3( 1.0,  2.0,  1.0);
+	sobel[1] = vec3( 0.0,  0.0,  0.0);
+	sobel[2] = vec3(-1.0, -2.0, -1.0);
+
+	for (int i = 0; i < 3; i++)
+	{
+		for (int j = 0; j < 3; j++)
+		{
+			if (i != 1 || j != 1)
+			{
+				// Compute coordinates:
+				xCoords[i][j] = texNormal.x + texelSize.x * offsets[i];
+				yCoords[i][j] = texNormal.y + texelSize.y * offsets[j];
+
+				// Get surrounding samples:
+				heights[i][j] = abs(texture2D(u_HeightMap, vec2(xCoords[i][j], yCoords[i][j])).r);
+
+				if (i != 1)
+				{
+					// Sobel computation for X component (use the X sobel kernel):
+					normal.x += heights[i][j] * sobel[i][j];
+				}
+
+				if (j != 1)
+				{
+					// Sobel computation for Y component (use the rotated X sobel kernel as Y one):
+					normal.y += heights[i][j] * sobel[j][i];
+				}
+			}
+		}
+	}
+
+	/* Reconstruct Z component while making sure to not take the square root
+	of a negative number. That may occur because of compression artifacts.
+
+	Xonotic texture known to produce black normalmap artifacts when doing
+	Z reconstruction from X and Y computed from jpg heightmap:
+	textures/map_glowplant/sand_bump
+	*/
+
+	normal.z = sqrt(max(0.0, 1.0 - dot(normal.xy, normal.xy)));
+
+	normal.xy = 2.0 * normal.xy;
+#else // !r_sobelFiltering
+	/* Useful thing to know:
+	- Only three samples are required to determine two vectors
+	  they would be used to generate the normal at this texel:
+
+	  T S
+	  S
+
+	The texel itself is used as sample.
+	*/
+
+	vec2 hOffsets;
+	vec2 vOffsets;
+	vec2 hCoords;
+	vec2 vCoords;
+	vec3 heights;
+	vec3 xVector;
+	vec3 yVector;
+
+	// Set horizontal and vertical offsets:
+	hOffsets = vec2(texelSize.x, 0.0);
+	vOffsets = vec2(0.0, texelSize.y);
+
+	// Compute coordinates:
+	hCoords = vec2(texNormal.x + hOffsets.x, texNormal.y + hOffsets.y);
+	vCoords = vec2(texNormal.x + vOffsets.x, texNormal.y + vOffsets.y);
+
+	// Get samples:
+	heights.x = texture2D(u_HeightMap, texNormal).r; // No offset.
+	heights.y = texture2D(u_HeightMap, hCoords).r;
+	heights.z = texture2D(u_HeightMap, vCoords).r;
+
+	xVector = vec3(hOffsets.x, vOffsets.x, heights.y - heights.x);
+	yVector = vec3(hOffsets.y, vOffsets.y, heights.z - heights.x);
+
+	normal = cross(xVector, yVector);
+#endif // !r_sobelFiltering
+
+	return normal;
+}
+#endif // USE_NORMALMAP_FROM_HEIGHTMAP
+
 // compute normal in tangent space
 vec3 NormalInTangentSpace(vec2 texNormal)
 {
 	vec3 normal;
 
 #if defined(r_normalMapping)
-#if defined(USE_HEIGHTMAP_IN_NORMALMAP)
+#if defined(USE_NORMALMAP_FROM_HEIGHTMAP)
+	normal = NormalFromHeightMap(texNormal);
+#elif defined(USE_HEIGHTMAP_IN_NORMALMAP)
 	// alpha channel contains the height map so do not try to reconstruct normal map from it
 	normal = texture2D(u_NormalMap, texNormal).rgb;
 	normal = 2.0 * normal - 1.0;
+
+	// HACK: the GLSL code is currently assuming
+	// DirectX normal map format (+X -Y +Z)
+	// but engine is assuming the OpenGL way (+X +Y +Z)
+	normal.y *= -1;
 #else // !USE_HEIGHTMAP_IN_NORMALMAP
 	// the Capcom trick abusing alpha channel of DXT1/5 formats to encode normal map
 	// https://github.com/DaemonEngine/Daemon/issues/183#issuecomment-473691252
@@ -74,6 +231,11 @@ vec3 NormalInTangentSpace(vec2 texNormal)
 	// This might happen with other formats too. So we must take care not to
 	// take the square root of a negative number here.
 	normal.z = sqrt(max(0, 1.0 - dot(normal.xy, normal.xy)));
+
+	// HACK: the GLSL code is currently assuming
+	// DirectX normal map format (+X -Y +Z)
+	// but engine is assuming the OpenGL way (+X +Y +Z)
+	normal.y *= -1;
 #endif // !USE_HEIGHTMAP_IN_NORMALMAP
 	/* Disable normal map scaling when normal Z scale is set to zero.
 
@@ -87,11 +249,6 @@ vec3 NormalInTangentSpace(vec2 texNormal)
 	{
 		normal *= u_NormalScale;
 	}
-
-	// HACK: the GLSL code is currently assuming
-	// DirectX normal map format (+X -Y +Z)
-	// but engine is assuming the OpenGL way (+X +Y +Z)
-	normal.y *= -1;
 #else // !r_normalMapping
 	// Flat normal map is {0.5, 0.5, 1.0} in [ 0.0, 1.0]
 	// which is stored as {0.0, 0.0, 1.0} in [-1.0, 1.0].

src/engine/renderer/tr_init.cpp

@@ -171,6 +171,7 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 	cvar_t      *r_glowMapping;
 	cvar_t      *r_reflectionMapping;
 
+	cvar_t      *r_sobelFiltering;
 	cvar_t      *r_wrapAroundLighting;
 	cvar_t      *r_halfLambertLighting;
 	cvar_t      *r_rimLighting;
@@ -1203,6 +1204,7 @@ ScreenshotCmd screenshotPNGRegistration("screenshotPNG", ssFormat_t::SSF_PNG, "p
 		r_glowMapping = ri.Cvar_Get( "r_glowMapping", "1", CVAR_LATCH );
 		r_reflectionMapping = ri.Cvar_Get( "r_reflectionMapping", "0", CVAR_CHEAT );
 
+		r_sobelFiltering = ri.Cvar_Get( "r_sobelFiltering", "1", CVAR_LATCH );
 		r_wrapAroundLighting = ri.Cvar_Get( "r_wrapAroundLighting", "0.7", CVAR_CHEAT | CVAR_LATCH );
 		r_halfLambertLighting = ri.Cvar_Get( "r_halfLambertLighting", "1", CVAR_CHEAT | CVAR_LATCH );
 		r_rimLighting = ri.Cvar_Get( "r_rimLighting", "0",  CVAR_LATCH | CVAR_ARCHIVE );

src/engine/renderer/tr_local.h

@@ -1143,6 +1143,7 @@ static inline void halfToFloat( const f16vec4_t in, vec4_t out )
 		bool hasNormalMap;
 		bool hasHeightMap;
 		bool isHeightMapInNormalMap;
+		bool isNormalMapFromHeightMap;
 		bool hasMaterialMap;
 		bool isMaterialPhysical;
 		bool hasGlowMap;
@@ -2895,6 +2896,7 @@ static inline void halfToFloat( const f16vec4_t in, vec4_t out )
 	extern cvar_t *r_glowMapping;
 	extern cvar_t *r_reflectionMapping;
 
+	extern cvar_t *r_sobelFiltering;
 	extern cvar_t *r_wrapAroundLighting;
 	extern cvar_t *r_halfLambertLighting;
 	extern cvar_t *r_rimLighting;