Improve shader building

Previously, the GLSL shader building was done in the following manner:
1. When the renderer is started, some of the shader source was initialised/processed. This included the main shader text, #insert's, and material system post-processing.
2. When `buildAll()` was called, all permutations with empty deform shaders were built. The deform shaders themselves are loaded either at the start (for the default, empty one), or when the map itself is being loaded. These permutations included adding macros to shaders in the form of `#ifndef #define #endif`, and attaching the default deform shader. When possible, shader program binaries were saved/loaded to/from disk.
3. UI shader and non-0 deform shaders were built on-demand.

There were multiple issues with that system:
1. Shader caches were unreliable. This is likely because the checksum was calculated *before* macros and such were added to the shader text.
2. Every shader permutation was always compiled and linked every time.
3. Shader programs with non-0 deformVertexes were never cached.

This commit changes the underlying system to use an `std::vector` of shader program descriptors, and another one for shader descriptors. The initialisation/building of shaders is now more generic for each type, and as a result each unique shader is only compiled once. The permutations are still linked each time (unless downloaded from cache), but this change lays the foundation to easily add shader pipelines, which would solve this problem.

Shader cache binaries are now identified by each shader compiled into the corresponding shader program, which allows caching non-0 deformVertexes too.

This also fixes the incorrect #line numbers emitted by #insert due to macros and such being added after #insert is processed.

Also adds more information into how many shader/shader programs were built and how much time it took (there's currently a small discrepancy, because the output is shown at the end of `BuildAll()`, but is gathered every time `BuildPermutation()` or `InitShader()` is called).

Author: VReaperV

src/engine/renderer/gl_shader.cpp

@@ -42,6 +42,11 @@ layout (local_size_x = 64, local_size_y = 1, local_size_z = 1) in;
 
 layout(binding = 0) uniform sampler2D depthImage;
 
+struct SurfaceDescriptor {
+	BoundingSphere boundingSphere;
+	uint surfaceCommandIDs[MAX_SURFACE_COMMANDS];
+};
+
 layout(std430, binding = BIND_SURFACE_DESCRIPTORS) readonly restrict buffer surfaceDescriptorsSSBO {
 	SurfaceDescriptor surfaces[];
 };

src/engine/renderer/gl_shader.h

@@ -44,9 +44,9 @@ void DeformVertex( inout vec4 pos,
 		   inout vec3 normal,
 		   inout vec2 st,
 		   inout vec4 color,
-		   in    float time);
+		   in    float time );
 
-void	main()
+void main()
 {
 	#insert material_vp
 
@@ -63,7 +63,7 @@ void	main()
 		      LB.normal,
 		      texCoord,
 		      color,
-		      u_Time);
+		      u_Time );
 
 	// transform vertex position into homogenous clip-space
 	gl_Position = u_ModelViewProjectionMatrix * position;

src/engine/renderer/glsl_source/cull_cp.glsl

@@ -55,9 +55,9 @@ void DeformVertex( inout vec4 pos,
 		   inout vec3 normal,
 		   inout vec2 st,
 		   inout vec4 color,
-		   in    float time);
+		   in    float time );
 
-void	main()
+void main()
 {
 	#insert material_vp
 
@@ -72,11 +72,7 @@ void	main()
 	color = color * ColorModulateToColor( u_ColorModulateColorGen, lightFactor )
 		+ unpackUnorm4x8( u_Color ) * vec4( lightFactor, lightFactor, lightFactor, 1.0 );
 
-	DeformVertex( position,
-		      LB.normal,
-		      texCoord,
-		      color,
-		      u_Time);
+	DeformVertex( position, LB.normal, texCoord, color, u_Time );
 
 	// transform vertex position into homogenous clip-space
 	gl_Position = u_ModelViewProjectionMatrix * position;

src/engine/renderer/glsl_source/fogQuake3_vp.glsl

@@ -64,7 +64,7 @@ OUT(smooth) vec3	var_Normal;
 
 OUT(smooth) vec4	var_Color;
 
-void DeformVertex(inout vec4 pos, inout vec3 normal, inout vec2 st, inout vec4 color, in float time);
+void DeformVertex( inout vec4 pos, inout vec3 normal, inout vec2 st, inout vec4 color, in float time );
 
 void main()
 {
@@ -78,7 +78,7 @@ void main()
 
 	color = color * ColorModulateToColor( u_ColorModulateColorGen ) + unpackUnorm4x8( u_Color );
 
-	DeformVertex(position, LB.normal, texCoord, color, u_Time);
+	DeformVertex( position, LB.normal, texCoord, color, u_Time );
 
 	// transform vertex position into homogenous clip-space
 	gl_Position = u_ModelViewProjectionMatrix * position;

src/engine/renderer/glsl_source/generic_vp.glsl

@@ -39,10 +39,10 @@ struct BoundingSphere {
 	float radius;
 };
 
-struct SurfaceDescriptor {
+/* struct SurfaceDescriptor {
 	BoundingSphere boundingSphere;
 	uint surfaceCommandIDs[MAX_SURFACE_COMMANDS];
-};
+}; */
 
 struct PortalSurface {
 	BoundingSphere boundingSphere;

src/engine/renderer/glsl_source/lightMapping_vp.glsl

@@ -120,7 +120,7 @@ GLuint64 BindAnimatedImage( int unit, const textureBundle_t *bundle )
 	return GL_BindToTMU( unit, bundle->image[ index ] );
 }
 
-void GL_BindProgram( shaderProgram_t *program )
+void GL_BindProgram( ShaderProgramDescriptor* program )
 {
 	if ( !program )
 	{
@@ -130,7 +130,7 @@ void GL_BindProgram( shaderProgram_t *program )
 
 	if ( glState.currentProgram != program )
 	{
-		glUseProgram( program->program );
+		glUseProgram( program->id );
 		glState.currentProgram = program;
 	}
 }
@@ -1378,7 +1378,7 @@ static void RB_SetupLightForShadowing( trRefLight_t *light, int index,
 				       bool shadowClip )
 {
 	// HACK: bring OpenGL into a safe state or strange FBO update problems will occur
-	GL_BindProgram( nullptr );
+	GL_BindNullProgram();
 	GL_State( GLS_DEFAULT );
 
 	GL_Bind( tr.whiteImage );

src/engine/renderer/glsl_source/material_cp.glsl

@@ -5054,6 +5054,7 @@ void RE_LoadWorldMap( const char *name )
 
 	Q_strncpyz( s_worldData.baseName, COM_SkipPath( s_worldData.name ), sizeof( s_worldData.name ) );
 	COM_StripExtension3( s_worldData.baseName, s_worldData.baseName, sizeof( s_worldData.baseName ) );
+	tr.loadingMap = s_worldData.baseName;
 
 	startMarker = (byte*) ri.Hunk_Alloc( 0, ha_pref::h_low );
 
@@ -5222,6 +5223,7 @@ void RE_LoadWorldMap( const char *name )
 	}
 
 	tr.worldLoaded = true;
+	tr.loadingMap = "";
 	GLSL_InitWorldShaders();
 
 	if ( glConfig2.reflectionMappingAvailable ) {

src/engine/renderer/tr_backend.cpp

@@ -878,7 +878,7 @@ ScreenshotCmd screenshotPNGRegistration("screenshotPNG", ssFormat_t::SSF_PNG, "p
 		glState.vertexAttribsState = 0;
 		glState.vertexAttribPointersSet = 0;
 
-		GL_BindProgram( nullptr );
+		GL_BindNullProgram();
 
 		glBindBuffer( GL_ARRAY_BUFFER, 0 );
 		glBindBuffer( GL_ELEMENT_ARRAY_BUFFER, 0 );
@@ -1141,7 +1141,7 @@ ScreenshotCmd screenshotPNGRegistration("screenshotPNG", ssFormat_t::SSF_PNG, "p
 				}
 
 				int deformIndex =
-					gl_shaderManager.getDeformShaderIndex( shader.deforms, shader.numDeforms );
+					gl_shaderManager.GetDeformShaderIndex( shader.deforms, shader.numDeforms );
 
 				for ( shaderStage_t *stage = shader.stages; stage != shader.lastStage; stage++ )
 				{

src/engine/renderer/tr_bsp.cpp

@@ -1415,18 +1415,7 @@ enum class shaderProfilerRenderSubGroupsMode {
 	};
 
 // *INDENT-ON*
-
-// Tr3B - shaderProgram_t represents a pair of one
-// GLSL vertex and one GLSL fragment shader
-	struct shaderProgram_t
-	{
-		GLuint    program;
-		GLuint    VS, FS, CS;
-		uint32_t  attribs; // vertex array attributes
-		GLint    *uniformLocations;
-		GLuint   *uniformBlockIndexes;
-		byte     *uniformFirewall;
-	};
+	struct ShaderProgramDescriptor;
 
 // trRefdef_t holds everything that comes in refdef_t,
 // as well as the locally generated scene information
@@ -2439,7 +2428,7 @@ enum class shaderProfilerRenderSubGroupsMode {
 		float           vertexAttribsInterpolation; // 0 = no interpolation, 1 = final position
 		uint32_t        vertexAttribsNewFrame; // offset for VBO vertex animations
 		uint32_t        vertexAttribsOldFrame; // offset for VBO vertex animations
-		shaderProgram_t *currentProgram;
+		ShaderProgramDescriptor* currentProgram;
 		FBO_t           *currentFBO;
 		VBO_t           *currentVBO;
 		IBO_t           *currentIBO;
@@ -2723,6 +2712,7 @@ enum class shaderProfilerRenderSubGroupsMode {
 
 		bool worldLoaded;
 		world_t    *world;
+		std::string loadingMap;
 
 		TextureManager textureManager;
 
@@ -3245,7 +3235,7 @@ inline bool checkGLErrors()
 	void GL_Unbind( image_t *image );
 	GLuint64 BindAnimatedImage( int unit, const textureBundle_t *bundle );
 	void GL_TextureFilter( image_t *image, filterType_t filterType );
-	void GL_BindProgram( shaderProgram_t *program );
+	void GL_BindProgram( ShaderProgramDescriptor* program );
 	GLuint64 GL_BindToTMU( int unit, image_t *image );
 	void GL_BindNullProgram();
 	void GL_SetDefaultState();

src/engine/renderer/tr_init.cpp

@@ -2469,7 +2469,7 @@ static void R_DebugGraphics()
 		// the render thread can't make callbacks to the main thread
 		R_SyncRenderThread();
 
-		GL_BindProgram( nullptr );
+		GL_BindNullProgram();
 
 		GL_Bind( tr.whiteImage );