Fix and improve FXAA

src/engine/renderer/GLUtils.h

@@ -140,6 +140,7 @@ struct GLConfig
 	bool bufferStorageAvailable;
 	bool uniformBufferObjectAvailable;
 	bool mapBufferRangeAvailable;
+	bool samplerObjectsAvailable;
 	bool syncAvailable;
 	bool textureBarrierAvailable;
 	bool halfFloatVertexAvailable;
@@ -155,8 +156,9 @@ struct GLConfig
 	bool reflectionMappingAvailable;
 	bool reflectionMapping;
 	bool bloom;
+	bool FXAA; // automatically disabled when MSAA is not null
 	int  MSAA; // 0 == disabled, otherwise used as sample count
-	bool ssao;
+	bool SSAO;
 	bool motionBlur;
 };
 

src/engine/renderer/gl_shader.cpp

@@ -797,6 +797,25 @@ static std::string GenEngineConstants() {
 		AddDefine( str, "r_colorGrading", 1 );
 	}
 
+	if ( r_showLuma.Get() )
+	{
+		AddDefine( str, "r_showLuma", 1 );
+	}
+
+	if ( glConfig.FXAA )
+	{
+		AddDefine( str, "r_FXAA", 1 );
+
+		AddDefine( str, "r_FXAASubPix", r_FXAASubPix.Get() );
+		AddDefine( str, "r_FXAAEdgeThreshold", r_FXAAEdgeThreshold.Get() );
+		AddDefine( str, "r_FXAAEdgeThresholdMin", r_FXAAEdgeThresholdMin.Get() );
+
+		if ( r_showFXAA.Get() )
+		{
+			AddDefine( str, "r_showFXAA", 1 );
+		}
+	}
+
 	if ( r_highPrecisionRendering.Get() ) {
 		AddDefine( str, "r_highPrecisionRendering", 1 );
 	}

src/engine/renderer/glsl_source/cameraEffects_fp.glsl

@@ -111,5 +111,23 @@ void main()
 
 	color.xyz = pow(color.xyz, vec3(u_InverseGamma));
 
+	#if defined(r_FXAA) || defined(r_showLuma)
+	{
+		// That luma vector comes from a comment in fxaa3_11_fp.glsl.
+		vec3 lumaVector = vec3( 0.299, 0.587, 0.114 );
+
+		float luma = dot( color.rgb, lumaVector );
+
+		#if defined(r_showLuma)
+			color.rgb = vec3( luma );
+		#endif
+
+		#if defined(r_FXAA)
+			// Encode luma in alpha channel.
+			color.a = luma;
+		#endif
+	}
+	#endif
+
 	outputColor = color;
 }

src/engine/renderer/glsl_source/fxaa3_11_fp.glsl

@@ -20,20 +20,6 @@ DAMAGES.
 
 kangz: This code has been set in the public domain by TIMOTHY LOTTES
 
-============================================================================*/
-
-//Due to our shader system, we put the defines for the control knobs here
-#define FXAA_PC 1
-#if __VERSION__ == 120
-#define FXAA_GLSL_120 1
-#else
-#define FXAA_GLSL_130 1
-#endif
-
-#define FXAA_QUALITY_PRESET 12
-#define FXAA_GREEN_AS_LUMA 1
-
-/*============================================================================
 ------------------------------------------------------------------------------
                            INTEGRATION CHECKLIST
 ------------------------------------------------------------------------------

src/engine/renderer/glsl_source/fxaa_fp.glsl

@@ -33,13 +33,25 @@ ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 
 /* fxaa_fp.glsl */
 
-// The FXAA parameters are put directly in fxaa3_11_fp.glsl
-// because we cannot #include in the middle of a shader
-// ^This is no longer true, but I'm not touching that mess
+// Control knobs.
+#if __VERSION__ == 120
+#define FXAA_GLSL_120 1
+#else
+#define FXAA_GLSL_130 1
+#endif
+
+#define FXAA_PC 1
+#define FXAA_QUALITY_PRESET 12
+#define FXAA_GREEN_AS_LUMA 0
 
 #insert fxaa3_11_fp
 
+#if defined(HAVE_ARB_bindless_texture)
+uniform sampler2D u_ColorMap_linear;
+#define u_ColorMap u_ColorMap_linear
+#else
 uniform sampler2D	u_ColorMap;
+#endif
 
 #if __VERSION__ > 120
 out vec4 outputColor;
@@ -49,7 +61,7 @@ out vec4 outputColor;
 
 void	main()
 {
-	outputColor = FxaaPixelShader(
+	vec4 color = FxaaPixelShader(
 		gl_FragCoord.xy / r_FBufSize, //pos
 		vec4(0.0), //not used
 		u_ColorMap, //tex
@@ -59,12 +71,27 @@ void	main()
 		vec4(0.0), //not used
 		vec4(0.0), //not used
 		vec4(0.0), //not used
-		0.75, //fxaaQualitySubpix
-		0.166, //fxaaQualityEdgeThreshold
-		0.0625, //fxaaQualityEdgeThresholdMin
+		r_FXAASubPix, //fxaaQualitySubpix
+		r_FXAAEdgeThreshold, //fxaaQualityEdgeThreshold
+		r_FXAAEdgeThresholdMin, //fxaaQualityEdgeThresholdMin
 		0.0, //not used
 		0.0, //not used
 		0.0, //not used
 		vec4(0.0) //not used
 	);
+
+	#if defined(r_showFXAA)
+	{
+		vec4 originalColor = FxaaTexTop( u_ColorMap, gl_FragCoord.xy / r_FBufSize );
+
+		if ( color.r != originalColor.r
+			|| color.g != originalColor.g
+			|| color.b != originalColor.b )
+		{
+			color.rgb = vec3(1.0, 0.0, 0.0);
+		}
+	}
+	#endif
+
+	outputColor = vec4( color.rgb, 1.0f );
 }

src/engine/renderer/tr_backend.cpp

@@ -1552,7 +1552,7 @@ void RB_RenderMotionBlur()
 
 void RB_RenderSSAO()
 {
-	if ( !glConfig.ssao )
+	if ( !glConfig.SSAO )
 	{
 		return;
 	}
@@ -1570,7 +1570,7 @@ void RB_RenderSSAO()
 	GL_State( GLS_DEPTHTEST_DISABLE | GLS_SRCBLEND_DST_COLOR | GLS_DSTBLEND_ZERO );
 	GL_Cull( cullType_t::CT_TWO_SIDED );
 
-	if ( glConfig.ssao && r_ssao.Get() == Util::ordinal( ssaoMode::SHOW ) ) {
+	if ( glConfig.SSAO && r_SSAO.Get() == Util::ordinal( ssaoMode::SHOW ) ) {
 		// clear the screen to show only SSAO
 		GL_ClearColor( 1.0, 1.0, 1.0, 1.0 );
 		glClear( GL_COLOR_BUFFER_BIT );
@@ -1603,7 +1603,7 @@ void RB_RenderSSAO()
 
 void RB_FXAA()
 {
-	if ( !r_FXAA->integer || !gl_fxaaShader )
+	if ( !glConfig.FXAA || !gl_fxaaShader )
 	{
 		return;
 	}
@@ -1621,15 +1621,46 @@ void RB_FXAA()
 	// set the shader parameters
 	gl_fxaaShader->BindProgram();
 
-	// Swap main FBOs
 	gl_fxaaShader->SetUniform_ColorMapBindless(
 		GL_BindToTMU( 0, tr.currentRenderImage[backEnd.currentMainFBO] )
 	);
-	backEnd.currentMainFBO = 1 - backEnd.currentMainFBO;
-	R_BindFBO( tr.mainFBO[ backEnd.currentMainFBO ] );
+
+	// FXAA expects GL_LINEAR for the sampling to work.
+	GLuint64 handle = 0;
+
+	if ( glConfig.usingBindlessTextures )
+	{
+		// Set a handler.
+		GLuint texture = tr.currentRenderImage[backEnd.currentMainFBO]->texnum;
+		handle = glGetTextureSamplerHandleARB( texture, tr.linearSampler );
+		glMakeTextureHandleResidentARB( handle );
+		GLuint program = gl_fxaaShader->GetProgram()->id;
+		GLint location = glGetUniformLocation( program, "u_ColorMap_linear" );
+		glUniformHandleui64ARB( location, handle );
+	}
+	else
+	{
+		// Bind a sampler.
+		glBindSampler( 0, tr.linearSampler );
+	}
+
+	// This shader is run last, so let it render to screen.
+	R_BindNullFBO();
 
 	Tess_InstantScreenSpaceQuad();
 
+	// Make sure we didn't break other effects expecting GL_NEAREST.
+	if ( glConfig.usingBindlessTextures )
+	{
+		// Unset the handler.
+		glMakeTextureHandleNonResidentARB( handle );
+	}
+	else
+	{
+		// Unbind the sampler.
+		glBindSampler( 0, 0 );
+	}
+
 	GL_CheckErrors();
 }
 
@@ -1693,13 +1724,22 @@ void RB_CameraPostFX() {
 	}
 	gl_cameraEffectsShader->SetUniform_Tonemap( tonemap );
 
-	// This shader is run last, so let it render to screen instead of
-	// tr.mainFBO
-	R_BindNullFBO();
 	gl_cameraEffectsShader->SetUniform_CurrentMapBindless(
 		GL_BindToTMU( 0, tr.currentRenderImage[backEnd.currentMainFBO] ) 
 	);
 
+	if ( r_FXAA.Get() && gl_fxaaShader )
+	{
+		// Swap main FBOs.
+		backEnd.currentMainFBO = 1 - backEnd.currentMainFBO;
+		R_BindFBO( tr.mainFBO[ backEnd.currentMainFBO ] );
+	}
+	else
+	{
+		// Without FXAA this shader is run last, so let it render to screen.
+		R_BindNullFBO();
+	}
+
 	if ( glConfig.colorGrading ) {
 		gl_cameraEffectsShader->SetUniform_ColorMap3DBindless( GL_BindToTMU( 3, tr.colorGradeImage ) );
 	}
@@ -2815,11 +2855,11 @@ static void RB_RenderPostProcess()
 
 	TransitionMSAAToMain( GL_COLOR_BUFFER_BIT );
 
-	RB_FXAA();
-
 	// render chromatic aberration
 	RB_CameraPostFX();
 
+	RB_FXAA();
+
 	// copy to given byte buffer that is NOT a FBO
 	if ( tr.refdef.pixelTarget != nullptr ) {
 		glReadPixels( 0, 0, tr.refdef.pixelTargetWidth, tr.refdef.pixelTargetHeight, GL_RGBA,
@@ -3839,6 +3879,12 @@ void R_ShutdownBackend()
 		glDisableVertexAttribArray( i );
 	}
 	glState.vertexAttribsState = 0;
+
+	if ( tr.linearSampler )
+	{
+		glDeleteSamplers( 1, &tr.linearSampler );
+		tr.linearSampler = 0;
+	}
 }
 
 const RenderCommand *EndOfListCommand::ExecuteSelf( ) const

src/engine/renderer/tr_init.cpp

@@ -285,9 +285,21 @@ Cvar::Cvar<int> r_rendererAPI( "r_rendererAPI", "Renderer API: 0: OpenGL, 1: Vul
 	Cvar::Cvar<bool> r_bloom( "r_bloom", "Use bloom", Cvar::ARCHIVE, false );
 	Cvar::Cvar<float> r_bloomBlur( "r_bloomBlur", "Bloom strength", Cvar::NONE, 0.2 );
 	Cvar::Cvar<int> r_bloomPasses( "r_bloomPasses", "Amount of bloom passes in each direction", Cvar::NONE, 2 );
-	cvar_t      *r_FXAA;
-	Cvar::Range<Cvar::Cvar<int>> r_msaa( "r_msaa", "Amount of MSAA samples. 0 to disable", Cvar::NONE, 0, 0, 64 );
-	Cvar::Range<Cvar::Cvar<int>> r_ssao( "r_ssao",
+
+	Cvar::Cvar<bool> r_showLuma( "r_showLuma", "Show luminance", Cvar::CHEAT, false );
+
+	Cvar::Cvar<bool> r_FXAA( "r_FXAA", "Fast approximate anti-aliasing", Cvar::NONE, false );
+
+	// Values taken from comments in fxaa3_11_fp.glsl.
+	Cvar::Range<Cvar::Cvar<float>> r_FXAASubPix( "r_FXAASubPix", "0: off, 0.25: almost off, 0.50: sharper, 0.75, standard, 1: softer", Cvar::NONE, 0.75f, 0.0f, 1.0f );
+	Cvar::Range<Cvar::Cvar<float>> r_FXAAEdgeThreshold( "r_FXAAEdgeThreshold", "0.063: overkill and slower, 0.125: high quality, 0.166: standard, 0.250: low quality, 0.333 too little and faster", Cvar::NONE, 0.250f, 0.063f, 0.333f );
+	Cvar::Range<Cvar::Cvar<float>> r_FXAAEdgeThresholdMin( "r_FXAAEdgeThresholdMin", "0.0312: visible limit, 0.0625: high quality, 0.0833: upper limit", Cvar::NONE, 0.0625f, 0.0312f, 0.0833f );
+
+	Cvar::Cvar<bool> r_showFXAA( "r_showFXAA", "Show pixels modified by FXAA", Cvar::CHEAT, false );
+
+	Cvar::Range<Cvar::Cvar<int>> r_MSAA( "r_MSAA", "Amount of MSAA samples. 0 to disable", Cvar::NONE, 0, 0, 64 );
+
+	Cvar::Range<Cvar::Cvar<int>> r_SSAO( "r_SSAO",
 		"Screen space ambient occlusion: "
 		"-1: show, 0: disabled, 1: enabled",
 		Cvar::NONE,
@@ -1205,9 +1217,17 @@ ScreenshotCmd screenshotPNGRegistration("screenshotPNG", ssFormat_t::SSF_PNG, "p
 		r_printShaders = Cvar_Get( "r_printShaders", "0", 0 );
 
 		Cvar::Latch( r_bloom );
-		r_FXAA = Cvar_Get( "r_FXAA", "0", CVAR_LATCH | CVAR_ARCHIVE );
-		Cvar::Latch( r_ssao );
-		Cvar::Latch( r_msaa );
+		Cvar::Latch( r_SSAO );
+
+		Cvar::Latch( r_showLuma );
+
+		Cvar::Latch( r_FXAA );
+		Cvar::Latch( r_FXAASubPix );
+		Cvar::Latch( r_FXAAEdgeThreshold );
+		Cvar::Latch( r_FXAAEdgeThresholdMin );
+		Cvar::Latch( r_showFXAA );
+
+		Cvar::Latch( r_MSAA );
 
 		// temporary variables that can change at any time
 		r_showImages = Cvar_Get( "r_showImages", "0", CVAR_TEMP );

src/engine/renderer/tr_local.h

@@ -2595,6 +2595,8 @@ enum
 		float         inverseSawToothTable[ FUNCTABLE_SIZE ];
 
 		scissorState_t scissor;
+
+		GLuint linearSampler;
 	};
 
 	extern const matrix_t quakeToOpenGLMatrix;
@@ -2779,9 +2781,17 @@ enum
 	extern Cvar::Cvar<bool> r_bloom;
 	extern Cvar::Cvar<float> r_bloomBlur;
 	extern Cvar::Cvar<int> r_bloomPasses;
-	extern cvar_t *r_FXAA;
-	extern Cvar::Range<Cvar::Cvar<int>> r_ssao;
-	extern Cvar::Range<Cvar::Cvar<int>> r_msaa;
+	extern Cvar::Range<Cvar::Cvar<int>> r_SSAO;
+
+	extern Cvar::Cvar<bool> r_showLuma;
+
+	extern Cvar::Cvar<bool> r_FXAA;
+	extern Cvar::Range<Cvar::Cvar<float>> r_FXAASubPix;
+	extern Cvar::Range<Cvar::Cvar<float>> r_FXAAEdgeThreshold;
+	extern Cvar::Range<Cvar::Cvar<float>> r_FXAAEdgeThresholdMin;
+	extern Cvar::Cvar<bool> r_showFXAA;
+
+	extern Cvar::Range<Cvar::Cvar<int>> r_MSAA;
 
 	extern cvar_t *r_evsmPostProcess;