Depth Masking

Depth masking is a technique where you split linear view depth into separate ranges in order to mask off parts of the scene from a fullscreen effect.

This is most useful with Depth of Field techniques, specifically to prevent foreground blurs from bleeding into the background. It was originally developed for a bokeh effect, where bleeding from foreground was making distant pixels look falsely lit. The normalization process in the code may not be the best way to go about it, feel free to experiment there as it is a bit costly on temporary registers.

To make use of this effect you need a couple things:

• 1) A render target with linear view depth, or depth values easily convertible into linear view depth.
• 2) Two render targets that will receive the depth mask, any 4-component format will work, but Color will suffice.

After the shader is ran, the two render targets will contain values between 0 to 1 in each component.

• RenderTarget0 = Nearest to Middle, RGBA in that order.
• RenderTarget1 = Middle to Farthest, RGBA in that order.

The DrawMask technique is just to output the mask to screen for visual confirmation that it's working.

// =============================================================================================
//	Textures & Samplers
// =============================================================================================
 
texture DepthMaskTexture;
sampler DepthMaskSampler = sampler_state
{
	Texture = <DepthMaskTexture>;
	MinFilter = Point; MagFilter = Point;
	AddressU = Clamp; AddressV = Clamp;
};
 
// =============================================================================================
//	Vertex Shaders
// =============================================================================================
 
void VS_Passive(inout float4 position : POSITION0,
		inout float2 texCoord : TEXCOORD0,
		uniform bool moveToFarPlane)
{
	if (moveToFarPlane)
		position.zw = 1;
}
 
// =============================================================================================
//	Pixel Shaders
// =============================================================================================
 
void PS_DepthMasks(float2 texCoord : TEXCOORD0, 
		   out float4 nearMasks : COLOR0, out float4 farMasks : COLOR1)
{
	static const float4 MinValuesNear = float4(0 * 0.125, 1 * 0.125, 2 * 0.125, 3 * 0.125);
	static const float4 MinValuesFar = float4(4 * 0.125, 5 * 0.125, 6 * 0.125, 7 * 0.125);
	static const float4 MaxValuesNear = float4(1 * 0.125, 2 * 0.125, 3 * 0.125, 4 * 0.125);
	static const float4 MaxValuesFar = float4(5 * 0.125, 6 * 0.125, 7 * 0.125, 8 * 0.125);
 
	texCoord = OffsetHalfPixel(texCoord, Camera.Size); // REPLACE WITH HOWEVER YOU DO HALF PIXEL OFFSETS
 
	float depth = GetDepth(texCoord); // REPLACE WITH HOWEVER YOU SAMPLE DEPTH
	float depthFactor[8];
 
	// clamp the ranges to their respective masks
	nearMasks = 1 - (clamp(depth.xxxx, MinValuesNear, MaxValuesNear) - MinValuesNear) / 0.125;
	farMasks = 1 - (clamp(depth.xxxx, MinValuesFar, MaxValuesFar) - MinValuesFar) / 0.125;
 
	// normalize the masks
	nearMasks.y = (nearMasks.y - nearMasks.x);
	nearMasks.z = (nearMasks.z - nearMasks.y - nearMasks.x);
	nearMasks.w = (nearMasks.w - nearMasks.z - nearMasks.y - nearMasks.x);
	farMasks.x = (farMasks.x - nearMasks.w - nearMasks.z - nearMasks.y - nearMasks.x);
	farMasks.y = (farMasks.y - farMasks.x - nearMasks.w - nearMasks.z - nearMasks.y - nearMasks.x);
	farMasks.z = (farMasks.z - farMasks.y - farMasks.x - nearMasks.w - nearMasks.z - nearMasks.y - nearMasks.x);
	farMasks.w = (farMasks.w - farMasks.z - farMasks.y - farMasks.x - nearMasks.w - nearMasks.z - nearMasks.y - nearMasks.x);
 
	// just a safety measure
	nearMasks = saturate(nearMasks);
	farMasks = saturate(farMasks);
}
 
float4 PS_DrawMask(float2 texCoord : TEXCOORD0) : COLOR0
{
	return tex2D(DepthMaskSampler, OffsetHalfPixel(texCoord, Camera.Size));
}
 
// =============================================================================================
//	Techniques
// =============================================================================================
 
technique DepthMasks
{
	pass DepthMasks
	{
		VertexShader = compile vs_2_0 VS_Passive(false);
		PixelShader = compile ps_2_0 PS_DepthMasks();
	}
}
 
technique DrawMask
{
	pass DrawMask
	{
		VertexShader = compile vs_2_0 VS_Passive(false);
		PixelShader = compile ps_2_0 PS_DrawMask();
	}
}