Direct3D9 / HLSL Depth semantic not working

Question

i have o problem with my shader code (HLSL). I use "DirectX for Managed Code" and Shader Model 3.0. I try to write a custom depth value into the depth buffer by using the DEPTH semantic in the pixel shader output struct:

struct PSOutput
{
    float4 col : COLOR0;
    float dept : DEPTH;
};

and i use this struct as return value in my pixel shader:

PSOutput PSFunction(VertexShaderOutput input)
{
    PSOutput output;
    ...
    output.col = float4(...);
    output.dept = ...;

    return output;
}

DirectX throws an exeption when i try to compile this shader, but gives no detailed information why. But when i remove the depth variable from the output struct it works! I also tried to write DEPTH0 as semantic, but no success. I hope anyone can help my with that.

EDIT:

If i write the following, it fails:

PSOutput PSFunction(VertexShaderOutput input)
{
    PSOutput output;

    float resDepth = input.Position[2] / input.Position[3];

    if(...)
    {
       resDepth = ...; 
    }
    output.col = float4(...);
    output.dept = resDepth;

    return output;
}

but if i write this code, it compiles:

PSOutput PSFunction(VertexShaderOutput input)
{
    PSOutput output;

    float resDepth = input.Position[2] / input.Position[3];

    if(...)
    {
       resDepth = ...; 
    }
    output.col = float4(...);
    output.dept = 0.5;

    return output;
}

any ideas?

Here's the full code:

float4x4 World;
float4x4 View;
float4x4 Projection;

float4 CamPos;
float4 LightDir;
float4 ObjColor;

static const float PI = 3.14159265f;

static const int MAX_FU = 32;

float fuPercent[MAX_FU];
float4 fuColor[MAX_FU];

int buildDir;
static int fuCount = 2;

float4 boxMin;
float4 boxMax;

struct VertexShaderInput
{
    float4 Position : POSITION0;
    float3 Normal : NORMAL;
};

struct VertexShaderOutput
{
    float4 Position : POSITION0;
    float3 Normal : NORMAL;
    float3 ExactPos : TEXCOORD1;
};

struct PSOutput
{
    float4 col : COLOR0;
    //float dept : DEPTH;
};

VertexShaderOutput VSFunction(VertexShaderInput input)
{
    VertexShaderOutput output;

    float4 worldPosition = mul(input.Position, World);
    float4 viewPosition = mul(worldPosition, View);
    output.Position = mul(viewPosition, Projection);
    output.Normal = mul(input.Normal, World);
    output.ExactPos = input.Position;

    return output;
}

PSOutput PSFunction(VertexShaderOutput input)
{
    PSOutput output;

    float4 resColor = ObjColor;
    float resDepth = input.Position[2] / input.Position[3];

    float prpos = 0;

    if (buildDir == 0)
    {
        prpos = (input.ExactPos[1] - boxMin[1]) / (boxMax[1] - boxMin[1]);
    }
    else if (buildDir == 1)
    {
        prpos = 1.0 - ((input.ExactPos[1] - boxMin[1]) / (boxMax[1] - boxMin[1]));
    }
    else if (buildDir == 2)
    {
        prpos = (input.ExactPos[2] - boxMin[2]) / (boxMax[2] - boxMin[2]);
    }
    else if (buildDir == 3)
    {
        prpos = 1.0 - ((input.ExactPos[2] - boxMin[2]) / (boxMax[1] - boxMin[2]));
    }
    else if (buildDir == 4)
    {
        prpos = (input.ExactPos[0] - boxMin[0]) / (boxMax[0] - boxMin[0]);
    }
    else if (buildDir == 5)
    {
        prpos = 1.0 - ((input.ExactPos[0] - boxMin[0]) / (boxMax[0] - boxMin[0]));
    }

    float currPerc = 1.1;

    for (int i = 0; i < fuCount; i++)
    {
        if (prpos - 0.0001 <= fuPercent[i])
        {
            if (fuPercent[i] < currPerc)
            {
                currPerc = fuPercent[i];
                resColor = fuColor[i];
            }
        }
        else
        {
            resDepth = 1.0;
            resColor[3] = 0.0;
        }
    }

    float3 nor = input.Normal;
    float3 pos = input.ExactPos;
    float glo = 0.5;

    float id = (acos(dot(LightDir,nor) / pow(dot(LightDir,LightDir) * dot(nor, nor), 0.5))  / PI );
    id = pow(id,2);

    float3 look = reflect(normalize(pos - CamPos), nor);
    float gl = (acos(dot(LightDir,look) / pow(dot(LightDir,LightDir) * dot(look, look), 0.5))  / PI );
    gl = max(gl * 10.0 - 9.0, 0.0);
    gl = pow(gl,2) * glo;

    output.col = float4(resColor[0] * id + gl, resColor[1] * id + gl, resColor[2] * id + gl, resColor[3]);
    //output.dept = resDepth;

    return output;
}

technique MyTechnique
{
    pass Pass1
    {
        VertexShader = compile vs_3_0 VSFunction();
        PixelShader = compile ps_3_0 PSFunction();
    }
}

Answer 1

If FXC is throwing an exception during compilation rather than giving you a compilation error it's probably not anything you've done wrong.

If you're using the DirectX SDK make sure you're using the most recent version (June 2010). If you're using the Windows Kit 8.0 SDK then you may have found a compiler bug. What version of the SDK / fxc are you using?

Can you post a shader that actually compiles (one with the missing VertexShaderOutput struct and without ...'s in place of actual code)? I've filled in the missing code and have no problem compiling it using fxc from Windows Kit 8.0.

EDIT:

Nope, I hadn't spotted you'd commented out the code that made it not compile.

Sure enough, it doesn't compile, but that's because it's not valid code (as reported by the compile errors). You're using the POSITION semantic as an input to your pixel shader, which is not valid. If you want to use the outputted position from a vertex shader as input to a pixel shader, copy it into a second attribute and use that instead. If I substitute the following code into your shader it then compiles:

struct VertexShaderOutput
{
    float4 ClipPosition : POSITION; // Renamed this to ClipPosition.
    float4 Position : TEXCOORD0;    // This is valid to use as an input to the pixel shader.
    float3 Normal : NORMAL;
    float3 ExactPos : TEXCOORD1;
};

struct PSOutput
{
    float4 col : COLOR0;
    float dept : DEPTH;
};

VertexShaderOutput VSFunction(VertexShaderInput input)
{
    VertexShaderOutput output;

    float4 worldPosition = mul(input.Position, World);
    float4 viewPosition = mul(worldPosition, View);
    output.ClipPosition = mul(viewPosition, Projection); 
    output.Position = output.ClipPosition; // Copy output position to our other attribute.
    output.Normal = mul(input.Normal, World);
    output.ExactPos = input.Position;

    return output;
}