Search code examples
iosmetalvertex-shader

Getting error in Metal Shader Vertex function


The code base is bit old, and am trying to support latest iOS versions. It use to work fine but now am getting the bellow error.

vertex function has invalid return type 'ColorInOut'

Below is a part of the Metal shader code. You can get the full code here.

typedef struct
{
    packed_float3 position;
    packed_float3 normal;
    packed_float2 texCoord1;
    packed_float3 tangent;
    packed_float3 bitagent;
    packed_float4 vertColor;
} vertex_t;

typedef struct
{
    float4 position [[position]] , vertexPosition_ws;
    float2 uv,texture2UV;
    float hasLighting;
    float pointSize [[point_size]];
    float vertexDepth, reflection;
    float hasMeshColor;
    float shadowDarkness, transparency;
    float2 pointCoord [[ point_coord ]];
    float4 eyeVec, lightDir, lightColor;
    float3 meshColor;
    float3 T;
    float3 B;
    float3 N;
} ColorInOut;

vertex ColorInOut Mesh_Vertex(device vertex_t* vertex_array [[ buffer(0) ]],
                                constant float4x4& vp [[ buffer(SHADER_MESH_mvp) ]],
                                constant MatArray* model [[ buffer(SHADER_MESH_model) ]],
                                constant float4x4& lvp [[ buffer(SHADER_MESH_lightViewProjMatrix) ]],
                                constant packed_float3& eyePos [[ buffer(SHADER_MESH_eyePos) ]],
                                constant float& shadowDarkness [[ buffer(SHADER_MESH_shadowDarkness) ]],
                                constant float *hasMeshColor [[ buffer(SHADER_MESH_hasMeshColor) ]],
                                constant float *reflectionValue [[ buffer(SHADER_MESH_reflectionValue) ]],
                                constant float *transparency [[ buffer(SHADER_MESH_transparency) ]],
                                constant float *hasLighting [[ buffer(SHADER_MESH_hasLighting) ]],
                                constant float *uvScaleValue [[ buffer(SHADER_MESH_uvScale) ]],
                                constant float3Struct *meshColor [[buffer(SHADER_MESH_meshColor)]],
                                unsigned int vid [[ vertex_id ]],
                                unsigned int iId [[ instance_id ]]
                                )
{

    float4 vertexPosition_ws = model[iId].data * float4(float3(vertex_array[vid].position), 1.0);
    float4 vertColor = float4(vertex_array[vid].vertColor);
    float3 tangent = float3(vertex_array[vid].tangent);
    float3 bitangent = float3(vertex_array[vid].bitagent);
    float3 normal = float3(vertex_array[vid].normal);

    ColorInOut out;
    out.hasMeshColor = hasMeshColor[iId];
    out.meshColor = (meshColor[iId].data[0] == -1.0) ? vertColor.xyz : float3(meshColor[iId].data);
    out.reflection = reflectionValue[iId];

    out.T = normalize(float3(model[iId].data * float4(tangent, 0.0)));
    out.B = normalize(float3(model[iId].data * float4(bitangent, 0.0)));
    out.N = normalize(float3(model[iId].data * float4(normal,  0.0)));

    out.vertexPosition_ws = vertexPosition_ws;
    out.transparency = transparency[iId];
    out.hasLighting = hasLighting[iId];
    out.position = vp * vertexPosition_ws;
    out.uv = vertex_array[vid].texCoord1 * uvScaleValue[iId];
    out.shadowDarkness = 0.0;

    if(bool(hasLighting[iId])) {
        float4 vertexLightCoord = lvp * vertexPosition_ws;
        float4 texCoords = vertexLightCoord/vertexLightCoord.w;
        out.texture2UV = float4((texCoords / 2.0) + 0.5).xy;
        out.texture2UV.y = (1.0 - out.texture2UV.y); // need to flip metal texture vertically
        out.vertexDepth = texCoords.z;
        out.shadowDarkness = shadowDarkness;
    }

    float4 eye_position_ws = vertexPosition_ws - float4(float3(eyePos), 1.0);
    out.eyeVec = normalize(eye_position_ws);

    return out;
}

Solution

  • I doubt that what you quoted was the entire error message. When asking about an error message, always include the entire, verbatim message.

    That said, my best guess is that [[ point_coord ]] is not a valid attribute for vertex function output.

    The other thing is that you should declare the input parameter vertex_array as const so that Metal knows you're not going to write to it:

    device const vertex_t* vertex_array [[ buffer(0) ]]
    

    Vertex shaders which write to buffers or textures can only return void.