I'm trying to write a shader for unity that will remove all overlapping fragments of the underlying objects. I found different solutions for this problem, which were really helpful.
For me this Link (Unity shader highlighting overlaps) was the most usefull one.
But now i have another problem. In the pictures you can see 6 buttons which normally have the same size and a transparent background. If one of these buttons gets selected, it overlaps its neighbours.
The following picture shows how it should look like with a custom shader. I have solved the problem by cutting a hole trough the background to show the image, but now i want to add a text to these buttons; if i would do the same thing again, the text would look scraggy.
The following code shows you how i solved my problem by cutting a hole into the transparent box:
Shader "Custom/GUI/Mask" {
Properties
{
_Color("Color (white = none)", COLOR) = (1,1,1,1)
_MainTex("Texture", 2D) = "white" {}
_CutOff("Cut off", Range(-0.001,1)) = 0.1
}
SubShader
{
Tags{ "RenderType" = "Transparent" "Queue" = "Geometry" }
LOD 100
Blend SrcAlpha OneMinusSrcAlpha
ZWrite off
Pass
{
Stencil
{
Ref 0
Comp Equal
Pass IncrSat
Fail IncrSat
}
CGPROGRAM
#pragma vertex vert
#pragma fragment frag
#include "UnityCG.cginc"
uniform sampler2D _MainTex;
uniform float4 _MainTex_ST;
float4 _Color;
float _CutOff;
struct appdata
{
float4 vertex : POSITION;
float2 uv : TEXCOORD0;
};
struct v2f
{
float4 vertex : SV_POSITION;
float2 uv : TEXCOORD0;
};
v2f vert(appdata v)
{
v2f o;
o.vertex = mul(UNITY_MATRIX_MVP, v.vertex);
o.uv = TRANSFORM_TEX(v.uv, _MainTex);
return o;
}
fixed4 frag(v2f i) : SV_Target
{
float4 color = tex2D(_MainTex, i.uv);
color.rgb *= _Color.rgb;
color.a *= _Color.a;
if (color.a <= _CutOff)
{
discard;
}
return color;
}
ENDCG
}
Pass
{
Blend SrcAlpha OneMinusSrcAlpha
ZWrite off
Stencil
{
Ref 1
Comp Less
}
CGPROGRAM
#pragma vertex vert
#pragma fragment frag
#include "UnityCG.cginc"
float4 _Color;
uniform sampler2D _MainTex;
struct appdata
{
float4 vertex : POSITION;
float uv : TEXCOORD0;
};
struct v2f
{
float4 vertex : SV_POSITION;
float uv : TEXCOORD0;
};
v2f vert(appdata v)
{
v2f o;
o.vertex = mul(UNITY_MATRIX_MVP, v.vertex);
o.uv = v.uv;
return o;
}
fixed4 frag(v2f i) : SV_Target
{
fixed4 color = tex2D(_MainTex, i.uv);
color.a = 0;
return color;
}
ENDCG
}
Pass
{
Blend SrcAlpha OneMinusSrcAlpha
Stencil
{
Ref 2
Comp Less
}
CGPROGRAM
#pragma vertex vert
#pragma fragment frag
#include "UnityCG.cginc"
uniform sampler2D _MainTex;
float4 _Color;
struct appdata
{
float4 vertex : POSITION;
float uv : TEXCOORD0;
};
struct v2f
{
float4 vertex : SV_POSITION;
float uv : TEXCOORD0;
};
v2f vert(appdata v)
{
v2f o;
o.vertex = mul(UNITY_MATRIX_MVP, v.vertex);
o.uv = v.uv;
return o;
}
fixed4 frag(v2f i) : SV_Target
{
fixed4 color = tex2D(_MainTex, i.uv);
color.a = 0;
return color;
}
ENDCG
}
}
}
This picture shows, how it would look like without a custom shader which removes the overlapping parts. In this picture you can also see, that the transparent background has normally a stencil value of 0 and when it overlaps, the value changes to 1. The problem is that the image on the background has also a stencil value of 1. So if i would remove all object with a stencil value of 1, i would remove all images on the transparent backgrounds. By the way the image and the background contain the same shader.
Can you please help me to solve the problem without cutting a hole into the transparent box? The cutting through option has the problem, if i have round images, which have some transparent pixels, the background shines through very bright.
Thank you very much for your help.
Finally I have solved the problem myself. For those who have the same problem, here is the solution. I created two different shaders. I created one for the background and one for the picture in front of the background.
The following code is for the background:
Shader "Custom/GUI/Background" {
Properties
{
_Color("Color (white = none)", COLOR) = (1,1,1,1)
_MainTex("Texture", 2D) = "white" {}
_CutOff("Cut off", Range(-0.001,1)) = 0.1
}
SubShader
{
Tags{ "RenderType" = "Transparent" "Queue" = "Geometry-100" "LightMode" = "ForwardBase" }
LOD 100
Blend SrcAlpha OneMinusSrcAlpha
ZWrite off
Pass
{
Stencil
{
Ref 0
Comp Equal
Pass IncrSat
Fail IncrSat
}
CGPROGRAM
#pragma vertex vert
#pragma fragment frag
#include "UnityCG.cginc"
uniform sampler2D _MainTex;
uniform float4 _MainTex_ST;
float4 _Color;
float _CutOff;
struct appdata
{
float4 vertex : POSITION;
float2 uv : TEXCOORD0;
};
struct v2f
{
float4 vertex : SV_POSITION;
float2 uv : TEXCOORD0;
};
v2f vert(appdata v)
{
v2f o;
o.vertex = mul(UNITY_MATRIX_MVP, v.vertex);
o.uv = TRANSFORM_TEX(v.uv, _MainTex);
return o;
}
fixed4 frag(v2f i) : SV_Target
{
float4 color = tex2D(_MainTex, i.uv);
color.rgb *= _Color.rgb;
color.a *= _Color.a;
return color;
}
ENDCG
}
}
}
And this one for the image:
Shader "Custom/GUI/ImageShader"
{
Properties
{
_MainTex("Texture", 2D) = "white" {}
_Color("Color (white = none)", COLOR) = (1,1,1,1)
}
SubShader
{
Tags { "RenderType" = "Transparent" }
LOD 100
Blend SrcAlpha OneMinusSrcAlpha
ZWrite off
Pass
{
Stencil
{
Ref 0
Comp Equal
Pass IncrSat
Fail IncrSat
}
}
// picture layer
Pass
{
Stencil
{
Ref 2
Comp Equal
}
CGPROGRAM
#pragma vertex vert
#pragma fragment frag
// make fog work
#pragma multi_compile_fog
#include "UnityCG.cginc"
struct appdata
{
float4 vertex : POSITION;
float2 uv : TEXCOORD0;
};
struct v2f
{
float2 uv : TEXCOORD0;
UNITY_FOG_COORDS(1)
float4 vertex : SV_POSITION;
};
sampler2D _MainTex;
float4 _MainTex_ST;
float4 _Color;
v2f vert(appdata v)
{
v2f o;
o.vertex = UnityObjectToClipPos(v.vertex);
o.uv = TRANSFORM_TEX(v.uv, _MainTex);
UNITY_TRANSFER_FOG(o, o.vertex);
return o;
}
fixed4 frag(v2f i) : SV_Target
{
// sample the texture
fixed4 col = tex2D(_MainTex, i.uv);
col.rgb *= _Color.rgb;
col.a *= _Color.a;
// apply fog
UNITY_APPLY_FOG(i.fogCoord, col);
return col;
}
ENDCG
}
}
}
The Hierarchy is now: the background is on the bottom, then text and above the image.
To be honest, I’m not sure why it is working and I know that this is not the best answer, but I hope I can help all the people, which are having the same or nearly the same problem.