Skip to content

feat: add color-key transparency #160

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
CyberShadow wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: add color-key transparency #160

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions include/render/fx_renderer/shaders.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,11 @@ struct tex_shader {

GLint discard_transparent;

// Color-key transparency
GLint colorkey_enabled;
GLint colorkey_src;
GLint colorkey_dst;

GLint clip_size;
GLint clip_position;
GLint clip_radius_top_left;
Expand Down
4 changes: 4 additions & 0 deletions include/scenefx/render/pass.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,10 @@ struct fx_render_texture_options {
int corner_radius;
bool discard_transparent;
struct clipped_region clipped_region;
// Color-key transparency
bool colorkey_enabled;
float colorkey_src[4]; // Source RGBA to match
float colorkey_dst[4]; // Destination RGBA replacement
};

struct fx_render_rect_options {
Expand Down
16 changes: 16 additions & 0 deletions include/scenefx/types/wlr_scene.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,12 @@ struct wlr_scene_buffer {
enum corner_location corners;

float opacity;

// Color-key transparency: replace colorkey_src color with colorkey_dst
bool colorkey_enabled;
float colorkey_src[4]; // Source RGBA to match (normalized 0.0-1.0)
float colorkey_dst[4]; // Destination RGBA replacement (normalized 0.0-1.0)

enum wlr_scale_filter_mode filter_mode;
struct wlr_fbox src_box;
int dst_width, dst_height;
Expand Down Expand Up @@ -761,6 +767,16 @@ void wlr_scene_buffer_set_transform(struct wlr_scene_buffer *scene_buffer,
void wlr_scene_buffer_set_opacity(struct wlr_scene_buffer *scene_buffer,
float opacity);

/**
* Sets color-key transparency for this buffer.
* Pixels matching colorkey_src (within tolerance) are replaced with colorkey_dst.
* Colors are specified as normalized RGBA values (0.0-1.0).
* Set enabled to false to disable colorkey transparency.
*/
void wlr_scene_buffer_set_colorkey(struct wlr_scene_buffer *scene_buffer,
bool enabled, const float colorkey_src[static 4],
const float colorkey_dst[static 4]);

/**
* Sets the filter mode to use when scaling the buffer
*/
Expand Down
9 changes: 8 additions & 1 deletion render/fx_renderer/fx_pass.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -344,7 +344,8 @@ void fx_render_pass_add_texture(struct fx_gles_render_pass *pass,
bool has_alpha = texture->has_alpha
|| alpha < 1.0
|| fx_options->corner_radius > 0
|| fx_options->discard_transparent;
|| fx_options->discard_transparent
|| (fx_options->colorkey_enabled && fx_options->colorkey_dst[3] < 1.0);
setup_blending(!has_alpha ? WLR_RENDER_BLEND_MODE_NONE : options->blend_mode);

pixman_region32_t clip_region;
Expand Down Expand Up @@ -394,6 +395,12 @@ void fx_render_pass_add_texture(struct fx_gles_render_pass *pass,
glUniform2f(shader->size, clip_box->width, clip_box->height);
glUniform2f(shader->position, clip_box->x, clip_box->y);
glUniform1f(shader->discard_transparent, fx_options->discard_transparent);

// Color-key transparency
glUniform1i(shader->colorkey_enabled, fx_options->colorkey_enabled);
glUniform4fv(shader->colorkey_src, 1, fx_options->colorkey_src);
glUniform4fv(shader->colorkey_dst, 1, fx_options->colorkey_dst);

glUniform1f(shader->radius_top_left, (CORNER_LOCATION_TOP_LEFT & corners) == CORNER_LOCATION_TOP_LEFT ?
fx_options->corner_radius : 0);
glUniform1f(shader->radius_top_right, (CORNER_LOCATION_TOP_RIGHT & corners) == CORNER_LOCATION_TOP_RIGHT ?
Expand Down
18 changes: 17 additions & 1 deletion render/fx_renderer/gles2/shaders/tex.frag
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,11 @@ uniform float clip_radius_bottom_right;

uniform bool discard_transparent;

// Color-key transparency
uniform bool colorkey_enabled;
uniform vec4 colorkey_src;
uniform vec4 colorkey_dst;

vec4 sample_texture() {
#if SOURCE == SOURCE_TEXTURE_RGBA || SOURCE == SOURCE_TEXTURE_EXTERNAL
return texture2D(tex, v_texcoord);
Expand All @@ -55,6 +60,17 @@ vec4 sample_texture() {
float corner_alpha(vec2 size, vec2 position, float round_tl, float round_tr, float round_bl, float round_br);

void main() {
vec4 c = sample_texture();

// Color-key transparency: replace matching colors
if (colorkey_enabled) {
// Use small epsilon for float comparison (~1/255 tolerance)
vec4 diff = abs(c - colorkey_src);
if (all(lessThan(diff, vec4(0.004)))) {
c = colorkey_dst;
}
}

float quad_corner_alpha = corner_alpha(
size - 0.5,
position + 0.25,
Expand All @@ -74,7 +90,7 @@ void main() {
clip_radius_bottom_right
);

gl_FragColor = mix(sample_texture() * alpha, vec4(0.0), quad_corner_alpha) * clip_corner_alpha;
gl_FragColor = mix(c * alpha, vec4(0.0), quad_corner_alpha) * clip_corner_alpha;

if (discard_transparent && gl_FragColor.a == 0.0) {
discard;
Expand Down
18 changes: 17 additions & 1 deletion render/fx_renderer/gles3/shaders/tex.frag
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,11 @@ uniform float clip_radius_bottom_right;

uniform bool discard_transparent;

// Color-key transparency
uniform bool colorkey_enabled;
uniform vec4 colorkey_src;
uniform vec4 colorkey_dst;

out vec4 fragColor;

vec4 sample_texture() {
Expand All @@ -55,6 +60,17 @@ vec4 sample_texture() {
float corner_alpha(vec2 size, vec2 position, float round_tl, float round_tr, float round_bl, float round_br);

void main() {
vec4 c = sample_texture();

// Color-key transparency: replace matching colors
if (colorkey_enabled) {
Copy link
Copy Markdown
Member

@the-eater the-eater Jan 14, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please use #define/#if and a new shader object for this, so perfomance stays identical when disabled

Copy link
Copy Markdown
Author

@CyberShadow CyberShadow Jan 14, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

AFAIU, because this is a uniform, the if should already be inlined. Are you aware of this causing any impact beyond shader compilation?

Copy link
Copy Markdown
Author

@CyberShadow CyberShadow Jan 14, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure how accurate this is, but

https://gist.github.com/CyberShadow/d5608b15998194aa75504016b7dd5b75

$ nix-shell --run ./run_benchmark.sh                   
Compiling branchless benchmark...
Compiling branching benchmark...

Running benchmarks...
========================================
OpenGL Version: 4.6 (Core Profile) Mesa 25.1.5
Renderer: AMD Radeon RX 6900 XT (radeonsi, navi21, LLVM 19.1.7, DRM 3.63, 6.15.5)

=== BRANCHLESS SHADER BENCHMARK ===
Resolution: 1920x1080
Iterations: 100000
Total time: 0.729 seconds
Average: 0.0073 ms/frame
FPS: 137188.91

OpenGL Version: 4.6 (Core Profile) Mesa 25.1.5
Renderer: AMD Radeon RX 6900 XT (radeonsi, navi21, LLVM 19.1.7, DRM 3.63, 6.15.5)

=== BRANCHING SHADER BENCHMARK (80% false, 20% true) ===
Resolution: 1920x1080
Iterations: 100000
Total time: 0.729 seconds
Average: 0.0073 ms/frame
FPS: 137139.10

========================================
Benchmark complete!

Copy link
Copy Markdown
Member

@ErikReider ErikReider Jan 14, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure how accurate this is, but

The issue isn't with high-end hardware, but with power limited "weak" (in comparison) laptop GPUs, especially with all the other things being rendered during the same frame, like poorly damaged toplevels, blur, shadows, etc.

Copy link
Copy Markdown
Author

@CyberShadow CyberShadow Jan 14, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The issue isn't with high-end hardware, but with power limited "weak" (in comparison) laptop GPUs

Fair, here are the results on my 8-year-old laptop on the integrated GPU:

Compiling branchless benchmark...
Compiling branching benchmark...

Running benchmarks...
========================================
OpenGL Version: 4.6 (Core Profile) Mesa 25.2.4
Renderer: Mesa Intel(R) UHD Graphics 620 (KBL GT2)

=== BRANCHLESS SHADER BENCHMARK ===
Resolution: 1920x1080
Iterations: 10000
Total time: 4.240 seconds
Average: 0.4240 ms/frame
FPS: 2358.35

OpenGL Version: 4.6 (Core Profile) Mesa 25.2.4
Renderer: Mesa Intel(R) UHD Graphics 620 (KBL GT2)

=== BRANCHING SHADER BENCHMARK (80% false, 20% true) ===
Resolution: 1920x1080
Iterations: 10000
Total time: 4.227 seconds
Average: 0.4227 ms/frame
FPS: 2365.82

========================================
Benchmark complete!

especially with all the other things being rendered during the same frame, like poorly damaged toplevels, blur, shadows, etc.

If you insist, I'll apply the patch if that's what you want. Or do you want me to test any specific combination?

Copy link
Copy Markdown
Member

@ErikReider ErikReider Feb 10, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

AFAIU, because this is a uniform, the if should already be inlined. Are you aware of this causing any impact beyond shader compilation?

Sorry, I missed this comment, yeah this is true so shouldn't be anything to worry about, but I am worrying about the increasing size and number of uniforms of the shader though. Might be too much for low end hardware, but would require testing ofc :)

Copy link
Copy Markdown
Author

@CyberShadow CyberShadow Feb 10, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, fair! Let me have a look.

// Use small epsilon for float comparison (~1/255 tolerance)
vec4 diff = abs(c - colorkey_src);
if (all(lessThan(diff, vec4(0.004)))) {
c = colorkey_dst;
}
}

float quad_corner_alpha = corner_alpha(
size - 0.5,
position + 0.25,
Expand All @@ -74,7 +90,7 @@ void main() {
clip_radius_bottom_right
);

fragColor = mix(sample_texture() * alpha, vec4(0.0), quad_corner_alpha) * clip_corner_alpha;
fragColor = mix(c * alpha, vec4(0.0), quad_corner_alpha) * clip_corner_alpha;

if (discard_transparent && fragColor.a == 0.0) {
discard;
Expand Down
9 changes: 7 additions & 2 deletions render/fx_renderer/shaders.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,8 +265,8 @@ bool link_quad_grad_round_program(struct quad_grad_round_shader *shader, GLint c
}

bool link_tex_program(struct tex_shader *shader, GLint client_version, enum fx_tex_shader_source source) {
GLchar frag_src_part[2048];
GLchar frag_src[4096];
GLchar frag_src_part[4096];
GLchar frag_src[8192];
if (client_version > 2) {
snprintf(frag_src_part, sizeof(frag_src_part),
tex_frag_gles3_src, source);
Expand Down Expand Up @@ -298,6 +298,11 @@ bool link_tex_program(struct tex_shader *shader, GLint client_version, enum fx_t
shader->radius_bottom_right = glGetUniformLocation(prog, "radius_bottom_right");
shader->discard_transparent = glGetUniformLocation(prog, "discard_transparent");

// Color-key transparency
shader->colorkey_enabled = glGetUniformLocation(prog, "colorkey_enabled");
shader->colorkey_src = glGetUniformLocation(prog, "colorkey_src");
shader->colorkey_dst = glGetUniformLocation(prog, "colorkey_dst");

shader->clip_size = glGetUniformLocation(prog, "clip_size");
shader->clip_position = glGetUniformLocation(prog, "clip_position");
shader->clip_radius_top_left = glGetUniformLocation(prog, "clip_radius_top_left");
Expand Down
46 changes: 46 additions & 0 deletions types/scene/wlr_scene.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,11 @@ static void scene_node_opaque_region(struct wlr_scene_node *node, int x, int y,
return;
}

// Colorkey with transparent destination means buffer is not opaque
if (scene_buffer->colorkey_enabled && scene_buffer->colorkey_dst[3] < 1.0f) {
return;
}

if (!scene_buffer->buffer_is_opaque) {
pixman_region32_copy(opaque, &scene_buffer->opaque_region);
pixman_region32_intersect_rect(opaque, opaque, 0, 0, width, height);
Expand Down Expand Up @@ -1550,6 +1555,34 @@ void wlr_scene_buffer_set_opacity(struct wlr_scene_buffer *scene_buffer,
scene_node_update(&scene_buffer->node, NULL);
}

void wlr_scene_buffer_set_colorkey(struct wlr_scene_buffer *scene_buffer,
bool enabled, const float colorkey_src[static 4],
const float colorkey_dst[static 4]) {
bool changed = scene_buffer->colorkey_enabled != enabled;
if (enabled) {
for (int i = 0; i < 4; i++) {
if (scene_buffer->colorkey_src[i] != colorkey_src[i] ||
scene_buffer->colorkey_dst[i] != colorkey_dst[i]) {
changed = true;
break;
}
}
Comment on lines +1563 to +1569
Copy link
Copy Markdown
Member

@ErikReider ErikReider Feb 10, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not really a fan of the loops, but I guess the compiler will unroll it though.

Any one else with input regarding this?

}

if (!changed) {
return;
}

scene_buffer->colorkey_enabled = enabled;
if (enabled) {
for (int i = 0; i < 4; i++) {
scene_buffer->colorkey_src[i] = colorkey_src[i];
scene_buffer->colorkey_dst[i] = colorkey_dst[i];
}
Comment on lines +1578 to +1581
Copy link
Copy Markdown
Member

@ErikReider ErikReider Feb 10, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same here

}
scene_node_update(&scene_buffer->node, NULL);
}

void wlr_scene_buffer_set_filter_mode(struct wlr_scene_buffer *scene_buffer,
enum wlr_scale_filter_mode filter_mode) {
if (scene_buffer->filter_mode == filter_mode) {
Expand Down Expand Up @@ -2063,6 +2096,19 @@ static void scene_entry_render(struct render_list_entry *entry, const struct ren
.corners = buffer_corners,
.corner_radius = scene_buffer->corner_radius * data->scale,
.clipped_region = {0},
.colorkey_enabled = scene_buffer->colorkey_enabled,
.colorkey_src = {
scene_buffer->colorkey_src[0],
scene_buffer->colorkey_src[1],
scene_buffer->colorkey_src[2],
scene_buffer->colorkey_src[3],
},
.colorkey_dst = {
scene_buffer->colorkey_dst[0],
scene_buffer->colorkey_dst[1],
scene_buffer->colorkey_dst[2],
scene_buffer->colorkey_dst[3],
},
};

fx_render_pass_add_texture(data->render_pass, &tex_options);
Expand Down