From dcbbda7702212ffb7b89f412ca6bbd536c75a827 Mon Sep 17 00:00:00 2001 From: Wez Furlong Date: Thu, 22 Jul 2021 09:46:08 -0700 Subject: [PATCH] allow images to overlay text This commit introduces a 4th draw pass for rendering sixel and iterm images that are attached to cells. Previously, a cell could container either text or an image from the perspective of the renderer. If it had an image then the glyph bitmap would be ignored in favor of the image. However, that causes sixel behavior to diverge from other terminals (https://github.com/wez/wezterm/issues/942) so we need to be render both of these. The simplest way to achieve this is to add a distinct set of texture coordinates for the attached image and then add a draw pass to alpha blend it over the glyph content. The sixel/iterm image processing stage is also adjusted to preserve the prior cell information and "simply" attach the image info to the cell. Previously, the cell would be replaced with a blank cell with the image attached. The result of this is that the notcurses-demo intro section can now render the orca "enveloped in the soft glow of glyphs" rather than caged in a black box. Note that there are some cases where the render turns blocky but I suspect that that is due to some other misunderstanding between wezterm and notcurses and that we'll root cause it as a follow up. --- term/src/screen.rs | 6 +++++ term/src/terminalstate.rs | 26 ++++++++++----------- wezterm-gui/src/fragment-common.glsl | 1 + wezterm-gui/src/glyph-frag.glsl | 3 +-- wezterm-gui/src/img-frag.glsl | 16 +++++++++++++ wezterm-gui/src/img-vertex.glsl | 16 +++++++++++++ wezterm-gui/src/quad.rs | 10 ++++++++ wezterm-gui/src/renderstate.rs | 34 +++++++++++++++++++++------- wezterm-gui/src/termwindow/render.rs | 19 +++++++++++++--- wezterm-gui/src/vertex-common.glsl | 3 +++ 10 files changed, 107 insertions(+), 27 deletions(-) create mode 100644 wezterm-gui/src/img-frag.glsl create mode 100644 wezterm-gui/src/img-vertex.glsl diff --git a/term/src/screen.rs b/term/src/screen.rs index c153a6acd..fa2da51a1 100644 --- a/term/src/screen.rs +++ b/term/src/screen.rs @@ -318,6 +318,12 @@ impl Screen { line.set_cell(x, cell.clone()) } + pub fn get_cell(&self, x: usize, y: VisibleRowIndex) -> Option<&Cell> { + let line_idx = self.phys_row(y); + let line = self.lines.get(line_idx)?; + line.cells().get(x) + } + pub fn clear_line(&mut self, y: VisibleRowIndex, cols: Range, attr: &CellAttributes) { let line_idx = self.phys_row(y); let line = self.line_mut(line_idx); diff --git a/term/src/terminalstate.rs b/term/src/terminalstate.rs index b39c561e5..2c97db3dc 100644 --- a/term/src/terminalstate.rs +++ b/term/src/terminalstate.rs @@ -1620,20 +1620,18 @@ impl TerminalState { cursor_x + width_in_cells ); for x in 0..width_in_cells { - self.screen_mut().set_cell( - cursor_x + x, - cursor_y, // + y as VisibleRowIndex, - &Cell::new( - ' ', - CellAttributes::default() - .set_image(Some(Box::new(ImageCell::new( - TextureCoordinate::new(xpos, ypos), - TextureCoordinate::new(xpos + x_delta, ypos + y_delta), - image_data.clone(), - )))) - .clone(), - ), - ); + let mut cell = self + .screen() + .get_cell(cursor_x + x, cursor_y) + .cloned() + .unwrap_or_else(|| Cell::new(' ', CellAttributes::default())); + cell.attrs_mut().set_image(Some(Box::new(ImageCell::new( + TextureCoordinate::new(xpos, ypos), + TextureCoordinate::new(xpos + x_delta, ypos + y_delta), + image_data.clone(), + )))); + + self.screen_mut().set_cell(cursor_x + x, cursor_y, &cell); xpos += x_delta; } ypos += y_delta; diff --git a/wezterm-gui/src/fragment-common.glsl b/wezterm-gui/src/fragment-common.glsl index 357a774fd..7c9c4e23a 100644 --- a/wezterm-gui/src/fragment-common.glsl +++ b/wezterm-gui/src/fragment-common.glsl @@ -5,6 +5,7 @@ precision highp float; in float o_has_color; in vec2 o_cursor; in vec2 o_tex; +in vec2 o_img_tex; in vec2 o_underline; in vec3 o_hsv; in vec4 o_bg_color; diff --git a/wezterm-gui/src/glyph-frag.glsl b/wezterm-gui/src/glyph-frag.glsl index 6764de4f3..109525d15 100644 --- a/wezterm-gui/src/glyph-frag.glsl +++ b/wezterm-gui/src/glyph-frag.glsl @@ -1,6 +1,5 @@ // This is the Glyph fragment shader. -// It is the last stage in drawing, and is responsible -// for laying down the glyph graphics on top of the other layers. +// It is responsible for laying down the glyph graphics on top of the other layers. // Note: fragment-common.glsl is automatically prepended! diff --git a/wezterm-gui/src/img-frag.glsl b/wezterm-gui/src/img-frag.glsl new file mode 100644 index 000000000..c7169e9ab --- /dev/null +++ b/wezterm-gui/src/img-frag.glsl @@ -0,0 +1,16 @@ +// This is the per-cell image attachment fragment shader. + +// Note: fragment-common.glsl is automatically prepended! + +uniform sampler2D atlas_nearest_sampler; + +void main() { + if (o_has_color >= 2.0) { + // Don't render the background image on anything other than + // the window_bg_layer. + discard; + return; + } + color = sample_texture(atlas_nearest_sampler, o_img_tex); + color = apply_hsv(color, o_hsv); +} diff --git a/wezterm-gui/src/img-vertex.glsl b/wezterm-gui/src/img-vertex.glsl new file mode 100644 index 000000000..abdf390b8 --- /dev/null +++ b/wezterm-gui/src/img-vertex.glsl @@ -0,0 +1,16 @@ +// This is the image vertex shader. +// It is responsible for placing per-cell attached images in the +// correct place on screen. + +// Note: vertex-common.glsl is automatically prepended! + +void main() { + pass_through_vertex(); + if (o_has_color == 2.0) { + // If we're the background image and we're not rendering + // the background layer, then move this off screen + gl_Position = off_screen(); + } else { + gl_Position = projection * vec4(position, 0.0, 1.0); + } +} diff --git a/wezterm-gui/src/quad.rs b/wezterm-gui/src/quad.rs index b5194ef75..d53ec1e80 100644 --- a/wezterm-gui/src/quad.rs +++ b/wezterm-gui/src/quad.rs @@ -24,6 +24,8 @@ pub struct Vertex { pub adjust: (f32, f32), // glyph texture pub tex: (f32, f32), + // iterm/sixel/image protocol texture + pub img_tex: (f32, f32), // underline texture pub underline: (f32, f32), // cursor texture @@ -52,6 +54,7 @@ pub struct Vertex { position, adjust, tex, + img_tex, underline, cursor, cursor_color, @@ -141,6 +144,13 @@ impl<'a> Quad<'a> { self.vert[V_BOT_RIGHT].tex = (coords.max_x(), coords.max_y()); } + pub fn set_image_texture(&mut self, coords: TextureRect) { + self.vert[V_TOP_LEFT].img_tex = (coords.min_x(), coords.min_y()); + self.vert[V_TOP_RIGHT].img_tex = (coords.max_x(), coords.min_y()); + self.vert[V_BOT_LEFT].img_tex = (coords.min_x(), coords.max_y()); + self.vert[V_BOT_RIGHT].img_tex = (coords.max_x(), coords.max_y()); + } + /// Apply bearing adjustment for the glyph texture. pub fn set_texture_adjust(&mut self, left: f32, top: f32, right: f32, bottom: f32) { self.vert[V_TOP_LEFT].adjust = (left, top); diff --git a/wezterm-gui/src/renderstate.rs b/wezterm-gui/src/renderstate.rs index 90fce2649..36ec0d31f 100644 --- a/wezterm-gui/src/renderstate.rs +++ b/wezterm-gui/src/renderstate.rs @@ -23,6 +23,7 @@ pub struct RenderState { pub background_prog: glium::Program, pub line_prog: glium::Program, pub glyph_prog: glium::Program, + pub img_prog: glium::Program, pub glyph_vertex_buffer: RefCell, pub glyph_index_buffer: IndexBuffer, pub quads: Quads, @@ -44,16 +45,15 @@ impl RenderState { let result = UtilSprites::new(&mut *glyph_cache.borrow_mut(), metrics); match result { Ok(util_sprites) => { - let background_prog = Self::compile_prog( - &context, - cfg!(target_os = "macos"), - Self::background_shader, - )?; - let line_prog = - Self::compile_prog(&context, cfg!(target_os = "macos"), Self::line_shader)?; + let do_gamma = cfg!(target_os = "macos"); + let background_prog = + Self::compile_prog(&context, do_gamma, Self::background_shader)?; + let line_prog = Self::compile_prog(&context, do_gamma, Self::line_shader)?; + + let glyph_prog = Self::compile_prog(&context, do_gamma, Self::glyph_shader)?; // Last prog outputs srgb for gamma correction - let glyph_prog = Self::compile_prog(&context, true, Self::glyph_shader)?; + let img_prog = Self::compile_prog(&context, true, Self::img_shader)?; let (glyph_vertex_buffer, glyph_index_buffer, quads) = Self::compute_vertices( config, @@ -70,6 +70,7 @@ impl RenderState { background_prog, line_prog, glyph_prog, + img_prog, glyph_vertex_buffer: RefCell::new(glyph_vertex_buffer), glyph_index_buffer, quads, @@ -155,6 +156,23 @@ impl RenderState { ) } + fn img_shader(version: &str) -> (String, String) { + ( + format!( + "#version {}\n{}\n{}", + version, + include_str!("vertex-common.glsl"), + include_str!("img-vertex.glsl") + ), + format!( + "#version {}\n{}\n{}", + version, + include_str!("fragment-common.glsl"), + include_str!("img-frag.glsl") + ), + ) + } + fn line_shader(version: &str) -> (String, String) { ( format!( diff --git a/wezterm-gui/src/termwindow/render.rs b/wezterm-gui/src/termwindow/render.rs index 348f98cd1..0ab0a9357 100644 --- a/wezterm-gui/src/termwindow/render.rs +++ b/wezterm-gui/src/termwindow/render.rs @@ -532,6 +532,20 @@ impl super::TermWindow { atlas_linear_sampler: atlas_linear_sampler, foreground_text_hsb: foreground_text_hsb, }, + &alpha_blending, + )?; + + // Pass 4: Draw image attachments + frame.draw( + &vb.bufs[vb.index], + &gl_state.glyph_index_buffer, + &gl_state.img_prog, + &uniform! { + projection: projection, + atlas_nearest_sampler: atlas_nearest_sampler, + atlas_linear_sampler: atlas_linear_sampler, + foreground_text_hsb: foreground_text_hsb, + }, &blend_but_set_alpha_to_one, )?; @@ -661,6 +675,7 @@ impl super::TermWindow { quad.set_bg_color(params.default_bg); quad.set_texture(params.white_space); + quad.set_image_texture(params.white_space); quad.set_texture_adjust(0., 0., 0., 0.); quad.set_underline(params.white_space); quad.set_cursor(params.white_space); @@ -822,7 +837,6 @@ impl super::TermWindow { style_params.underline_color, bg_color, )?; - continue; } if self.config.custom_block_glyphs && glyph_idx == 0 { @@ -1109,8 +1123,7 @@ impl super::TermWindow { quad.set_fg_color(glyph_color); quad.set_underline_color(underline_color); quad.set_bg_color(bg_color); - quad.set_texture(texture_rect); - quad.set_texture_adjust(0., 0., 0., 0.); + quad.set_image_texture(texture_rect); quad.set_underline(params.white_space); quad.set_has_color(true); quad.set_cursor( diff --git a/wezterm-gui/src/vertex-common.glsl b/wezterm-gui/src/vertex-common.glsl index 2cf5423f1..0c69cd549 100644 --- a/wezterm-gui/src/vertex-common.glsl +++ b/wezterm-gui/src/vertex-common.glsl @@ -6,6 +6,7 @@ precision highp float; in vec2 position; in vec2 adjust; in vec2 tex; +in vec2 img_tex; in vec2 underline; in vec4 bg_color; in vec4 fg_color; @@ -20,6 +21,7 @@ uniform mat4 projection; out float o_has_color; out vec2 o_cursor; out vec2 o_tex; +out vec2 o_img_tex; out vec2 o_underline; out vec3 o_hsv; out vec4 o_bg_color; @@ -29,6 +31,7 @@ out vec4 o_underline_color; void pass_through_vertex() { o_tex = tex; + o_img_tex = img_tex; o_has_color = has_color; o_fg_color = fg_color; o_bg_color = bg_color;