allow images to overlay text

This commit introduces a 4th draw pass for rendering sixel and
iterm images that are attached to cells.

Previously, a cell could container either text or an image from
the perspective of the renderer.  If it had an image then the glyph
bitmap would be ignored in favor of the image.

However, that causes sixel behavior to diverge from other terminals
(https://github.com/wez/wezterm/issues/942) so we need to be render
both of these.

The simplest way to achieve this is to add a distinct set of texture
coordinates for the attached image and then add a draw pass to alpha
blend it over the glyph content.

The sixel/iterm image processing stage is also adjusted to preserve
the prior cell information and "simply" attach the image info to
the cell.  Previously, the cell would be replaced with a blank cell
with the image attached.

The result of this is that the notcurses-demo intro section can
now render the orca "enveloped in the soft glow of glyphs" rather
than caged in a black box.

Note that there are some cases where the render turns blocky but
I suspect that that is due to some other misunderstanding between
wezterm and notcurses and that we'll root cause it as a follow up.

term/src/screen.rs

@@ -318,6 +318,12 @@ impl Screen {
         line.set_cell(x, cell.clone())
     }
 
+    pub fn get_cell(&self, x: usize, y: VisibleRowIndex) -> Option<&Cell> {
+        let line_idx = self.phys_row(y);
+        let line = self.lines.get(line_idx)?;
+        line.cells().get(x)
+    }
+
     pub fn clear_line(&mut self, y: VisibleRowIndex, cols: Range<usize>, attr: &CellAttributes) {
         let line_idx = self.phys_row(y);
         let line = self.line_mut(line_idx);

term/src/terminalstate.rs

@@ -1620,20 +1620,18 @@ impl TerminalState {
                 cursor_x + width_in_cells
             );
             for x in 0..width_in_cells {
-                self.screen_mut().set_cell(
+                let mut cell = self
-                    cursor_x + x,
+                    .screen()
-                    cursor_y, // + y as VisibleRowIndex,
+                    .get_cell(cursor_x + x, cursor_y)
-                    &Cell::new(
+                    .cloned()
-                        ' ',
+                    .unwrap_or_else(|| Cell::new(' ', CellAttributes::default()));
-                        CellAttributes::default()
+                cell.attrs_mut().set_image(Some(Box::new(ImageCell::new(
-                            .set_image(Some(Box::new(ImageCell::new(
                     TextureCoordinate::new(xpos, ypos),
                     TextureCoordinate::new(xpos + x_delta, ypos + y_delta),
                     image_data.clone(),
-                            ))))
+                ))));
-                            .clone(),
+
-                    ),
+                self.screen_mut().set_cell(cursor_x + x, cursor_y, &cell);
-                );
                 xpos += x_delta;
             }
             ypos += y_delta;

wezterm-gui/src/fragment-common.glsl

@@ -5,6 +5,7 @@ precision highp float;
 in float o_has_color;
 in vec2 o_cursor;
 in vec2 o_tex;
+in vec2 o_img_tex;
 in vec2 o_underline;
 in vec3 o_hsv;
 in vec4 o_bg_color;

wezterm-gui/src/glyph-frag.glsl

@@ -1,6 +1,5 @@
 // This is the Glyph fragment shader.
-// It is the last stage in drawing, and is responsible
+// It is responsible for laying down the glyph graphics on top of the other layers.
-// for laying down the glyph graphics on top of the other layers.
 
 // Note: fragment-common.glsl is automatically prepended!
 

wezterm-gui/src/img-frag.glsl

@@ -0,0 +1,16 @@
+// This is the per-cell image attachment fragment shader.
+
+// Note: fragment-common.glsl is automatically prepended!
+
+uniform sampler2D atlas_nearest_sampler;
+
+void main() {
+  if (o_has_color >= 2.0) {
+    // Don't render the background image on anything other than
+    // the window_bg_layer.
+    discard;
+    return;
+  }
+  color = sample_texture(atlas_nearest_sampler, o_img_tex);
+  color = apply_hsv(color, o_hsv);
+}

wezterm-gui/src/img-vertex.glsl

@@ -0,0 +1,16 @@
+// This is the image vertex shader.
+// It is responsible for placing per-cell attached images in the
+// correct place on screen.
+
+// Note: vertex-common.glsl is automatically prepended!
+
+void main() {
+  pass_through_vertex();
+  if (o_has_color == 2.0) {
+    // If we're the background image and we're not rendering
+    // the background layer, then move this off screen
+    gl_Position = off_screen();
+  } else {
+    gl_Position = projection * vec4(position, 0.0, 1.0);
+  }
+}

wezterm-gui/src/quad.rs

@@ -24,6 +24,8 @@ pub struct Vertex {
     pub adjust: (f32, f32),
     // glyph texture
     pub tex: (f32, f32),
+    // iterm/sixel/image protocol texture
+    pub img_tex: (f32, f32),
     // underline texture
     pub underline: (f32, f32),
     // cursor texture
@@ -52,6 +54,7 @@ pub struct Vertex {
     position,
     adjust,
     tex,
+    img_tex,
     underline,
     cursor,
     cursor_color,
@@ -141,6 +144,13 @@ impl<'a> Quad<'a> {
         self.vert[V_BOT_RIGHT].tex = (coords.max_x(), coords.max_y());
     }
 
+    pub fn set_image_texture(&mut self, coords: TextureRect) {
+        self.vert[V_TOP_LEFT].img_tex = (coords.min_x(), coords.min_y());
+        self.vert[V_TOP_RIGHT].img_tex = (coords.max_x(), coords.min_y());
+        self.vert[V_BOT_LEFT].img_tex = (coords.min_x(), coords.max_y());
+        self.vert[V_BOT_RIGHT].img_tex = (coords.max_x(), coords.max_y());
+    }
+
     /// Apply bearing adjustment for the glyph texture.
     pub fn set_texture_adjust(&mut self, left: f32, top: f32, right: f32, bottom: f32) {
         self.vert[V_TOP_LEFT].adjust = (left, top);

wezterm-gui/src/renderstate.rs

@@ -23,6 +23,7 @@ pub struct RenderState {
     pub background_prog: glium::Program,
     pub line_prog: glium::Program,
     pub glyph_prog: glium::Program,
+    pub img_prog: glium::Program,
     pub glyph_vertex_buffer: RefCell<TripleVertexBuffer>,
     pub glyph_index_buffer: IndexBuffer<u32>,
     pub quads: Quads,
@@ -44,16 +45,15 @@ impl RenderState {
             let result = UtilSprites::new(&mut *glyph_cache.borrow_mut(), metrics);
             match result {
                 Ok(util_sprites) => {
-                    let background_prog = Self::compile_prog(
+                    let do_gamma = cfg!(target_os = "macos");
-                        &context,
-                        cfg!(target_os = "macos"),
-                        Self::background_shader,
-                    )?;
-                    let line_prog =
-                        Self::compile_prog(&context, cfg!(target_os = "macos"), Self::line_shader)?;
 
+                    let background_prog =
+                        Self::compile_prog(&context, do_gamma, Self::background_shader)?;
+                    let line_prog = Self::compile_prog(&context, do_gamma, Self::line_shader)?;
+
+                    let glyph_prog = Self::compile_prog(&context, do_gamma, Self::glyph_shader)?;
                     // Last prog outputs srgb for gamma correction
-                    let glyph_prog = Self::compile_prog(&context, true, Self::glyph_shader)?;
+                    let img_prog = Self::compile_prog(&context, true, Self::img_shader)?;
 
                     let (glyph_vertex_buffer, glyph_index_buffer, quads) = Self::compute_vertices(
                         config,
@@ -70,6 +70,7 @@ impl RenderState {
                         background_prog,
                         line_prog,
                         glyph_prog,
+                        img_prog,
                         glyph_vertex_buffer: RefCell::new(glyph_vertex_buffer),
                         glyph_index_buffer,
                         quads,
@@ -155,6 +156,23 @@ impl RenderState {
         )
     }
 
+    fn img_shader(version: &str) -> (String, String) {
+        (
+            format!(
+                "#version {}\n{}\n{}",
+                version,
+                include_str!("vertex-common.glsl"),
+                include_str!("img-vertex.glsl")
+            ),
+            format!(
+                "#version {}\n{}\n{}",
+                version,
+                include_str!("fragment-common.glsl"),
+                include_str!("img-frag.glsl")
+            ),
+        )
+    }
+
     fn line_shader(version: &str) -> (String, String) {
         (
             format!(

wezterm-gui/src/termwindow/render.rs

@@ -532,6 +532,20 @@ impl super::TermWindow {
                 atlas_linear_sampler:  atlas_linear_sampler,
                 foreground_text_hsb: foreground_text_hsb,
             },
+            &alpha_blending,
+        )?;
+
+        // Pass 4: Draw image attachments
+        frame.draw(
+            &vb.bufs[vb.index],
+            &gl_state.glyph_index_buffer,
+            &gl_state.img_prog,
+            &uniform! {
+                projection: projection,
+                atlas_nearest_sampler:  atlas_nearest_sampler,
+                atlas_linear_sampler:  atlas_linear_sampler,
+                foreground_text_hsb: foreground_text_hsb,
+            },
             &blend_but_set_alpha_to_one,
         )?;
 
@@ -661,6 +675,7 @@ impl super::TermWindow {
 
             quad.set_bg_color(params.default_bg);
             quad.set_texture(params.white_space);
+            quad.set_image_texture(params.white_space);
             quad.set_texture_adjust(0., 0., 0., 0.);
             quad.set_underline(params.white_space);
             quad.set_cursor(params.white_space);
@@ -822,7 +837,6 @@ impl super::TermWindow {
                             style_params.underline_color,
                             bg_color,
                         )?;
-                        continue;
                     }
 
                     if self.config.custom_block_glyphs && glyph_idx == 0 {
@@ -1109,8 +1123,7 @@ impl super::TermWindow {
         quad.set_fg_color(glyph_color);
         quad.set_underline_color(underline_color);
         quad.set_bg_color(bg_color);
-        quad.set_texture(texture_rect);
+        quad.set_image_texture(texture_rect);
-        quad.set_texture_adjust(0., 0., 0., 0.);
         quad.set_underline(params.white_space);
         quad.set_has_color(true);
         quad.set_cursor(

wezterm-gui/src/vertex-common.glsl

@@ -6,6 +6,7 @@ precision highp float;
 in vec2 position;
 in vec2 adjust;
 in vec2 tex;
+in vec2 img_tex;
 in vec2 underline;
 in vec4 bg_color;
 in vec4 fg_color;
@@ -20,6 +21,7 @@ uniform mat4 projection;
 out float o_has_color;
 out vec2 o_cursor;
 out vec2 o_tex;
+out vec2 o_img_tex;
 out vec2 o_underline;
 out vec3 o_hsv;
 out vec4 o_bg_color;
@@ -29,6 +31,7 @@ out vec4 o_underline_color;
 
 void pass_through_vertex() {
   o_tex = tex;
+  o_img_tex = img_tex;
   o_has_color = has_color;
   o_fg_color = fg_color;
   o_bg_color = bg_color;