From a267ebb5763d579792d7791534973d0f9f2eddba Mon Sep 17 00:00:00 2001 From: Wez Furlong Date: Wed, 8 Aug 2018 07:07:31 -0700 Subject: [PATCH] add theoretical support for storing Images in the Surface There's basic rendering also, but it is not complete. Needs more tests; will come back to those once more scaffolding is in place. --- termwiz/src/cell.rs | 5 + termwiz/src/escape/osc.rs | 14 +-- termwiz/src/image.rs | 69 ++++++------- termwiz/src/render/terminfo.rs | 129 +++++++++++++++++------ termwiz/src/render/windows.rs | 18 ++++ termwiz/src/surface/change.rs | 36 +++++++ termwiz/src/surface/mod.rs | 184 ++++++++++++++++++++++++++++++++- 7 files changed, 380 insertions(+), 75 deletions(-) diff --git a/termwiz/src/cell.rs b/termwiz/src/cell.rs index e24575d0f..006f46752 100644 --- a/termwiz/src/cell.rs +++ b/termwiz/src/cell.rs @@ -163,6 +163,11 @@ impl CellAttributes { self } + pub fn set_image(&mut self, image: Option>) -> &mut Self { + self.image = image; + self + } + /// Clone the attributes, but exclude fancy extras such /// as hyperlinks or future sprite things pub fn clone_sgr_only(&self) -> Self { diff --git a/termwiz/src/escape/osc.rs b/termwiz/src/escape/osc.rs index 8be30d986..a4cd6f156 100644 --- a/termwiz/src/escape/osc.rs +++ b/termwiz/src/escape/osc.rs @@ -270,21 +270,21 @@ pub enum ITermProprietary { #[derive(Debug, Clone, PartialEq, Eq)] pub struct ITermFileData { /// file name - name: Option, + pub name: Option, /// size of the data in bytes; this is used by iterm to show progress /// while waiting for the rest of the payload - size: Option, + pub size: Option, /// width to render - width: ITermDimension, + pub width: ITermDimension, /// height to render - height: ITermDimension, + pub height: ITermDimension, /// if true, preserve aspect ratio when fitting to width/height - preserve_aspect_ratio: bool, + pub preserve_aspect_ratio: bool, /// if true, attempt to display in the terminal rather than downloading to /// the users download directory - inline: bool, + pub inline: bool, /// The data to transfer - data: Vec, + pub data: Vec, } impl ITermFileData { diff --git a/termwiz/src/image.rs b/termwiz/src/image.rs index 31b85d726..c8bf986bd 100644 --- a/termwiz/src/image.rs +++ b/termwiz/src/image.rs @@ -11,15 +11,25 @@ // protocol appears to track the images out of band as attachments with // z-order. -use failure::Error; -use image_crate::load_from_memory; use ordered_float::NotNaN; use std::rc::Rc; #[derive(Debug, Clone, PartialEq, Eq)] pub struct TextureCoordinate { - x: NotNaN, - y: NotNaN, + pub x: NotNaN, + pub y: NotNaN, +} + +impl TextureCoordinate { + pub fn new(x: NotNaN, y: NotNaN) -> Self { + Self { x, y } + } + + pub fn new_f32(x: f32, y: f32) -> Self { + let x = NotNaN::new(x).unwrap(); + let y = NotNaN::new(y).unwrap(); + Self::new(x, y) + } } /// Tracks data for displaying an image in the place of the normal cell @@ -39,39 +49,34 @@ pub struct ImageCell { data: Rc, } +impl ImageCell { + pub fn new( + top_left: TextureCoordinate, + bottom_right: TextureCoordinate, + data: Rc, + ) -> Self { + Self { + top_left, + bottom_right, + data, + } + } +} + static IMAGE_ID: ::std::sync::atomic::AtomicUsize = ::std::sync::atomic::ATOMIC_USIZE_INIT; #[derive(Clone, Debug, PartialEq, Eq)] pub struct ImageData { id: usize, - /// Width of the image, in pixels - width: usize, - /// Height of the image, in pixels, - height: usize, - /// The image data bytes. Data is SRGBA, 32 bits per pixel + /// The image data bytes. Data is the native image file format data: Vec, } impl ImageData { - /// Guess the image format from the contained buffer and return the - /// decoded image data. - pub fn load_from_memory(buffer: &[u8]) -> Result { - let img = load_from_memory(buffer)?.to_rgba(); - let width = img.width() as usize; - let height = img.height() as usize; - let data = img.into_raw(); - - Ok(Self::with_raw_data(width, height, data)) - } - - pub fn with_raw_data(width: usize, height: usize, data: Vec) -> Self { + /// Create a new ImageData struct with the provided raw data. + pub fn with_raw_data(data: Vec) -> Self { let id = IMAGE_ID.fetch_add(1, ::std::sync::atomic::Ordering::Relaxed); - Self { - id, - width, - height, - data, - } + Self { id, data } } #[inline] @@ -83,14 +88,4 @@ impl ImageData { pub fn id(&self) -> usize { self.id } - - #[inline] - pub fn width(&self) -> usize { - self.width - } - - #[inline] - pub fn height(&self) -> usize { - self.height - } } diff --git a/termwiz/src/render/terminfo.rs b/termwiz/src/render/terminfo.rs index 66e738856..5cf512e21 100644 --- a/termwiz/src/render/terminfo.rs +++ b/termwiz/src/render/terminfo.rs @@ -3,8 +3,9 @@ use caps::{Capabilities, ColorLevel}; use cell::{AttributeChange, Blink, CellAttributes, Intensity, Underline}; use color::{ColorAttribute, ColorSpec}; use escape::csi::{Cursor, Edit, EraseInDisplay, EraseInLine, Sgr, CSI}; -use escape::osc::OperatingSystemCommand; -use failure; +use escape::osc::{ITermDimension, ITermFileData, ITermProprietary, OperatingSystemCommand}; +use failure::{self, Error}; +use image::TextureCoordinate; use std::io::{Read, Write}; use surface::{Change, CursorShape, Position}; use terminal::unix::UnixTty; @@ -239,6 +240,40 @@ impl TerminfoRenderer { Ok(()) } + + fn cursor_up(&mut self, n: u32, out: &mut W) -> Result<(), Error> { + if let Some(attr) = self.get_capability::() { + attr.expand().count(n).to(out.by_ref())?; + } else { + write!(out, "{}", CSI::Cursor(Cursor::Up(n)))?; + } + Ok(()) + } + fn cursor_down(&mut self, n: u32, out: &mut W) -> Result<(), Error> { + if let Some(attr) = self.get_capability::() { + attr.expand().count(n).to(out.by_ref())?; + } else { + write!(out, "{}", CSI::Cursor(Cursor::Down(n)))?; + } + Ok(()) + } + + fn cursor_left(&mut self, n: u32, out: &mut W) -> Result<(), Error> { + if let Some(attr) = self.get_capability::() { + attr.expand().count(n).to(out.by_ref())?; + } else { + write!(out, "{}", CSI::Cursor(Cursor::Left(n)))?; + } + Ok(()) + } + fn cursor_right(&mut self, n: u32, out: &mut W) -> Result<(), Error> { + if let Some(attr) = self.get_capability::() { + attr.expand().count(n).to(out.by_ref())?; + } else { + write!(out, "{}", CSI::Cursor(Cursor::Right(n)))?; + } + Ok(()) + } } impl TerminfoRenderer { @@ -412,43 +447,31 @@ impl TerminfoRenderer { } Change::CursorPosition { x: Position::NoChange, - y: Position::Relative(1), - } => { - if let Some(attr) = self.get_capability::() { - attr.expand().to(out.by_ref())?; - } else { - write!(out, "{}", CSI::Cursor(Cursor::Down(1)))?; - } + y: Position::Relative(n), + } if *n > 0 => + { + self.cursor_down(*n as u32, out)?; } Change::CursorPosition { x: Position::NoChange, - y: Position::Relative(-1), - } => { - if let Some(attr) = self.get_capability::() { - attr.expand().to(out.by_ref())?; - } else { - write!(out, "{}", CSI::Cursor(Cursor::Up(1)))?; - } + y: Position::Relative(n), + } if *n < 0 => + { + self.cursor_up(*n as u32, out)?; } Change::CursorPosition { - x: Position::Relative(-1), + x: Position::Relative(n), y: Position::NoChange, - } => { - if let Some(attr) = self.get_capability::() { - attr.expand().to(out.by_ref())?; - } else { - write!(out, "{}", CSI::Cursor(Cursor::Left(1)))?; - } + } if *n < 0 => + { + self.cursor_left(*n as u32, out)?; } Change::CursorPosition { - x: Position::Relative(1), + x: Position::Relative(n), y: Position::NoChange, - } => { - if let Some(attr) = self.get_capability::() { - attr.expand().to(out.by_ref())?; - } else { - write!(out, "{}", CSI::Cursor(Cursor::Right(1)))?; - } + } if *n > 0 => + { + self.cursor_right(*n as u32, out)?; } Change::CursorPosition { x: Position::Absolute(x), @@ -512,6 +535,52 @@ impl TerminfoRenderer { } } }, + Change::Image(image) => { + if self.caps.iterm2_image() { + let data = if image.top_left == TextureCoordinate::new_f32(0.0, 0.0) + && image.bottom_right == TextureCoordinate::new_f32(1.0, 1.0) + { + // The whole image is requested, so we can send the + // original image bytes over + image.image.data().to_vec() + } else { + // TODO: slice out the requested region of the image, + // and encode as a PNG. + unimplemented!(); + }; + + let file = ITermFileData { + name: None, + size: Some(data.len()), + width: ITermDimension::Cells(image.width as i64), + height: ITermDimension::Cells(image.height as i64), + preserve_aspect_ratio: true, + inline: true, + data, + }; + + let osc = OperatingSystemCommand::ITermProprietary(ITermProprietary::File( + Box::new(file), + )); + + write!(out, "{}", osc)?; + + // TODO: } else if self.caps.sixel() { + } else { + // Blank out the cells and move the cursor to the right spot + for y in 0..image.height { + for _ in 0..image.width { + write!(out, " ")?; + } + + if y != image.height - 1 { + write!(out, "\n")?; + self.cursor_left(image.width as u32, out)?; + } + } + self.cursor_up(image.height as u32, out)?; + } + } } } diff --git a/termwiz/src/render/windows.rs b/termwiz/src/render/windows.rs index fcb31de24..bd8e044ad 100644 --- a/termwiz/src/render/windows.rs +++ b/termwiz/src/render/windows.rs @@ -262,6 +262,24 @@ impl WindowsConsoleRenderer { } Change::CursorColor(_color) => {} Change::CursorShape(_shape) => {} + Change::Image(image) => { + // Images are not supported, so just blank out the cells and + // move the cursor to the right spot + out.flush()?; + let info = out.get_buffer_info()?; + for y in 0..image.height { + out.fill_char( + ' ', + info.dwCursorPosition.X, + y as i16 + info.dwCursorPosition.Y, + image.width as u32, + )?; + } + out.set_cursor_position( + info.dwCursorPosition.X + image.width as i16, + info.dwCursorPosition.Y, + )?; + } } } out.flush()?; diff --git a/termwiz/src/surface/change.rs b/termwiz/src/surface/change.rs index 1f985e692..dc7c63bc4 100644 --- a/termwiz/src/surface/change.rs +++ b/termwiz/src/surface/change.rs @@ -1,5 +1,7 @@ use cell::{AttributeChange, CellAttributes}; use color::ColorAttribute; +pub use image::{ImageData, TextureCoordinate}; +use std::rc::Rc; use surface::{CursorShape, Position}; /// `Change` describes an update operation to be applied to a `Surface`. @@ -40,6 +42,14 @@ pub enum Change { /// Change the cursor shape CursorShape(CursorShape), /* ChangeScrollRegion{top: usize, bottom: usize}, */ + /// Place an image at the current cursor position. + /// The image defines the dimensions in cells. + /// TODO: check iterm rendering behavior when the image is larger than the width of the screen. + /// If the image is taller than the remaining space at the bottom + /// of the screen, the screen will scroll up. + /// The cursor Y position is unchanged by rendering the Image. + /// The cursor X position will be incremented by `Image::width` cells. + Image(Image), } impl Change { @@ -69,3 +79,29 @@ impl From for Change { Change::Attribute(c) } } + +/// The `Image` `Change` needs to support adding an image that spans multiple +/// rows and columns, as well as model the content for just one of those cells. +/// For instance, if some of the cells inside an image are replaced by textual +/// content, and the screen is scrolled, computing the diff change stream needs +/// to be able to express that a single cell holds a slice from a larger image. +/// The `Image` struct expresses its dimensions in cells and references a region +/// in the shared source image data using texture coordinates. +/// A 4x3 cell image would set `width=3`, `height=3`, `top_left=(0,0)`, `bottom_right=(1,1)`. +/// The top left cell from that image, if it were to be included in a diff, +/// would be recorded as `width=1`, `height=1`, `top_left=(0,0)`, `bottom_right=(1/4,1/3)`. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct Image { + /// measured in cells + pub width: usize, + /// measure in cells + pub height: usize, + /// Texture coordinate for the top left of this image block. + /// (0,0) is the top left of the ImageData. (1, 1) is + /// the bottom right. + pub top_left: TextureCoordinate, + /// Texture coordinates for the bottom right of this image block. + pub bottom_right: TextureCoordinate, + /// the image data + pub image: Rc, +} diff --git a/termwiz/src/surface/mod.rs b/termwiz/src/surface/mod.rs index b3e53323c..22e93bdac 100644 --- a/termwiz/src/surface/mod.rs +++ b/termwiz/src/surface/mod.rs @@ -1,5 +1,7 @@ use cell::{AttributeChange, Cell, CellAttributes}; use color::ColorAttribute; +use image::ImageCell; +use ordered_float::NotNaN; use std::borrow::Cow; use std::cmp::min; use unicode_segmentation::UnicodeSegmentation; @@ -7,7 +9,7 @@ use unicode_segmentation::UnicodeSegmentation; pub mod change; pub mod line; -pub use self::change::Change; +pub use self::change::{Change, Image, TextureCoordinate}; pub use self::line::Line; /// Position holds 0-based positioning information, where @@ -178,9 +180,55 @@ impl Surface { Change::ClearToEndOfScreen(color) => self.clear_eos(color), Change::CursorColor(color) => self.cursor_color = color.clone(), Change::CursorShape(shape) => self.cursor_shape = shape.clone(), + Change::Image(image) => self.add_image(image), } } + fn add_image(&mut self, image: &Image) { + let xsize = (image.bottom_right.x - image.top_left.x) / image.width as f32; + let ysize = (image.bottom_right.y - image.top_left.y) / image.height as f32; + + if self.ypos + image.height > self.height { + let scroll = (self.ypos + image.height) - self.height; + for _ in 0..scroll { + self.scroll_screen_up(); + } + self.ypos -= scroll; + } + + let mut ypos = NotNaN::new(0.0).unwrap(); + for y in 0..image.height { + let mut xpos = NotNaN::new(0.0).unwrap(); + for x in 0..image.width { + self.lines[self.ypos + y].set_cell( + self.xpos + x, + Cell::new( + ' ', + self.attributes + .clone() + .set_image(Some(Box::new(ImageCell::new( + TextureCoordinate::new( + image.top_left.x + xpos, + image.top_left.y + ypos, + ), + TextureCoordinate::new( + image.top_left.x + xpos + xsize, + image.top_left.y + ypos + ysize, + ), + image.image.clone(), + )))) + .clone(), + ), + ); + + xpos += xsize; + } + ypos += ysize; + } + + self.xpos += image.width; + } + fn clear_screen(&mut self, color: &ColorAttribute) { self.attributes = CellAttributes::default() .set_background(color.clone()) @@ -699,6 +747,8 @@ mod test { use super::*; use cell::Intensity; use color::AnsiColor; + use image::ImageData; + use std::rc::Rc; // The \x20's look a little awkward, but we can't use a plain // space in the first chararcter of a multi-line continuation; @@ -1356,4 +1406,136 @@ mod test { s.add_change("A\u{200b}B"); assert_eq!(s.screen_chars_to_string(), "A\u{200b}B \n"); } + + #[test] + fn images() { + // a dummy image blob with nonsense content + let data = Rc::new(ImageData::with_raw_data(0, 0, vec![])); + let mut s = Surface::new(2, 2); + s.add_change(Change::Image(Image { + top_left: TextureCoordinate::new_f32(0.0, 0.0), + bottom_right: TextureCoordinate::new_f32(1.0, 1.0), + image: data.clone(), + width: 4, + height: 2, + })); + + // We're checking that we slice the image up and assign the correct + // texture coordinates for each cell. The width and height are + // different from each other to help ensure that the right terms + // are used by add_image() function. + assert_eq!( + s.screen_cells(), + [ + [ + Cell::new( + ' ', + CellAttributes::default() + .set_image(Some(Box::new(ImageCell::new( + TextureCoordinate::new_f32(0.0, 0.0), + TextureCoordinate::new_f32(0.25, 0.5), + data.clone() + )))) + .clone() + ), + Cell::new( + ' ', + CellAttributes::default() + .set_image(Some(Box::new(ImageCell::new( + TextureCoordinate::new_f32(0.25, 0.0), + TextureCoordinate::new_f32(0.5, 0.5), + data.clone() + )))) + .clone() + ), + Cell::new( + ' ', + CellAttributes::default() + .set_image(Some(Box::new(ImageCell::new( + TextureCoordinate::new_f32(0.5, 0.0), + TextureCoordinate::new_f32(0.75, 0.5), + data.clone() + )))) + .clone() + ), + Cell::new( + ' ', + CellAttributes::default() + .set_image(Some(Box::new(ImageCell::new( + TextureCoordinate::new_f32(0.75, 0.0), + TextureCoordinate::new_f32(1.0, 0.5), + data.clone() + )))) + .clone() + ), + ], + [ + Cell::new( + ' ', + CellAttributes::default() + .set_image(Some(Box::new(ImageCell::new( + TextureCoordinate::new_f32(0.0, 0.5), + TextureCoordinate::new_f32(0.25, 1.0), + data.clone() + )))) + .clone() + ), + Cell::new( + ' ', + CellAttributes::default() + .set_image(Some(Box::new(ImageCell::new( + TextureCoordinate::new_f32(0.25, 0.5), + TextureCoordinate::new_f32(0.5, 1.0), + data.clone() + )))) + .clone() + ), + Cell::new( + ' ', + CellAttributes::default() + .set_image(Some(Box::new(ImageCell::new( + TextureCoordinate::new_f32(0.5, 0.5), + TextureCoordinate::new_f32(0.75, 1.0), + data.clone() + )))) + .clone() + ), + Cell::new( + ' ', + CellAttributes::default() + .set_image(Some(Box::new(ImageCell::new( + TextureCoordinate::new_f32(0.75, 0.5), + TextureCoordinate::new_f32(1.0, 1.0), + data.clone() + )))) + .clone() + ), + ], + ] + ); + + // Check that starting at not the texture origin coordinates + // gives reasonable values in the resultant cell + let mut other = Surface::new(1, 1); + other.add_change(Change::Image(Image { + top_left: TextureCoordinate::new_f32(0.25, 0.3), + bottom_right: TextureCoordinate::new_f32(0.75, 0.8), + image: data.clone(), + width: 1, + height: 1, + })); + assert_eq!( + other.screen_cells(), + [[Cell::new( + ' ', + CellAttributes::default() + .set_image(Some(Box::new(ImageCell::new( + TextureCoordinate::new_f32(0.25, 0.3), + TextureCoordinate::new_f32(0.75, 0.8), + data.clone() + )))) + .clone() + ),]] + ); + } }