1
1
mirror of https://github.com/wez/wezterm.git synced 2024-12-23 13:21:38 +03:00

move image decoding into termwiz

Adds a use_image feature to termwiz that enables an optional
dep on the image crate.  This in turn allows decoding of animation
formats (gif, apng) from file data, but more crucially, allows
modeling animation frames at the termwiz layer, which is a pre-req
for enabling kitty img protocol animation support.

refs: #986
This commit is contained in:
Wez Furlong 2021-08-02 16:22:55 -07:00
parent b862c8d111
commit 90b16b9518
16 changed files with 237 additions and 221 deletions

1
Cargo.lock generated
View File

@ -4325,6 +4325,7 @@ dependencies = [
"filedescriptor",
"fnv",
"hex",
"image",
"lazy_static",
"libc",
"log",

View File

@ -39,3 +39,4 @@ k9 = "0.11.0"
[dependencies.termwiz]
version = "0.13"
path = "../termwiz"
features = ["use_image"]

View File

@ -162,6 +162,7 @@ impl TerminalState {
/// cache recent images and avoid assigning a new id for repeated data!
pub(crate) fn raw_image_to_image_data(&mut self, data: ImageDataType) -> Arc<ImageData> {
let data = data.decode();
let key = data.compute_hash();
if let Some(item) = self.image_cache.get(&key) {
Arc::clone(item)

View File

@ -114,7 +114,7 @@ impl TerminalState {
(true, _) => match ::image::load_from_memory(&image.data) {
Ok(im) => {
let im = im.resize_exact(width as u32, height as u32, FilterType::CatmullRom);
let data = im.into_rgba8().into_vec().into_boxed_slice();
let data = im.into_rgba8().into_vec();
ImageDataType::Rgba8 {
width: width as u32,
height: height as u32,

View File

@ -102,7 +102,8 @@ impl TerminalState {
let decoded = ::image::load_from_memory(data).context("decode png")?;
decoded.dimensions()
}
ImageDataType::Rgba8 { width, height, .. } => (*width, *height),
ImageDataType::AnimRgba8 { width, height, .. }
| ImageDataType::Rgba8 { width, height, .. } => (*width, *height),
};
let saved_cursor = self.cursor.clone();
@ -369,9 +370,9 @@ impl TerminalState {
.ok_or_else(|| anyhow::anyhow!("failed to decode image"))?,
);
let img = img.into_rgba8();
img.into_vec().into_boxed_slice()
img.into_vec()
}
_ => data.into_boxed_slice(),
_ => data,
};
let image_data = ImageDataType::Rgba8 {
@ -385,7 +386,7 @@ impl TerminalState {
Some(KittyImageFormat::Png) => {
let decoded = image::load_from_memory(&data).context("decode png")?;
let (width, height) = decoded.dimensions();
let data = decoded.into_rgba8().into_vec().into_boxed_slice();
let data = decoded.into_rgba8().into_vec();
let image_data = ImageDataType::Rgba8 {
width,
height,

View File

@ -106,7 +106,7 @@ impl TerminalState {
let image_data = ImageDataType::Rgba8 {
width,
height,
data: image.into_vec().into_boxed_slice(),
data: image.into_vec(),
};
let image_data = self.raw_image_to_image_data(image_data);

View File

@ -19,6 +19,7 @@ anyhow = "1.0"
filedescriptor = { version="0.8", path = "../filedescriptor" }
fnv = {version="1.0", optional=true}
hex = "0.4"
image = {version="0.23", optional=true}
lazy_static = "1.4"
libc = "0.2"
log = "0.4"
@ -39,6 +40,7 @@ vtparse = { version="0.5", path="../vtparse" }
[features]
widgets = ["cassowary", "fnv"]
use_serde = ["serde"]
use_image = ["image"]
docs = ["widgets", "use_serde"]
[dev-dependencies]

View File

@ -845,7 +845,7 @@ pub struct ITermFileData {
/// the users download directory
pub inline: bool,
/// The data to transfer
pub data: Box<[u8]>,
pub data: Vec<u8>,
}
impl ITermFileData {
@ -873,7 +873,7 @@ impl ITermFileData {
let param = if idx == last {
// The final argument contains `:base64`, so look for that
if let Some(colon) = param.iter().position(|c| *c == b':') {
data = Some(base64::decode(&param[colon + 1..])?.into_boxed_slice());
data = Some(base64::decode(&param[colon + 1..])?);
&param[..colon]
} else {
// If we don't find the colon in the last piece, we've
@ -1565,7 +1565,7 @@ mod test {
height: ITermDimension::Automatic,
preserve_aspect_ratio: true,
inline: false,
data: b"hello".to_vec().into_boxed_slice(),
data: b"hello".to_vec(),
}
)))
);
@ -1583,7 +1583,7 @@ mod test {
height: ITermDimension::Automatic,
preserve_aspect_ratio: true,
inline: false,
data: b"hello".to_vec().into_boxed_slice(),
data: b"hello".to_vec(),
}
)))
);
@ -1601,7 +1601,7 @@ mod test {
height: ITermDimension::Automatic,
preserve_aspect_ratio: true,
inline: false,
data: b"hello".to_vec().into_boxed_slice(),
data: b"hello".to_vec(),
}
)))
);
@ -1619,7 +1619,7 @@ mod test {
height: ITermDimension::Automatic,
preserve_aspect_ratio: true,
inline: false,
data: b"hello".to_vec().into_boxed_slice(),
data: b"hello".to_vec(),
}
)))
);
@ -1642,7 +1642,7 @@ mod test {
height: ITermDimension::Automatic,
preserve_aspect_ratio: true,
inline: false,
data: b"hello".to_vec().into_boxed_slice(),
data: b"hello".to_vec(),
}
)))
);
@ -1660,7 +1660,7 @@ mod test {
height: ITermDimension::Automatic,
preserve_aspect_ratio: true,
inline: false,
data: b"hello".to_vec().into_boxed_slice(),
data: b"hello".to_vec(),
}
)))
);
@ -1684,7 +1684,7 @@ mod test {
height: ITermDimension::Percent(10),
preserve_aspect_ratio: true,
inline: false,
data: b"hello".to_vec().into_boxed_slice(),
data: b"hello".to_vec(),
}
)))
);
@ -1702,7 +1702,7 @@ mod test {
height: ITermDimension::Pixels(10),
preserve_aspect_ratio: false,
inline: true,
data: b"hello".to_vec().into_boxed_slice(),
data: b"hello".to_vec(),
}
)))
);

View File

@ -15,6 +15,7 @@ use ordered_float::NotNan;
#[cfg(feature = "use_serde")]
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use std::sync::Arc;
use std::time::Duration;
#[cfg(feature = "use_serde")]
fn deserialize_notnan<'de, D>(deserializer: D) -> Result<NotNan<f32>, D::Error>
@ -158,13 +159,20 @@ impl ImageCell {
pub enum ImageDataType {
/// Data is in the native image file format
/// (best for file formats that have animated content)
EncodedFile(Box<[u8]>),
EncodedFile(Vec<u8>),
/// Data is RGBA u8 data
Rgba8 {
data: Box<[u8]>,
data: Vec<u8>,
width: u32,
height: u32,
},
/// Data is an animated sequence
AnimRgba8 {
width: u32,
height: u32,
durations: Vec<Duration>,
frames: Vec<Vec<u8>>,
},
}
impl std::fmt::Debug for ImageDataType {
@ -184,6 +192,18 @@ impl std::fmt::Debug for ImageDataType {
.field("width", &width)
.field("height", &height)
.finish(),
Self::AnimRgba8 {
frames,
width,
height,
durations,
} => fmt
.debug_struct("AnimRgba8")
.field("frames_of_len", &frames.len())
.field("width", &width)
.field("height", &height)
.field("durations", durations)
.finish(),
}
}
}
@ -195,9 +215,98 @@ impl ImageDataType {
match self {
ImageDataType::EncodedFile(data) => hasher.update(data),
ImageDataType::Rgba8 { data, .. } => hasher.update(data),
ImageDataType::AnimRgba8 { frames, .. } => {
for data in frames {
hasher.update(data);
}
}
};
hasher.finalize().into()
}
/// Decode an encoded file into either an Rgba8 or AnimRgba8 variant
/// if we recognize the file format, otherwise the EncodedFile data
/// is preserved as is.
#[cfg(feature = "use_image")]
pub fn decode(self) -> Self {
use image::{AnimationDecoder, ImageFormat};
match self {
Self::EncodedFile(data) => {
let format = match image::guess_format(&data) {
Ok(format) => format,
_ => return Self::EncodedFile(data),
};
match format {
ImageFormat::Gif => image::gif::GifDecoder::new(&*data)
.and_then(|decoder| decoder.into_frames().collect_frames())
.and_then(|frames| Ok(Self::decode_frames(frames)))
.unwrap_or_else(|err| {
log::error!(
"Unable to parse animated gif: {:#}, trying as single frame",
err
);
Self::decode_single(data)
}),
ImageFormat::Png => {
let decoder = match image::png::PngDecoder::new(&*data) {
Ok(d) => d,
_ => return Self::EncodedFile(data),
};
if decoder.is_apng() {
match decoder.apng().into_frames().collect_frames() {
Ok(frames) => Self::decode_frames(frames),
_ => Self::EncodedFile(data),
}
} else {
Self::decode_single(data)
}
}
_ => Self::EncodedFile(data),
}
}
data => data,
}
}
#[cfg(feature = "use_image")]
fn decode_frames(img_frames: Vec<image::Frame>) -> Self {
let mut width = 0;
let mut height = 0;
let mut frames = vec![];
let mut durations = vec![];
for frame in img_frames.into_iter() {
let duration: Duration = frame.delay().into();
durations.push(duration);
let image = image::DynamicImage::ImageRgba8(frame.into_buffer()).to_rgba8();
let (w, h) = image.dimensions();
width = w;
height = h;
frames.push(image.into_vec());
}
Self::AnimRgba8 {
width,
height,
frames,
durations,
}
}
#[cfg(feature = "use_image")]
fn decode_single(data: Vec<u8>) -> Self {
match image::load_from_memory(&data) {
Ok(image) => {
let image = image.to_rgba8();
let (width, height) = image.dimensions();
Self::Rgba8 {
width,
height,
data: image.into_vec(),
}
}
_ => Self::EncodedFile(data),
}
}
}
static IMAGE_ID: ::std::sync::atomic::AtomicUsize = ::std::sync::atomic::AtomicUsize::new(0);
@ -212,7 +321,7 @@ pub struct ImageData {
impl ImageData {
/// Create a new ImageData struct with the provided raw data.
pub fn with_raw_data(data: Box<[u8]>) -> Self {
pub fn with_raw_data(data: Vec<u8>) -> Self {
Self::with_data(ImageDataType::EncodedFile(data))
}
@ -222,10 +331,12 @@ impl ImageData {
Self { id, hash, data }
}
/// Returns the in-memory footprint
pub fn len(&self) -> usize {
match &self.data {
ImageDataType::EncodedFile(d) => d.len(),
ImageDataType::Rgba8 { data, .. } => data.len(),
ImageDataType::AnimRgba8 { frames, .. } => frames.len() * frames[0].len(),
}
}

View File

@ -568,10 +568,10 @@ impl TerminfoRenderer {
// The whole image is requested, so we can send the
// original image bytes over
match image.image.data() {
ImageDataType::EncodedFile(data) => {
data.to_vec().into_boxed_slice()
ImageDataType::EncodedFile(data) => data.to_vec(),
ImageDataType::AnimRgba8 { .. } | ImageDataType::Rgba8 { .. } => {
unimplemented!()
}
ImageDataType::Rgba8 { .. } => unimplemented!(),
}
} else {
// TODO: slice out the requested region of the image,

View File

@ -1575,7 +1575,7 @@ mod test {
#[test]
fn images() {
// a dummy image blob with nonsense content
let data = Arc::new(ImageData::with_raw_data(vec![].into_boxed_slice()));
let data = Arc::new(ImageData::with_raw_data(vec![]));
let mut s = Surface::new(2, 2);
s.add_change(Change::Image(Image {
top_left: TextureCoordinate::new_f32(0.0, 0.0),

View File

@ -18,7 +18,7 @@ use std::collections::HashMap;
use std::convert::TryInto;
use std::rc::Rc;
use std::sync::Arc;
use std::time::{Duration, Instant};
use std::time::Instant;
use termwiz::image::{ImageData, ImageDataType};
use wezterm_font::units::*;
use wezterm_font::{FontConfiguration, GlyphInfo};
@ -131,149 +131,61 @@ struct LineKey {
overline: bool,
}
#[derive(Debug)]
pub enum ImageFrameBitmap {
Owned(::window::bitmaps::Image),
Ref(Arc<ImageData>),
}
impl BitmapImage for ImageFrameBitmap {
impl BitmapImage for DecodedImage {
unsafe fn pixel_data(&self) -> *const u8 {
match self {
Self::Owned(im) => im.pixel_data(),
Self::Ref(im) => match im.data() {
ImageDataType::EncodedFile(_) => unreachable!(),
ImageDataType::Rgba8 { data, .. } => data.as_ptr(),
},
match self.image.data() {
ImageDataType::Rgba8 { data, .. } => data.as_ptr(),
ImageDataType::AnimRgba8 { frames, .. } => frames[self.current_frame].as_ptr(),
ImageDataType::EncodedFile(_) => unreachable!(),
}
}
unsafe fn pixel_data_mut(&mut self) -> *mut u8 {
panic!("cannot mutate ImageFrameBitmap");
panic!("cannot mutate DecodedImage");
}
fn image_dimensions(&self) -> (usize, usize) {
match self {
Self::Owned(im) => im.image_dimensions(),
Self::Ref(im) => match im.data() {
ImageDataType::EncodedFile(_) => unreachable!(),
ImageDataType::Rgba8 { width, height, .. } => (*width as usize, *height as usize),
},
match self.image.data() {
ImageDataType::Rgba8 { width, height, .. }
| ImageDataType::AnimRgba8 { width, height, .. } => (*width as usize, *height as usize),
ImageDataType::EncodedFile(_) => unreachable!(),
}
}
}
#[derive(Debug)]
pub struct ImageFrame {
duration: Duration,
image: ImageFrameBitmap,
}
#[derive(Debug)]
pub enum CachedImage {
Animation(DecodedImage),
SingleFrame(ImageFrameBitmap),
}
#[derive(Debug)]
pub struct DecodedImage {
frame_start: Instant,
current_frame: usize,
frames: Vec<ImageFrame>,
image: Arc<ImageData>,
}
impl DecodedImage {
fn placeholder() -> Self {
let image = ::window::bitmaps::Image::new(1, 1);
let frame = ImageFrame {
duration: Duration::default(),
image: ImageFrameBitmap::Owned(image),
};
let image = ImageData::with_data(ImageDataType::Rgba8 {
// A single black pixel
data: vec![0, 0, 0, 0],
width: 1,
height: 1,
});
Self {
frame_start: Instant::now(),
current_frame: 0,
frames: vec![frame],
image: Arc::new(image),
}
}
fn with_frames(frames: Vec<image::Frame>) -> Self {
let frames = frames
.into_iter()
.map(|frame| {
let duration: Duration = frame.delay().into();
let image = image::DynamicImage::ImageRgba8(frame.into_buffer()).to_rgba8();
let (w, h) = image.dimensions();
let width = w as usize;
let height = h as usize;
let image = ImageFrameBitmap::Owned(::window::bitmaps::Image::from_raw(
width,
height,
image.into_vec(),
));
ImageFrame { duration, image }
})
.collect();
Self {
frame_start: Instant::now(),
current_frame: 0,
frames,
}
}
fn with_single(image_data: &Arc<ImageData>) -> anyhow::Result<Self> {
let image = match image_data.data() {
ImageDataType::EncodedFile(data) => {
let image = image::load_from_memory(data)?.to_rgba8();
let (width, height) = image.dimensions();
let width = width as usize;
let height = height as usize;
ImageFrameBitmap::Owned(::window::bitmaps::Image::from_raw(
width,
height,
image.into_vec(),
))
}
ImageDataType::Rgba8 { .. } => ImageFrameBitmap::Ref(Arc::clone(image_data)),
};
Ok(Self {
frame_start: Instant::now(),
current_frame: 0,
frames: vec![ImageFrame {
duration: Default::default(),
image,
}],
})
}
fn load(image_data: &Arc<ImageData>) -> anyhow::Result<Self> {
fn load(image_data: &Arc<ImageData>) -> Self {
match image_data.data() {
ImageDataType::Rgba8 { .. } => Self::with_single(image_data),
ImageDataType::EncodedFile(data) => {
use image::{AnimationDecoder, ImageFormat};
let format = image::guess_format(data)?;
match format {
ImageFormat::Gif => image::gif::GifDecoder::new(&**data)
.and_then(|decoder| decoder.into_frames().collect_frames())
.and_then(|frames| Ok(Self::with_frames(frames)))
.or_else(|err| {
log::error!(
"Unable to parse animated gif: {:#}, trying as single frame",
err
);
Self::with_single(image_data)
}),
ImageFormat::Png => {
let decoder = image::png::PngDecoder::new(&**data)?;
if decoder.is_apng() {
let frames = decoder.apng().into_frames().collect_frames()?;
Ok(Self::with_frames(frames))
} else {
Self::with_single(image_data)
}
}
_ => Self::with_single(image_data),
}
ImageDataType::EncodedFile(_) => {
log::warn!("Unexpected ImageDataType::EncodedFile; either file is unreadable or we missed a .decode call somewhere");
Self::placeholder()
}
_ => Self {
frame_start: Instant::now(),
current_frame: 0,
image: Arc::clone(image_data),
},
}
}
}
@ -282,7 +194,7 @@ pub struct GlyphCache<T: Texture2d> {
glyph_cache: HashMap<GlyphKey, Rc<CachedGlyph<T>>>,
pub atlas: Atlas<T>,
fonts: Rc<FontConfiguration>,
pub image_cache: LruCache<usize, CachedImage>,
pub image_cache: LruCache<usize, DecodedImage>,
frame_cache: HashMap<(usize, usize), Sprite<T>>,
line_glyphs: HashMap<LineKey, Sprite<T>>,
pub block_glyphs: HashMap<BlockKey, Sprite<T>>,
@ -604,6 +516,59 @@ impl<T: Texture2d> GlyphCache<T> {
Ok(Rc::new(glyph))
}
fn cached_image_impl(
frame_cache: &mut HashMap<(usize, usize), Sprite<T>>,
atlas: &mut Atlas<T>,
decoded: &mut DecodedImage,
padding: Option<usize>,
) -> anyhow::Result<(Sprite<T>, Option<Instant>)> {
let id = decoded.image.id();
match decoded.image.data() {
ImageDataType::Rgba8 { .. } => {
if let Some(sprite) = frame_cache.get(&(id, 0)) {
return Ok((sprite.clone(), None));
}
let sprite = atlas.allocate_with_padding(decoded, padding)?;
frame_cache.insert((id, 0), sprite.clone());
return Ok((sprite, None));
}
ImageDataType::AnimRgba8 {
frames, durations, ..
} => {
let mut next = None;
if frames.len() > 1 {
let now = Instant::now();
let mut next_due = decoded.frame_start + durations[decoded.current_frame];
if now >= next_due {
// Advance to next frame
decoded.current_frame += 1;
if decoded.current_frame >= frames.len() {
decoded.current_frame = 0;
}
decoded.frame_start = now;
next_due = decoded.frame_start + durations[decoded.current_frame];
}
next.replace(next_due);
}
if let Some(sprite) = frame_cache.get(&(id, decoded.current_frame)) {
return Ok((sprite.clone(), next));
}
let sprite = atlas.allocate_with_padding(decoded, padding)?;
frame_cache.insert((id, decoded.current_frame), sprite.clone());
return Ok((
sprite,
Some(decoded.frame_start + durations[decoded.current_frame]),
));
}
ImageDataType::EncodedFile(_) => unreachable!(),
}
}
pub fn cached_image(
&mut self,
@ -611,81 +576,19 @@ impl<T: Texture2d> GlyphCache<T> {
padding: Option<usize>,
) -> anyhow::Result<(Sprite<T>, Option<Instant>)> {
let id = image_data.id();
if let Some(cached) = self.image_cache.get_mut(&id) {
match cached {
CachedImage::SingleFrame(im) => {
// We can simply use the frame cache to manage
// the texture space; the frame is always 0 for
// a single frame
if let Some(sprite) = self.frame_cache.get(&(id, 0)) {
return Ok((sprite.clone(), None));
}
let sprite = self.atlas.allocate_with_padding(im, padding)?;
self.frame_cache.insert((id, 0), sprite.clone());
return Ok((sprite, None));
}
CachedImage::Animation(decoded) => {
let mut next = None;
if decoded.frames.len() > 1 {
let now = Instant::now();
let mut next_due =
decoded.frame_start + decoded.frames[decoded.current_frame].duration;
if now >= next_due {
// Advance to next frame
decoded.current_frame += 1;
if decoded.current_frame >= decoded.frames.len() {
decoded.current_frame = 0;
}
decoded.frame_start = now;
next_due = decoded.frame_start
+ decoded.frames[decoded.current_frame].duration;
}
next.replace(next_due);
}
if let Some(sprite) = self.frame_cache.get(&(id, decoded.current_frame)) {
return Ok((sprite.clone(), next));
}
let sprite = self.atlas.allocate_with_padding(
&decoded.frames[decoded.current_frame].image,
padding,
)?;
self.frame_cache
.insert((id, decoded.current_frame), sprite.clone());
return Ok((
sprite,
Some(decoded.frame_start + decoded.frames[decoded.current_frame].duration),
));
}
}
}
let start = Instant::now();
let mut decoded =
DecodedImage::load(image_data).or_else(|e| -> anyhow::Result<DecodedImage> {
log::debug!("Failed to decode image: {:#}", e);
// Use a placeholder instead
Ok(DecodedImage::placeholder())
})?;
metrics::histogram!("glyphcache.cached_image.decode.rate", 1.);
metrics::histogram!("glyphcache.cached_image.decode.latency", start.elapsed());
let sprite = self
.atlas
.allocate_with_padding(&decoded.frames[0].image, padding)?;
self.frame_cache.insert((id, 0), sprite.clone());
if decoded.frames.len() > 1 {
let next = Some(decoded.frame_start + decoded.frames[0].duration);
self.image_cache.put(id, CachedImage::Animation(decoded));
Ok((sprite, next))
if let Some(decoded) = self.image_cache.get_mut(&id) {
Self::cached_image_impl(&mut self.frame_cache, &mut self.atlas, decoded, padding)
} else {
self.image_cache
.put(id, CachedImage::SingleFrame(decoded.frames.remove(0).image));
Ok((sprite, None))
let mut decoded = DecodedImage::load(image_data);
let res = Self::cached_image_impl(
&mut self.frame_cache,
&mut self.atlas,
&mut decoded,
padding,
)?;
self.image_cache.put(id, decoded);
Ok(res)
}
}

View File

@ -170,9 +170,7 @@ impl RenderState {
let mut data = vec![];
if let Ok(_len) = f.read_to_end(&mut data) {
if let Ok(decoded_image) = image::load_from_memory(&data) {
let image = Arc::new(termwiz::image::ImageData::with_raw_data(
data.into_boxed_slice(),
));
let image = Arc::new(termwiz::image::ImageData::with_raw_data(data));
let scale = self.wrap_width as f32 / decoded_image.width() as f32;

View File

@ -337,7 +337,7 @@ fn load_background_image(config: &ConfigHandle) -> Option<Arc<ImageData>> {
Some(p) => match std::fs::read(p) {
Ok(data) => {
log::info!("loaded {}", p.display());
Some(Arc::new(ImageData::with_raw_data(data.into_boxed_slice())))
Some(Arc::new(ImageData::with_raw_data(data)))
}
Err(err) => {
log::error!(
@ -367,7 +367,7 @@ fn reload_background_image(
_ => {}
}
}
Some(Arc::new(ImageData::with_raw_data(data.into_boxed_slice())))
Some(Arc::new(ImageData::with_raw_data(data)))
}
Err(err) => {
log::error!(

View File

@ -293,7 +293,7 @@ fn set_banner_from_release_info(latest: &Release) {
height: ITermDimension::Cells(2),
preserve_aspect_ratio: true,
inline: true,
data: ICON_DATA.to_vec().into_boxed_slice(),
data: ICON_DATA.to_vec(),
};
let icon = OperatingSystemCommand::ITermProprietary(ITermProprietary::File(Box::new(icon)));
let top_line_pos = CSI::Cursor(Cursor::CharacterAndLinePosition {

View File

@ -210,8 +210,6 @@ impl ImgCatCommand {
stdin.read_to_end(&mut data)?;
}
let data = data.into_boxed_slice();
let osc = OperatingSystemCommand::ITermProprietary(ITermProprietary::File(Box::new(
ITermFileData {
name: None,