Image decoding is now done in a bg thread

Continuing from the previous commit, this shifts:

* In-memory data -> temporary file
* Image decoding -> background thread

The background thread asynchronously decodes frames and
sends them to the render thread via a bounded channel.

While decoding frames, it writes them, uncompressed, to
a scratch file so that when the animation loops, it is
a very cheap operation to rewind and pull that data
from the file, without having to burn CPU to re-decode
the data from the start.

Memory usage is bounded to 4 uncompressed frames while
decoding, then 3 uncompressed frames (triple buffered)
while looping over the rest.

However, disk usage is N uncompressed frames.

refs: https://github.com/wez/wezterm/issues/3263

Cargo.lock

@@ -2,12 +2,6 @@
 # It is not intended for manual editing.
 version = 3
 
-[[package]]
-name = "Inflector"
-version = "0.11.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3"
-
 [[package]]
 name = "addr2line"
 version = "0.19.0"
@@ -61,12 +55,6 @@ dependencies = [
  "memchr",
 ]
 
-[[package]]
-name = "aliasable"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd"
-
 [[package]]
 name = "android_system_properties"
 version = "0.1.5"
@@ -1444,7 +1432,7 @@ dependencies = [
 
 [[package]]
 name = "filedescriptor"
-version = "0.8.2"
+version = "0.8.3"
 dependencies = [
  "libc",
  "thiserror",
@@ -3416,29 +3404,6 @@ version = "6.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee"
 
-[[package]]
-name = "ouroboros"
-version = "0.15.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e1358bd1558bd2a083fed428ffeda486fbfb323e698cdda7794259d592ca72db"
-dependencies = [
- "aliasable",
- "ouroboros_macro",
-]
-
-[[package]]
-name = "ouroboros_macro"
-version = "0.15.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f7d21ccd03305a674437ee1248f3ab5d4b1db095cf1caf49f1713ddf61956b7"
-dependencies = [
- "Inflector",
- "proc-macro-error",
- "proc-macro2",
- "quote",
- "syn",
-]
-
 [[package]]
 name = "parking"
 version = "2.0.0"
@@ -5006,7 +4971,7 @@ checksum = "95059e91184749cb66be6dc994f67f182b6d897cb3df74a5bf66b5e709295fd8"
 
 [[package]]
 name = "termwiz"
-version = "0.20.0"
+version = "0.21.0"
 dependencies = [
  "anyhow",
  "base64 0.21.0",
@@ -5038,6 +5003,7 @@ dependencies = [
  "sha2 0.9.9",
  "signal-hook 0.1.17",
  "siphasher",
+ "tempfile",
  "terminfo",
  "termios 0.3.3",
  "thiserror",
@@ -5958,7 +5924,6 @@ dependencies = [
  "mux-lua",
  "open",
  "ordered-float",
- "ouroboros",
  "parking_lot 0.12.1",
  "portable-pty",
  "promise",
@@ -5973,6 +5938,7 @@ dependencies = [
  "shlex",
  "smol",
  "tabout",
+ "tempfile",
  "terminfo",
  "termwiz",
  "termwiz-funcs",

docs/changelog.md

@@ -140,6 +140,11 @@ As features stabilize some brief notes about them will accumulate here.
   [#928](https://github.com/wez/wezterm/issues/928)
 * Incorrect cursor position after processing iTerm2 image escape sequence
   [#3266](https://github.com/wez/wezterm/issues/3266)
+* Images are now buffered to temporary files and decoded in background
+  threads. This reduces the RAM overhead especially of long animations and
+  reduces the render latency due to decoding frames; animations now render as
+  soon as the first frame is decoded.
+  [#3263](https://github.com/wez/wezterm/issues/3263)
 
 #### Changed
 * `CTRL-SHIFT-P` now activates the new [command

filedescriptor/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "filedescriptor"
-version = "0.8.2"
+version = "0.8.3"
 authors = ["Wez Furlong"]
 edition = "2018"
 repository = "https://github.com/wez/wezterm"

filedescriptor/src/lib.rs

@@ -304,6 +304,13 @@ impl FileDescriptor {
         self.as_stdio_impl()
     }
 
+    /// A convenience method for creating a `std::fs::File` object.
+    /// The `File` is created using a duplicated handle so
+    /// that the source handle remains alive.
+    pub fn as_file(&self) -> Result<std::fs::File> {
+        self.as_file_impl()
+    }
+
     /// Attempt to change the non-blocking IO mode of the file descriptor.
     /// Not all kinds of file descriptor can be placed in non-blocking mode
     /// on all systems, and some file descriptors will claim to be in

filedescriptor/src/unix.rs

@@ -253,6 +253,14 @@ impl FileDescriptor {
         Ok(stdio)
     }
 
+    #[inline]
+    pub(crate) fn as_file_impl(&self) -> Result<std::fs::File> {
+        let duped = OwnedHandle::dup(self)?;
+        let fd = duped.into_raw_fd();
+        let stdio = unsafe { std::fs::File::from_raw_fd(fd) };
+        Ok(stdio)
+    }
+
     #[inline]
     pub(crate) fn set_non_blocking_impl(&mut self, non_blocking: bool) -> Result<()> {
         let on = if non_blocking { 1 } else { 0 };

filedescriptor/src/windows.rs

@@ -233,6 +233,14 @@ impl FileDescriptor {
         Ok(stdio)
     }
 
+    #[inline]
+    pub(crate) fn as_file_impl(&self) -> Result<std::fs::File> {
+        let duped = self.handle.try_clone()?;
+        let handle = duped.into_raw_handle();
+        let stdio = unsafe { std::fs::File::from_raw_handle(handle) };
+        Ok(stdio)
+    }
+
     #[inline]
     pub(crate) fn set_non_blocking_impl(&mut self, non_blocking: bool) -> Result<()> {
         if !self.handle.is_socket_handle() {

tabout/Cargo.toml

@@ -9,4 +9,4 @@ license = "MIT"
 documentation = "https://docs.rs/tabout"
 
 [dependencies]
-termwiz = { path = "../termwiz", version="0.20"}
+termwiz = { path = "../termwiz", version="0.21"}

term/Cargo.toml

@@ -40,6 +40,6 @@ env_logger = "0.10"
 k9 = "0.11.0"
 
 [dependencies.termwiz]
-version = "0.20"
+version = "0.21"
 path = "../termwiz"
 features = ["use_image"]

term/src/terminalstate/image.rs

@@ -227,14 +227,18 @@ impl TerminalState {
     }
 
     /// cache recent images and avoid assigning a new id for repeated data!
-    pub(crate) fn raw_image_to_image_data(&mut self, data: ImageDataType) -> Arc<ImageData> {
+    pub(crate) fn raw_image_to_image_data(
+        &mut self,
+        data: ImageDataType,
+    ) -> std::io::Result<Arc<ImageData>> {
         let key = data.compute_hash();
         if let Some(item) = self.image_cache.get(&key) {
-            Arc::clone(item)
+            Ok(Arc::clone(item))
         } else {
+            let data = data.swap_out()?;
             let image_data = Arc::new(ImageData::with_data(data));
             self.image_cache.put(key, Arc::clone(&image_data));
-            image_data
+            Ok(image_data)
         }
     }
 }

term/src/terminalstate/iterm.rs

@@ -113,7 +113,14 @@ impl TerminalState {
             },
         };
 
-        let image_data = self.raw_image_to_image_data(data);
+        let image_data = match self.raw_image_to_image_data(data) {
+            Ok(d) => d,
+            Err(err) => {
+                log::error!("error processing image data: {err:#}");
+                return;
+            }
+        };
+
         if let Err(err) = self.assign_image_to_cells(ImageAttachParams {
             image_width: width as u32,
             image_height: height as u32,

term/src/terminalstate/kitty.rs

@@ -107,14 +107,7 @@ impl TerminalState {
             )
         })?);
 
-        let (image_width, image_height) = match &*img.data() {
-            ImageDataType::EncodedFile(data) => {
-                let decoded = ::image::load_from_memory(data).context("decode png")?;
-                decoded.dimensions()
-            }
-            ImageDataType::AnimRgba8 { width, height, .. }
-            | ImageDataType::Rgba8 { width, height, .. } => (*width, *height),
-        };
+        let (image_width, image_height) = img.data().dimensions()?;
 
         let info = self.assign_image_to_cells(ImageAttachParams {
             image_width,
@@ -450,7 +443,9 @@ impl TerminalState {
 
         let mut img = img.data();
         match &mut *img {
-            ImageDataType::EncodedFile(_) => anyhow::bail!("invalid image type"),
+            ImageDataType::File(_) | ImageDataType::EncodedFile(_) => {
+                anyhow::bail!("invalid image type")
+            }
             ImageDataType::Rgba8 {
                 width,
                 height,
@@ -603,7 +598,7 @@ impl TerminalState {
         });
 
         match &mut *anim {
-            ImageDataType::EncodedFile(_) => {
+            ImageDataType::File(_) | ImageDataType::EncodedFile(_) => {
                 anyhow::bail!("Expected decoded image for image id {}", image_id)
             }
             ImageDataType::Rgba8 {
@@ -831,7 +826,9 @@ impl TerminalState {
         let (image_id, image_number, img) = self.kitty_img_transmit_inner(transmit)?;
         self.kitty_img.max_image_id = self.kitty_img.max_image_id.max(image_id);
 
-        let img = self.raw_image_to_image_data(img);
+        let img = self
+            .raw_image_to_image_data(img)
+            .context("storing image data")?;
         self.kitty_img.record_id_to_data(image_id, img);
 
         if image_number.is_some() {

term/src/terminalstate/sixel.rs

@@ -114,7 +114,13 @@ impl TerminalState {
         let data = image.into_vec();
         let image_data = ImageDataType::new_single_frame(width, height, data);
 
-        let image_data = self.raw_image_to_image_data(image_data);
+        let image_data = match self.raw_image_to_image_data(image_data) {
+            Ok(d) => d,
+            Err(err) => {
+                log::error!("error while processing sixel image: {err:#}");
+                return;
+            }
+        };
         let old_cursor = self.cursor;
         if self.sixel_display_mode {
             // Sixel Display Mode (DECSDM) requires placing the image

termwiz/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 authors = ["Wez Furlong"]
 name = "termwiz"
-version = "0.20.0"
+version = "0.21.0"
 edition = "2018"
 repository = "https://github.com/wez/wezterm"
 description = "Terminal Wizardry for Unix and Windows"
@@ -37,6 +37,7 @@ semver = "0.11"
 serde = {version="1.0", features = ["rc", "derive"], optional=true}
 siphasher = "0.3"
 sha2 = "0.9"
+tempfile = "3.4"
 terminfo = "0.7"
 thiserror = "1.0"
 unicode-segmentation = "1.8"

termwiz/src/image.rs

@@ -14,7 +14,9 @@
 use ordered_float::NotNan;
 #[cfg(feature = "use_serde")]
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
+use std::fs::File;
 use std::hash::{Hash, Hasher};
+use std::io::{Read, Seek, SeekFrom, Write};
 use std::sync::{Arc, Mutex, MutexGuard};
 use std::time::Duration;
 
@@ -192,12 +194,121 @@ impl ImageCell {
     }
 }
 
+#[derive(Clone)]
+pub struct ImageDataFile {
+    file: Arc<Mutex<File>>,
+    hash: [u8; 32],
+    len: usize,
+}
+
+impl ImageDataFile {
+    /// Loads a copy of the data from disk, and returns it
+    /// as a vec
+    pub fn data(&self) -> std::io::Result<Vec<u8>> {
+        self.with_rewound_file(|file| {
+            let mut data = vec![];
+            file.read_to_end(&mut data)?;
+            Ok(data)
+        })
+    }
+
+    pub fn len(&self) -> usize {
+        self.len
+    }
+
+    #[cfg(feature = "use_image")]
+    pub fn dimensions(&self) -> image::ImageResult<(u32, u32)> {
+        self.with_rewound_file(|file| {
+            let bufreader = std::io::BufReader::new(file);
+            let reader = image::io::Reader::new(bufreader).with_guessed_format()?;
+            Ok(reader.into_dimensions())
+        })?
+    }
+
+    pub fn with_rewound_file<R, F: FnOnce(&mut File) -> std::io::Result<R>>(
+        &self,
+        func: F,
+    ) -> std::io::Result<R> {
+        let mut file = self.file.lock().unwrap();
+        let position = file.stream_position()?;
+        file.rewind()?;
+        let result = (func)(&mut *file);
+        file.seek(SeekFrom::Start(position))?;
+        result
+    }
+
+    pub fn with_data(data: &[u8]) -> std::io::Result<Self> {
+        let mut file = tempfile::tempfile()?;
+
+        file.write_all(&data)?;
+
+        let hash = ImageDataType::hash_bytes(&data);
+        let len = data.len();
+
+        Ok(Self {
+            hash,
+            file: Arc::new(Mutex::new(file)),
+            len,
+        })
+    }
+
+    pub fn duplicate_file(&self) -> Result<File, filedescriptor::Error> {
+        let file = self.file.lock().unwrap();
+        let fd = filedescriptor::FileDescriptor::dup(&*file)?;
+        let mut file = fd.as_file()?;
+        file.rewind()?;
+        Ok(file)
+    }
+}
+
+#[cfg(feature = "use_serde")]
+impl Serialize for ImageDataFile {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let data = self.data().map_err(|err| serde::ser::Error::custom(err))?;
+        data.serialize(serializer)
+    }
+}
+
+#[cfg(feature = "use_serde")]
+impl<'de> Deserialize<'de> for ImageDataFile {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let data = <Vec<u8> as Deserialize>::deserialize(deserializer)?;
+        Self::with_data(&data).map_err(|err| serde::de::Error::custom(err))
+    }
+}
+
+impl Eq for ImageDataFile {}
+impl PartialEq for ImageDataFile {
+    fn eq(&self, rhs: &Self) -> bool {
+        self.hash == rhs.hash
+    }
+}
+impl std::fmt::Debug for ImageDataFile {
+    fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
+        fmt.debug_struct("ImageDataFile")
+            .field("hash", &self.hash)
+            .finish()
+    }
+}
+
 #[cfg_attr(feature = "use_serde", derive(Serialize, Deserialize))]
 #[derive(Clone, PartialEq, Eq)]
 pub enum ImageDataType {
     /// Data is in the native image file format
     /// (best for file formats that have animated content)
     EncodedFile(Vec<u8>),
+    /// Data is in the native image file format,
+    /// (best for file formats that have animated content)
+    /// and is stored in a file, rather than in memory.
+    /// The file will be cleaned up by the OS automatically
+    /// when this process terminates.
+    File(ImageDataFile),
     /// Data is RGBA u8 data
     Rgba8 {
         data: Vec<u8>,
@@ -222,6 +333,7 @@ impl std::fmt::Debug for ImageDataType {
                 .debug_struct("EncodedFile")
                 .field("data_of_len", &data.len())
                 .finish(),
+            Self::File(file) => file.fmt(fmt),
             Self::Rgba8 {
                 data,
                 width,
@@ -283,6 +395,7 @@ impl ImageDataType {
         let mut hasher = sha2::Sha256::new();
         match self {
             ImageDataType::EncodedFile(data) => hasher.update(data),
+            ImageDataType::File(f) => return f.hash,
             ImageDataType::Rgba8 { data, .. } => hasher.update(data),
             ImageDataType::AnimRgba8 {
                 frames, durations, ..
@@ -316,6 +429,31 @@ impl ImageDataType {
         }
     }
 
+    #[cfg(feature = "use_image")]
+    pub fn dimensions(&self) -> image::ImageResult<(u32, u32)> {
+        match self {
+            ImageDataType::EncodedFile(data) => {
+                let reader =
+                    image::io::Reader::new(std::io::Cursor::new(data)).with_guessed_format()?;
+                let (width, height) = reader.into_dimensions()?;
+
+                Ok((width, height))
+            }
+            ImageDataType::File(file) => file.dimensions(),
+            ImageDataType::AnimRgba8 { width, height, .. }
+            | ImageDataType::Rgba8 { width, height, .. } => Ok((*width, *height)),
+        }
+    }
+
+    /// Migrate an in-memory encoded image blob to on-disk to reduce
+    /// the memory footprint
+    pub fn swap_out(self) -> std::io::Result<Self> {
+        match self {
+            Self::EncodedFile(data) => Ok(Self::File(ImageDataFile::with_data(&data)?)),
+            other => Ok(other),
+        }
+    }
+
     /// Decode an encoded file into either an Rgba8 or AnimRgba8 variant
     /// if we recognize the file format, otherwise the EncodedFile data
     /// is preserved as is.
@@ -473,6 +611,7 @@ impl ImageData {
     pub fn len(&self) -> usize {
         match &*self.data() {
             ImageDataType::EncodedFile(d) => d.len(),
+            ImageDataType::File(f) => f.len(),
             ImageDataType::Rgba8 { data, .. } => data.len(),
             ImageDataType::AnimRgba8 { frames, .. } => frames.len() * frames[0].len(),
         }

termwiz/src/render/terminfo.rs

@@ -579,6 +579,7 @@ impl TerminfoRenderer {
                             // original image bytes over
                             match &*image.image.data() {
                                 ImageDataType::EncodedFile(data) => data.to_vec(),
+                                ImageDataType::File(file) => file.data()?,
                                 ImageDataType::AnimRgba8 { .. } | ImageDataType::Rgba8 { .. } => {
                                     unimplemented!()
                                 }

wezterm-gui/Cargo.toml

@@ -68,7 +68,6 @@ mux = { path = "../mux" }
 mux-lua = { path = "../lua-api-crates/mux" }
 open = "4.0"
 ordered-float = "3.0"
-ouroboros = "0.15"
 parking_lot = "0.12"
 portable-pty = { path = "../pty", features = ["serde_support"]}
 promise = { path = "../promise" }
@@ -82,6 +81,7 @@ serial = "0.4"
 shlex = "1.1"
 smol = "1.2"
 tabout = { path = "../tabout" }
+tempfile = "3.4"
 terminfo = "0.7"
 termwiz = { path = "../termwiz" }
 termwiz-funcs = { path = "../lua-api-crates/termwiz-funcs" }

wezterm-gui/src/glyphcache.rs

@@ -5,20 +5,24 @@ use ::window::bitmaps::atlas::{Atlas, OutOfTextureSpace, Sprite};
 use ::window::bitmaps::{BitmapImage, Image, ImageTexture, Texture2d};
 use ::window::color::SrgbaPixel;
 use ::window::{Point, Rect};
+use anyhow::Context;
 use config::{AllowSquareGlyphOverflow, TextStyle};
 use euclid::num::Zero;
 use image::io::Limits;
 use image::{AnimationDecoder, Frame, Frames, ImageDecoder, ImageFormat, ImageResult};
 use lfucache::LfuCacheU64;
 use ordered_float::NotNan;
-use ouroboros::self_referencing;
+use parking_lot::Mutex;
 use std::cell::RefCell;
 use std::collections::HashMap;
+use std::fs::File;
+use std::io::{BufReader, Read, Seek, Write};
 use std::rc::Rc;
+use std::sync::mpsc::{sync_channel, Receiver, SyncSender, TryRecvError};
 use std::sync::{Arc, MutexGuard};
 use std::time::{Duration, Instant};
 use termwiz::color::RgbColor;
-use termwiz::image::{ImageData, ImageDataType};
+use termwiz::image::{ImageData, ImageDataFile, ImageDataType};
 use termwiz::surface::CursorShape;
 use wezterm_font::units::*;
 use wezterm_font::{FontConfiguration, GlyphInfo, LoadedFont, LoadedFontId};
@@ -179,7 +183,7 @@ impl<'a> BitmapImage for DecodedImageHandle<'a> {
         match &*self.h {
             ImageDataType::Rgba8 { data, .. } => data.as_ptr(),
             ImageDataType::AnimRgba8 { frames, .. } => frames[self.current_frame].as_ptr(),
-            ImageDataType::EncodedFile(_) => unreachable!(),
+            ImageDataType::File(_) | ImageDataType::EncodedFile(_) => unreachable!(),
         }
     }
 
@@ -191,151 +195,246 @@ impl<'a> BitmapImage for DecodedImageHandle<'a> {
         match &*self.h {
             ImageDataType::Rgba8 { width, height, .. }
             | ImageDataType::AnimRgba8 { width, height, .. } => (*width as usize, *height as usize),
-            ImageDataType::EncodedFile(_) => unreachable!(),
+            ImageDataType::File(_) | ImageDataType::EncodedFile(_) => unreachable!(),
         }
     }
 }
 
-#[self_referencing]
-struct OwnedFrames {
-    data: Arc<Vec<u8>>,
-    #[borrows(data)]
-    #[covariant]
-    frames: Frames<'this>,
+struct DecodedFrame {
+    data: Arc<Mutex<Vec<u8>>>,
+    duration: Duration,
+    hash: [u8; 32],
+    width: usize,
+    height: usize,
 }
 
-impl std::fmt::Debug for OwnedFrames {
-    fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
-        fmt.debug_struct("OwnedFrames").finish()
-    }
-}
+struct FrameDecoder {}
 
-impl OwnedFrames {
-    pub fn make(data: &Arc<Vec<u8>>, format: ImageFormat, limits: Limits) -> anyhow::Result<Self> {
-        Ok(OwnedFramesTryBuilder {
-            data: Arc::clone(data),
-            frames_builder: |data: &Arc<Vec<u8>>| Self::get_frames(format, limits, data),
-        }
-        .try_build()?)
+impl FrameDecoder {
+    pub fn start(file: ImageDataFile) -> anyhow::Result<Receiver<DecodedFrame>> {
+        let (tx, rx) = sync_channel(2);
+
+        let file_dup = file.duplicate_file()?;
+        let buf_reader = std::io::BufReader::new(file_dup);
+        let reader = image::io::Reader::new(buf_reader).with_guessed_format()?;
+        let format = reader
+            .format()
+            .ok_or_else(|| anyhow::anyhow!("cannot determine image format"))?;
+
+        let scratch_file = tempfile::tempfile().context("make scratch file for decoding image")?;
+
+        std::thread::spawn(move || {
+            if let Err(err) = Self::run_decoder_thread(reader, format, file, scratch_file, tx) {
+                if err
+                    .downcast_ref::<std::sync::mpsc::SendError<DecodedFrame>>()
+                    .is_none()
+                {
+                    log::error!("Error decoding image: {err:#}");
+                }
+            }
+        });
+
+        Ok(rx)
     }
 
-    pub fn next(&mut self) -> Option<ImageResult<Frame>> {
-        self.with_frames_mut(|frames| frames.next())
-    }
-
-    fn single_frame<'a>(
+    fn run_decoder_thread(
+        reader: image::io::Reader<BufReader<File>>,
         format: ImageFormat,
-        limits: Limits,
-        data: &'a [u8],
-    ) -> anyhow::Result<Frames<'a>> {
-        let mut reader = image::io::Reader::with_format(std::io::Cursor::new(data), format);
-        reader.limits(limits);
-        let buf = reader.decode()?;
-        let delay = image::Delay::from_numer_denom_ms(u32::MAX, 1);
-        let frame = Frame::from_parts(buf.into_rgba8(), 0, 0, delay);
-        Ok(Frames::new(Box::new(std::iter::once(ImageResult::Ok(
-            frame,
-        )))))
-    }
-
-    fn get_frames<'a>(
-        format: ImageFormat,
-        limits: Limits,
-        data: &'a [u8],
-    ) -> anyhow::Result<Frames<'a>> {
-        Ok(match format {
+        image_data_file: ImageDataFile,
+        mut scratch_file: File,
+        tx: SyncSender<DecodedFrame>,
+    ) -> anyhow::Result<()> {
+        let start = Instant::now();
+        let limits = Limits::default();
+        let mut frames = match format {
             ImageFormat::Gif => {
-                let decoder = image::codecs::gif::GifDecoder::with_limits(data, limits)?;
+                let mut reader = reader.into_inner();
+                reader.rewind()?;
+                let decoder = image::codecs::gif::GifDecoder::with_limits(reader, limits)?;
                 decoder.into_frames()
             }
             ImageFormat::Png => {
-                let decoder = image::codecs::png::PngDecoder::with_limits(data, limits.clone())?;
+                let mut reader = reader.into_inner();
+                reader.rewind()?;
+                let decoder = image::codecs::png::PngDecoder::with_limits(reader, limits.clone())?;
                 if decoder.is_apng() {
                     decoder.apng().into_frames()
                 } else {
-                    Self::single_frame(format, limits, data)?
+                    let size = decoder.total_bytes() as usize;
+                    let mut buf = vec![0u8; size];
+                    let (width, height) = decoder.dimensions();
+                    let mut reader = decoder.into_reader()?;
+                    reader.read(&mut buf)?;
+                    let buf = image::RgbaImage::from_raw(width, height, buf).ok_or_else(|| {
+                        anyhow::anyhow!("inconsistent {width}x{height} -> {size}")
+                    })?;
+                    let delay = image::Delay::from_numer_denom_ms(u32::MAX, 1);
+                    let frame = Frame::from_parts(buf, 0, 0, delay);
+                    Frames::new(Box::new(std::iter::once(ImageResult::Ok(frame))))
                 }
             }
             ImageFormat::WebP => {
-                let mut decoder = image::codecs::webp::WebPDecoder::new(data)?;
+                let mut reader = reader.into_inner();
+                reader.rewind()?;
+                let mut decoder = image::codecs::webp::WebPDecoder::new(reader)?;
                 decoder.set_limits(limits)?;
                 decoder.into_frames()
             }
-            _ => Self::single_frame(format, limits, data)?,
-        })
+            _ => {
+                let buf = reader.decode()?;
+                let delay = image::Delay::from_numer_denom_ms(u32::MAX, 1);
+                let frame = Frame::from_parts(buf.into_rgba8(), 0, 0, delay);
+                Frames::new(Box::new(std::iter::once(ImageResult::Ok(frame))))
+            }
+        };
+
+        let frame = frames
+            .next()
+            .ok_or_else(|| anyhow::anyhow!("no frames!?"))?;
+        let frame = frame?;
+
+        let mut durations = vec![];
+        let mut hashes = vec![];
+        let (width, height) = frame.buffer().dimensions();
+        let width = width as usize;
+        let height = height as usize;
+
+        let duration: Duration = frame.delay().into();
+        let mut data = frame.into_buffer().into_raw();
+        data.shrink_to_fit();
+        let buf_2 = Arc::new(Mutex::new(data.clone()));
+        let buf_1 = Arc::new(Mutex::new(data.clone()));
+        let buf_0 = Arc::new(Mutex::new(data.clone()));
+
+        let buffers = [buf_0, buf_1, buf_2];
+        let mut which_buffer = 0;
+
+        fn next_buffer(which: &mut usize) {
+            *which += 1;
+            if *which > 2 {
+                *which = 0;
+            }
+        }
+
+        log::debug!("first frame took {:?} to decode", start.elapsed());
+
+        let hash = ImageDataType::hash_bytes(&data);
+        tx.send(DecodedFrame {
+            data: Arc::clone(&buffers[which_buffer]),
+            duration,
+            hash,
+            width,
+            height,
+        })?;
+        next_buffer(&mut which_buffer);
+        durations.push(duration);
+        hashes.push(hash);
+
+        scratch_file.write_all(&data)?;
+
+        while let Some(frame) = frames.next() {
+            let frame = frame?;
+
+            let duration: Duration = frame.delay().into();
+            let data = frame.into_buffer().into_raw();
+
+            let buf = Arc::clone(&buffers[which_buffer]);
+
+            buf.lock().copy_from_slice(&data);
+
+            let hash = ImageDataType::hash_bytes(&data);
+            tx.send(DecodedFrame {
+                data: buf,
+                duration,
+                hash,
+                width,
+                height,
+            })?;
+            next_buffer(&mut which_buffer);
+            durations.push(duration);
+            hashes.push(hash);
+
+            scratch_file.write_all(&data)?;
+        }
+
+        // We no longer need to keep the source file alive, as
+        // we can serve all frames from our scratch file
+        drop(image_data_file);
+        drop(frames);
+
+        let elapsed = start.elapsed();
+        let fps = durations.len() as f32 / elapsed.as_secs_f32();
+
+        log::debug!(
+            "decoded {} frames, {} bytes in {elapsed:?}, {fps} fps",
+            durations.len(),
+            durations.len() * buffers[which_buffer].lock().len()
+        );
+
+        loop {
+            scratch_file.rewind()?;
+            for (&duration, &hash) in durations.iter().zip(hashes.iter()) {
+                let buf = Arc::clone(&buffers[which_buffer]);
+                which_buffer += 1;
+                scratch_file.read(buf.lock().as_mut_slice())?;
+                tx.send(DecodedFrame {
+                    data: buf,
+                    duration,
+                    hash,
+                    width,
+                    height,
+                })?;
+                next_buffer(&mut which_buffer);
+            }
+        }
     }
 }
 
 struct FrameState {
-    data: Arc<Vec<u8>>,
-    frames: OwnedFrames,
-    format: ImageFormat,
-    limits: Limits,
-    current_frame: Frame,
-    hash: [u8; 32],
+    rx: Receiver<DecodedFrame>,
+    current_frame: DecodedFrame,
 }
 
 impl FrameState {
-    fn new(
-        data: Arc<Vec<u8>>,
-        frames: OwnedFrames,
-        format: ImageFormat,
-        limits: Limits,
-        current_frame: Frame,
-    ) -> Self {
-        let hash = ImageDataType::hash_bytes(current_frame.buffer().as_raw());
+    fn new(rx: Receiver<DecodedFrame>) -> Self {
+        let data = vec![0u8; 4];
+        let hash = ImageDataType::hash_bytes(&data);
         Self {
-            data,
-            frames,
-            format,
-            limits,
-            current_frame,
-            hash,
+            rx,
+            current_frame: DecodedFrame {
+                data: Arc::new(Mutex::new(data)),
+                width: 1,
+                height: 1,
+                duration: Duration::from_millis(0),
+                hash,
+            },
         }
     }
-
-    fn load_next_frame(&mut self) -> anyhow::Result<()> {
-        loop {
-            let start = Instant::now();
-            match self.frames.next() {
-                Some(Ok(frame)) => {
-                    log::info!("next took {:?}", start.elapsed());
-                    let duration: Duration = frame.delay().into();
-                    if duration.as_millis() == 0 {
-                        continue;
-                    }
-                    self.hash = ImageDataType::hash_bytes(frame.buffer().as_raw());
-                    self.current_frame = frame;
-                    return Ok(());
-                }
-                Some(Err(err)) => {
-                    log::warn!("error decoding next frame: {err:#}");
-                    continue;
-                }
-                None => {
-                    log::info!("last next took {:?}", start.elapsed());
-                    let start = Instant::now();
-                    let frames = OwnedFrames::make(&self.data, self.format, self.limits.clone())?;
-                    log::info!("make took {:?}", start.elapsed());
-                    self.frames = frames;
-                    continue;
-                }
+    fn load_next_frame(&mut self) -> anyhow::Result<bool> {
+        match self.rx.try_recv() {
+            Ok(frame) => {
+                self.current_frame = frame;
+                Ok(true)
+            }
+            Err(TryRecvError::Empty) => Ok(false),
+            Err(TryRecvError::Disconnected) => {
+                anyhow::bail!("decoded thread terminated");
             }
         }
     }
 
     fn frame_duration(&self) -> Duration {
-        self.current_frame.delay().into()
+        self.current_frame.duration
     }
 
     fn frame_hash(&self) -> &[u8; 32] {
-        &self.hash
+        &self.current_frame.hash
     }
 }
 
 impl BitmapImage for FrameState {
     unsafe fn pixel_data(&self) -> *const u8 {
-        self.current_frame.buffer().as_raw().as_slice().as_ptr()
+        self.current_frame.data.lock().as_slice().as_ptr()
     }
 
     unsafe fn pixel_data_mut(&mut self) -> *mut u8 {
@@ -343,8 +442,7 @@ impl BitmapImage for FrameState {
     }
 
     fn image_dimensions(&self) -> (usize, usize) {
-        let (width, height) = self.current_frame.buffer().dimensions();
-        (width as usize, height as usize)
+        (self.current_frame.width, self.current_frame.height)
     }
 }
 
@@ -376,40 +474,25 @@ impl DecodedImage {
 
     fn load(image_data: &Arc<ImageData>) -> Self {
         match &*image_data.data() {
-            ImageDataType::EncodedFile(data) => match image::guess_format(&data) {
-                Ok(format) => {
-                    let data = Arc::new(data.clone());
-                    let limits = Limits::default();
-                    match OwnedFrames::make(&data, format, limits.clone()) {
-                        Ok(mut frames) => match frames.next() {
-                            Some(Ok(current_frame)) => Self {
-                                frame_start: RefCell::new(Instant::now()),
-                                current_frame: RefCell::new(0),
-                                image: Arc::clone(image_data),
-                                frames: RefCell::new(Some(FrameState::new(
-                                    data,
-                                    frames,
-                                    format,
-                                    limits,
-                                    current_frame,
-                                ))),
-                            },
-                            _ => {
-                                log::warn!("unable to decode first image frame. Using placeholder");
-                                Self::placeholder()
-                            }
-                        },
-                        Err(err) => {
-                            log::warn!("unable to decode image: {err:#}. Using placeholder");
-                            Self::placeholder()
-                        }
-                    }
-                }
+            ImageDataType::File(file) => match FrameDecoder::start(file.clone()) {
+                Ok(rx) => Self {
+                    frame_start: RefCell::new(Instant::now()),
+                    current_frame: RefCell::new(0),
+                    image: Arc::clone(image_data),
+                    frames: RefCell::new(Some(FrameState::new(rx))),
+                },
                 Err(err) => {
-                    log::warn!("unable to decode image: {err:#}. Using placeholder");
+                    log::error!("failed to start FrameDecoder: {err:#}");
                     Self::placeholder()
                 }
             },
+            ImageDataType::EncodedFile(_) => {
+                log::error!("Unexpected EncodedFile; should have File here");
+                // The swap_out call happens in the terminal layer
+                // when normalizing/caching the data just prior to
+                // applying it to the terminal model
+                Self::placeholder()
+            }
             ImageDataType::AnimRgba8 { durations, .. } => {
                 let current_frame = if durations.len() > 1 && durations[0].as_millis() == 0 {
                     // Skip possible 0-duration root frame
@@ -833,7 +916,7 @@ impl GlyphCache {
                     ),
                 ));
             }
-            ImageDataType::EncodedFile(_) => {
+            ImageDataType::File(_) | ImageDataType::EncodedFile(_) => {
                 let mut frames = decoded.frames.borrow_mut();
                 let frames = frames.as_mut().expect("to have frames");
 
@@ -856,12 +939,13 @@ impl GlyphCache {
                     *decoded_frame_start + frames.frame_duration().max(min_frame_duration);
                 if now >= next_due {
                     // Advance to next frame
-                    frames.load_next_frame()?;
-                    *decoded_current_frame = *decoded_current_frame + 1;
-                    *decoded_frame_start = now;
-                    next_due =
-                        *decoded_frame_start + frames.frame_duration().max(min_frame_duration);
-                    handle.current_frame = *decoded_current_frame;
+                    if frames.load_next_frame()? {
+                        *decoded_current_frame = *decoded_current_frame + 1;
+                        *decoded_frame_start = now;
+                        next_due =
+                            *decoded_frame_start + frames.frame_duration().max(min_frame_duration);
+                        handle.current_frame = *decoded_current_frame;
+                    }
                 }
 
                 next.replace(next_due);

wezterm-ssh/Cargo.toml

@@ -49,5 +49,5 @@ env_logger = "0.10"
 rstest = "0.16"
 shell-words = "1.1"
 smol-potat = "1.1.2"
-termwiz = { version = "0.20", path = "../termwiz" }
+termwiz = { version = "0.21", path = "../termwiz" }
 whoami = "1.1"