Add back the zcool font so the Taipei map renders names. But this time, asynchronously load the font instead of bundling it in system assets and increasing the .wasm size. #535

Temporarily take on some technical debt with the new RawFileLoader...

abstio/src/abst_data.rs

@@ -49,6 +49,10 @@ impl Manifest {
                 remove.push(path.clone());
                 continue;
             }
+            if path.starts_with("data/system/extra_fonts") {
+                // Always grab all of these
+                continue;
+            }
 
             let parts = path.split("/").collect::<Vec<_>>();
             let city = format!("{}/{}", parts[2], parts[3]);

abstio/src/abst_paths.rs

@@ -86,7 +86,11 @@ impl CityName {
     pub fn list_all_cities_from_system_data() -> Vec<CityName> {
         let mut cities = Vec::new();
         for country in list_all_objects(path("system")) {
-            if country == "assets" || country == "proposals" || country == "study_areas" {
+            if country == "assets"
+                || country == "extra_fonts"
+                || country == "proposals"
+                || country == "study_areas"
+            {
                 continue;
             }
             for city in list_all_objects(path(format!("system/{}", country))) {

data/MANIFEST.json

@@ -1480,6 +1480,11 @@
       "uncompressed_size_bytes": 25964767,
       "compressed_size_bytes": 8714926
     },
+    "data/system/extra_fonts/ZCOOLXiaoWei-Regular.ttf": {
+      "checksum": "edea762b9f6ccee6d330ec37b6d1cd66",
+      "uncompressed_size_bytes": 6302056,
+      "compressed_size_bytes": 3409781
+    },
     "data/system/fr/charleville_mezieres/city.bin": {
       "checksum": "a283fcc74ca4d6d2b471204440a7a20f",
       "uncompressed_size_bytes": 292296,

map_gui/src/load.rs

@@ -20,10 +20,10 @@ use crate::tools::PopupMsg;
 use crate::AppLike;
 
 #[cfg(not(target_arch = "wasm32"))]
-pub use native_loader::FileLoader;
+pub use native_loader::{FileLoader, RawFileLoader};
 
 #[cfg(target_arch = "wasm32")]
-pub use wasm_loader::FileLoader;
+pub use wasm_loader::{FileLoader, RawFileLoader};
 
 pub struct MapLoader;
 
@@ -40,6 +40,26 @@ impl MapLoader {
             });
         }
 
+        // TODO Generalize this more, maybe with some kind of country code -> font config
+        let zcool = "ZCOOLXiaoWei-Regular.ttf";
+        if name.city.country == "tw" && !ctx.is_font_loaded(zcool) {
+            return RawFileLoader::<A>::new(
+                ctx,
+                abstio::path(format!("system/extra_fonts/{}", zcool)),
+                Box::new(move |ctx, app, bytes| match bytes {
+                    Ok(bytes) => {
+                        ctx.load_font(zcool, bytes);
+                        Transition::Replace(MapLoader::new(ctx, app, name, on_load))
+                    }
+                    Err(err) => Transition::Replace(PopupMsg::new(
+                        ctx,
+                        "Error",
+                        vec![format!("Couldn't load {}", zcool), err.to_string()],
+                    )),
+                }),
+            );
+        }
+
         FileLoader::<A, map_model::Map>::new(
             ctx,
             name.path(),
@@ -81,6 +101,7 @@ impl<A: AppLike + 'static> State<A> for MapAlreadyLoaded<A> {
 mod native_loader {
     use super::*;
 
+    // This loads a JSON or bincoded file, then deserializes it
     pub struct FileLoader<A: AppLike, T> {
         path: String,
         // Wrapped in an Option just to make calling from event() work. Technically this is unsafe
@@ -115,10 +136,45 @@ mod native_loader {
             g.clear(Color::BLACK);
         }
     }
+
+    // TODO Ideally merge with FileLoader
+    pub struct RawFileLoader<A: AppLike> {
+        path: String,
+        // Wrapped in an Option just to make calling from event() work. Technically this is unsafe
+        // if a caller fails to pop the FileLoader state in their transitions!
+        on_load: Option<Box<dyn FnOnce(&mut EventCtx, &mut A, Result<Vec<u8>>) -> Transition<A>>>,
+    }
+
+    impl<A: AppLike + 'static> RawFileLoader<A> {
+        pub fn new(
+            _: &mut EventCtx,
+            path: String,
+            on_load: Box<dyn FnOnce(&mut EventCtx, &mut A, Result<Vec<u8>>) -> Transition<A>>,
+        ) -> Box<dyn State<A>> {
+            Box::new(RawFileLoader {
+                path,
+                on_load: Some(on_load),
+            })
+        }
+    }
+
+    impl<A: AppLike + 'static> State<A> for RawFileLoader<A> {
+        fn event(&mut self, ctx: &mut EventCtx, app: &mut A) -> Transition<A> {
+            debug!("Loading {}", self.path);
+            let bytes = abstio::slurp_file(&self.path);
+            (self.on_load.take().unwrap())(ctx, app, bytes)
+        }
+
+        fn draw(&self, g: &mut GfxCtx, _: &A) {
+            g.clear(Color::BLACK);
+        }
+    }
 }
 
 #[cfg(target_arch = "wasm32")]
 mod wasm_loader {
+    use std::io::Read;
+
     use futures_channel::oneshot;
     use instant::Instant;
     use wasm_bindgen::JsCast;
@@ -251,6 +307,119 @@ mod wasm_loader {
         }
     }
 
+    // TODO This is a horrible copy of FileLoader. Make the serde FileLoader just build on top of
+    // this one!!!
+    pub struct RawFileLoader<A: AppLike> {
+        response: oneshot::Receiver<Result<Vec<u8>>>,
+        on_load: Option<Box<dyn FnOnce(&mut EventCtx, &mut A, Result<Vec<u8>>) -> Transition<A>>>,
+        panel: Panel,
+        started: Instant,
+        url: String,
+    }
+
+    impl<A: AppLike + 'static> RawFileLoader<A> {
+        pub fn new(
+            ctx: &mut EventCtx,
+            path: String,
+            on_load: Box<dyn FnOnce(&mut EventCtx, &mut A, Result<Vec<u8>>) -> Transition<A>>,
+        ) -> Box<dyn State<A>> {
+            // The current URL is of the index.html page. We can find the data directory relative
+            // to that.
+            let base_url = get_base_url().unwrap();
+            let file_path = path.strip_prefix(&abstio::path("")).unwrap();
+            // Note that files are gzipped on S3 and other deployments. When running locally, we
+            // just symlink the data/ directory, where files aren't compressed.
+            let url =
+                if base_url.contains("http://0.0.0.0") || base_url.contains("http://localhost") {
+                    format!("{}/{}", base_url, file_path)
+                } else if base_url.contains("abstreet.s3-website") {
+                    // The directory structure on S3 is a little weird -- the base directory has
+                    // data/ alongside game/, fifteen_min/, etc.
+                    format!("{}/../data/{}.gz", base_url, file_path)
+                } else {
+                    format!("{}/{}.gz", base_url, file_path)
+                };
+
+            // Make the HTTP request nonblockingly. When the response is received, send it through
+            // the channel.
+            let (tx, rx) = oneshot::channel();
+            let url_copy = url.clone();
+            debug!("Loading {}", url_copy);
+            wasm_bindgen_futures::spawn_local(async move {
+                let mut opts = RequestInit::new();
+                opts.method("GET");
+                opts.mode(RequestMode::Cors);
+                let request = Request::new_with_str_and_init(&url_copy, &opts).unwrap();
+
+                let window = web_sys::window().unwrap();
+                match JsFuture::from(window.fetch_with_request(&request)).await {
+                    Ok(resp_value) => {
+                        let resp: Response = resp_value.dyn_into().unwrap();
+                        if resp.ok() {
+                            let buf = JsFuture::from(resp.array_buffer().unwrap()).await.unwrap();
+                            let array = js_sys::Uint8Array::new(&buf);
+                            tx.send(Ok(array.to_vec())).unwrap();
+                        } else {
+                            let status = resp.status();
+                            let err = resp.status_text();
+                            tx.send(Err(anyhow!("HTTP {}: {}", status, err))).unwrap();
+                        }
+                    }
+                    Err(err) => {
+                        tx.send(Err(anyhow!("{:?}", err))).unwrap();
+                    }
+                }
+            });
+
+            Box::new(RawFileLoader {
+                response: rx,
+                on_load: Some(on_load),
+                panel: ctx.make_loading_screen(Text::from(Line(format!("Loading {}...", url)))),
+                started: Instant::now(),
+                url,
+            })
+        }
+    }
+
+    impl<A: AppLike + 'static> State<A> for RawFileLoader<A> {
+        fn event(&mut self, ctx: &mut EventCtx, app: &mut A) -> Transition<A> {
+            if let Some(maybe_resp) = self.response.try_recv().unwrap() {
+                let bytes = if self.url.ends_with(".gz") {
+                    maybe_resp.and_then(|gzipped| {
+                        let mut decoder = flate2::read::GzDecoder::new(&gzipped[..]);
+                        let mut buffer: Vec<u8> = Vec::new();
+                        decoder
+                            .read_to_end(&mut buffer)
+                            .map(|_| buffer)
+                            .map_err(|err| err.into())
+                    })
+                } else {
+                    maybe_resp
+                };
+                return (self.on_load.take().unwrap())(ctx, app, bytes);
+            }
+
+            self.panel = ctx.make_loading_screen(Text::from_multiline(vec![
+                Line(format!("Loading {}...", self.url)),
+                Line(format!(
+                    "Time spent: {}",
+                    Duration::realtime_elapsed(self.started)
+                )),
+            ]));
+
+            // Until the response is received, just ask winit to regularly call event(), so we can
+            // keep polling the channel.
+            ctx.request_update(UpdateType::Game);
+            Transition::Keep
+        }
+
+        fn draw(&self, g: &mut GfxCtx, _: &A) {
+            // TODO Progress bar for bytes received
+            g.clear(Color::BLACK);
+            self.panel.draw(g);
+        }
+    }
+
     /// Returns the base URL where the game is running, excluding query parameters and the
     /// implicit index.html that might be there.
     fn get_base_url() -> Result<String> {

release/INSTRUCTIONS.txt

@@ -39,6 +39,8 @@ Other binary data bundled in:
 - Overpass font (<https://fonts.google.com/specimen/Overpass>, Open Font
   License)
 - Bungee fonts (<https://fonts.google.com/specimen/Bungee>, Open Font License)
+- ZCOOL XiaoWei fonts (<https://fonts.google.com/specimen/ZCOOL+XiaoWei>, Open
+  Font License)
 - Material Design icons (<https://material.io/resources/icons>, Apache license)
 - Some Graphics textures (<https://www.kenney.nl/>, CC0 1.0 Universal)
 - Snowflake SVG (<https://www.svgrepo.com/page/licensing>, CC0)

updater/src/main.rs

@@ -133,6 +133,7 @@ fn upload(version: String) {
     }
 
     // Anything missing or needing updating?
+    // TODO Parallelize, since compression can be slow!
     for (path, entry) in &mut local.entries {
         let remote_path = format!("{}/{}.gz", remote_base, path);
         let changed = remote.entries.get(path).map(|x| &x.checksum) != Some(&entry.checksum);
@@ -187,6 +188,9 @@ fn opt_into_all() {
             data_packs.runtime.insert("us/huge_seattle".to_string());
             continue;
         }
+        if path.starts_with("data/system/extra_fonts") {
+            continue;
+        }
         let parts = path.split("/").collect::<Vec<_>>();
         let city = format!("{}/{}", parts[2], parts[3]);
         if parts[1] == "input" {

widgetry/src/assets.rs

@@ -1,5 +1,5 @@
 use std::cell::RefCell;
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 
 use lru::LruCache;
 use usvg::fontdb;
@@ -18,44 +18,35 @@ pub struct Assets {
     // Keyed by filename
     svg_cache: RefCell<HashMap<String, (GeomBatch, Bounds)>>,
     font_to_id: HashMap<Font, fontdb::ID>,
+    extra_fonts: RefCell<HashSet<String>>,
     pub(crate) style: RefCell<Style>,
-    pub text_opts: Options,
+    pub text_opts: RefCell<Options>,
     pub read_svg: Box<dyn Fn(&str) -> Vec<u8>>,
 }
 
 impl Assets {
     pub fn new(style: Style, read_svg: Box<dyn Fn(&str) -> Vec<u8>>) -> Assets {
+        // Many fonts are statically bundled with the library right now, on both native and web.
+        // ctx.is_font_loaded and ctx.load_font can be used to dynamically add more later.
+        let mut fontdb = fontdb::Database::new();
+        fontdb.load_font_data(include_bytes!("../fonts/BungeeInline-Regular.ttf").to_vec());
+        fontdb.load_font_data(include_bytes!("../fonts/Bungee-Regular.ttf").to_vec());
+        fontdb.load_font_data(include_bytes!("../fonts/Overpass-Bold.ttf").to_vec());
+        fontdb.load_font_data(include_bytes!("../fonts/OverpassMono-Bold.ttf").to_vec());
+        fontdb.load_font_data(include_bytes!("../fonts/Overpass-Regular.ttf").to_vec());
+        fontdb.load_font_data(include_bytes!("../fonts/Overpass-SemiBold.ttf").to_vec());
         let mut a = Assets {
             default_line_height: RefCell::new(0.0),
             text_cache: RefCell::new(LruCache::new(500)),
             line_height_cache: RefCell::new(HashMap::new()),
             svg_cache: RefCell::new(HashMap::new()),
             font_to_id: HashMap::new(),
-            text_opts: Options::default(),
+            extra_fonts: RefCell::new(HashSet::new()),
+            text_opts: RefCell::new(Options::default()),
             style: RefCell::new(style),
             read_svg,
         };
-        // All fonts are statically bundled with the library right now, on both native and web.
+        a.text_opts.borrow_mut().fontdb = fontdb;
-        // Eventually need to let people specify their own fonts dynamically at runtime.
-        a.text_opts.fontdb = fontdb::Database::new();
-        a.text_opts
-            .fontdb
-            .load_font_data(include_bytes!("../fonts/BungeeInline-Regular.ttf").to_vec());
-        a.text_opts
-            .fontdb
-            .load_font_data(include_bytes!("../fonts/Bungee-Regular.ttf").to_vec());
-        a.text_opts
-            .fontdb
-            .load_font_data(include_bytes!("../fonts/Overpass-Bold.ttf").to_vec());
-        a.text_opts
-            .fontdb
-            .load_font_data(include_bytes!("../fonts/OverpassMono-Bold.ttf").to_vec());
-        a.text_opts
-            .fontdb
-            .load_font_data(include_bytes!("../fonts/Overpass-Regular.ttf").to_vec());
-        a.text_opts
-            .fontdb
-            .load_font_data(include_bytes!("../fonts/Overpass-SemiBold.ttf").to_vec());
         for font in vec![
             Font::BungeeInlineRegular,
             Font::BungeeRegular,
@@ -67,6 +58,7 @@ impl Assets {
             a.font_to_id.insert(
                 font,
                 a.text_opts
+                    .borrow()
                     .fontdb
                     .query(&fontdb::Query {
                         families: &vec![fontdb::Family::Name(font.family())],
@@ -86,6 +78,18 @@ impl Assets {
         a
     }
 
+    pub fn is_font_loaded(&self, filename: &str) -> bool {
+        self.extra_fonts.borrow().contains(filename)
+    }
+
+    pub fn load_font(&self, filename: &str, bytes: Vec<u8>) {
+        info!("Loaded extra font {}", filename);
+        self.extra_fonts.borrow_mut().insert(filename.to_string());
+        self.text_opts.borrow_mut().fontdb.load_font_data(bytes);
+        // We don't need to fill out font_to_id, because we can't directly create text using this
+        // font.
+    }
+
     pub fn line_height(&self, font: Font, font_size: usize) -> f64 {
         let key = (font, font_size);
         if let Some(height) = self.line_height_cache.borrow().get(&key) {
@@ -95,6 +99,7 @@ impl Assets {
         // This seems to be missing line_gap, and line_gap is 0, so manually adjust here.
         let line_height = self
             .text_opts
+            .borrow()
             .fontdb
             .with_face_data(self.font_to_id[&font], |data, face_index| {
                 let font = ttf_parser::Face::from_slice(data, face_index).unwrap();

widgetry/src/event_ctx.rs

@@ -197,6 +197,17 @@ impl<'a> EventCtx<'a> {
         .aligned(HorizontalAlignment::Center, VerticalAlignment::Center)
         .build_custom(self)
     }
+
+    /// Checks if an extra font has previously been loaded with `load_font`. Returns false for
+    /// built-in system fonts.
+    pub fn is_font_loaded(&self, filename: &str) -> bool {
+        self.prerender.assets.is_font_loaded(filename)
+    }
+
+    /// Loads an extra font, used only for automatic fallback of missing glyphs.
+    pub fn load_font(&mut self, filename: &str, bytes: Vec<u8>) {
+        self.prerender.assets.load_font(filename, bytes)
+    }
 }
 
 struct LoadingScreen<'a> {

widgetry/src/text.rs

@@ -477,7 +477,7 @@ fn render_line(spans: Vec<TextSpan>, tolerance: f32, assets: &Assets) -> GeomBat
     }
     write!(&mut svg, "{}</text></svg>", contents).unwrap();
 
-    let svg_tree = match usvg::Tree::from_str(&svg, &assets.text_opts) {
+    let svg_tree = match usvg::Tree::from_str(&svg, &assets.text_opts.borrow()) {
         Ok(t) => t,
         Err(err) => panic!("render_line({}): {}", contents, err),
     };
@@ -571,7 +571,7 @@ impl TextSpan {
         )
         .unwrap();
 
-        let svg_tree = match usvg::Tree::from_str(&svg, &assets.text_opts) {
+        let svg_tree = match usvg::Tree::from_str(&svg, &assets.text_opts.borrow()) {
             Ok(t) => t,
             Err(err) => panic!("curvey({}): {}", self.text, err),
         };