implementing progress bars and trackers for large file readers, using in map loading phases that might be slow

abstutil/src/io.rs

@@ -7,8 +7,10 @@ use std;
 use std::collections::{BTreeMap, BTreeSet};
 use std::fs::File;
 use std::hash::Hash;
-use std::io::{Error, ErrorKind, Read, Write};
+use std::io::{stdout, BufReader, Error, ErrorKind, Read, Write};
 use std::path::Path;
+use std::time::Instant;
+use time::elapsed_seconds;
 
 pub fn to_json<T: Serialize>(obj: &T) -> String {
     serde_json::to_string_pretty(obj).unwrap()
@@ -45,8 +47,9 @@ pub fn write_binary<T: Serialize>(path: &str, obj: &T) -> Result<(), Error> {
 }
 
 pub fn read_binary<T: DeserializeOwned>(path: &str) -> Result<T, Error> {
-    let file = File::open(path)?;
+    let reader = FileWithProgress::new(path)?;
-    let obj: T = serde_cbor::from_reader(file).map_err(|err| Error::new(ErrorKind::Other, err))?;
+    let obj: T =
+        serde_cbor::from_reader(reader).map_err(|err| Error::new(ErrorKind::Other, err))?;
     Ok(obj)
 }
 
@@ -167,3 +170,61 @@ pub fn save_object<T: Serialize>(dir: &str, map_name: &str, obj_name: &str, obj:
     write_json(&path, obj).expect(&format!("Saving {} failed", path));
     println!("Saved {}", path);
 }
+
+struct FileWithProgress {
+    inner: BufReader<File>,
+
+    path: String,
+    processed_bytes: usize,
+    total_bytes: usize,
+    started_at: Instant,
+    last_printed_at: Instant,
+}
+
+impl FileWithProgress {
+    pub fn new(path: &str) -> Result<FileWithProgress, Error> {
+        let file = File::open(path)?;
+        let total_bytes = file.metadata()?.len() as usize;
+        Ok(FileWithProgress {
+            inner: BufReader::new(file),
+            path: path.to_string(),
+            processed_bytes: 0,
+            total_bytes,
+            started_at: Instant::now(),
+            last_printed_at: Instant::now(),
+        })
+    }
+}
+
+impl Read for FileWithProgress {
+    fn read(&mut self, buf: &mut [u8]) -> Result<usize, Error> {
+        let bytes = self.inner.read(buf)?;
+        self.processed_bytes += bytes;
+        if self.processed_bytes > self.total_bytes {
+            panic!(
+                "{} is too many bytes read from {}",
+                self.processed_bytes, self.path
+            );
+        }
+
+        let done = self.processed_bytes == self.total_bytes && bytes == 0;
+        if elapsed_seconds(self.last_printed_at) >= 1.0 || done {
+            self.last_printed_at = Instant::now();
+            print!(
+                "{}Reading {}: {}/{} MB... {}s",
+                "\r",
+                self.path,
+                self.processed_bytes / 1024 / 1024,
+                self.total_bytes / 1024 / 1024,
+                elapsed_seconds(self.started_at)
+            );
+            if done {
+                println!("");
+            } else {
+                stdout().flush().unwrap();
+            }
+        }
+
+        Ok(bytes)
+    }
+}

abstutil/src/lib.rs

@@ -21,4 +21,4 @@ pub use io::{
     write_json,
 };
 pub use logs::{format_log_record, LogAdapter};
-pub use time::elapsed_seconds;
+pub use time::{elapsed_seconds, Progress};

abstutil/src/time.rs

@@ -1,6 +1,55 @@
+use std::io::{stdout, Write};
 use std::time::Instant;
 
 pub fn elapsed_seconds(since: Instant) -> f64 {
     let dt = since.elapsed();
     (dt.as_secs() as f64) + (f64::from(dt.subsec_nanos()) * 1e-9)
 }
+
+pub struct Progress {
+    label: String,
+    processed_items: usize,
+    total_items: usize,
+    started_at: Instant,
+    last_printed_at: Instant,
+}
+
+impl Progress {
+    pub fn new(label: &str, total_items: usize) -> Progress {
+        Progress {
+            label: label.to_string(),
+            processed_items: 0,
+            total_items,
+            started_at: Instant::now(),
+            last_printed_at: Instant::now(),
+        }
+    }
+
+    pub fn next(&mut self) {
+        self.processed_items += 1;
+        if self.processed_items > self.total_items {
+            panic!(
+                "{} is too few items for {} progress",
+                self.total_items, self.label
+            );
+        }
+
+        let done = self.processed_items == self.total_items;
+        if elapsed_seconds(self.last_printed_at) >= 1.0 || done {
+            self.last_printed_at = Instant::now();
+            print!(
+                "{}{}: {}/{}... {}s",
+                "\r",
+                self.label,
+                self.processed_items,
+                self.total_items,
+                elapsed_seconds(self.started_at),
+            );
+            if done {
+                println!("");
+            } else {
+                stdout().flush().unwrap();
+            }
+        }
+    }
+}

map_model/src/make/buildings.rs

@@ -1,3 +1,4 @@
+use abstutil::Progress;
 use dimensioned::si;
 use geom::{Bounds, HashablePt2D, Line, PolyLine, Pt2D};
 use make::sidewalk_finder::find_sidewalk_points;
@@ -14,7 +15,9 @@ pub(crate) fn make_all_buildings(
     let mut pts_per_bldg: Vec<Vec<Pt2D>> = Vec::new();
     let mut center_per_bldg: Vec<HashablePt2D> = Vec::new();
     let mut query: HashSet<HashablePt2D> = HashSet::new();
+    let mut progress = Progress::new("get building center points", input.len());
     for b in input {
+        progress.next();
         let pts = b
             .points
             .iter()
@@ -28,7 +31,9 @@ pub(crate) fn make_all_buildings(
 
     let sidewalk_pts = find_sidewalk_points(query, lanes);
 
+    let mut progress = Progress::new("create building front paths", pts_per_bldg.len());
     for (idx, points) in pts_per_bldg.into_iter().enumerate() {
+        progress.next();
         let bldg_center = center_per_bldg[idx];
         let (sidewalk, dist_along) = sidewalk_pts[&bldg_center];
         let (sidewalk_pt, _) = lanes[sidewalk.0].dist_along(dist_along);

map_model/src/make/sidewalk_finder.rs

@@ -1,3 +1,4 @@
+use abstutil::Progress;
 use dimensioned::si;
 use geo;
 use geo::prelude::{ClosestPoint, EuclideanDistance};
@@ -11,9 +12,11 @@ pub fn find_sidewalk_points(
     lanes: &Vec<Lane>,
 ) -> HashMap<HashablePt2D, (LaneID, si::Meter<f64>)> {
     // Get LineStrings of all lanes once.
+    let mut progress = Progress::new("lanes to LineStrings", lanes.len());
     let line_strings: Vec<(LaneID, geo::LineString<f64>)> = lanes
         .iter()
         .filter_map(|l| {
+            progress.next();
             if l.is_sidewalk() {
                 Some((l.id, lane_to_line_string(l)))
             } else {
@@ -23,7 +26,9 @@ pub fn find_sidewalk_points(
 
     // For each point, find the closest point to any sidewalk
     let mut results: HashMap<HashablePt2D, (LaneID, si::Meter<f64>)> = HashMap::new();
+    let mut progress = Progress::new("find closest sidewalk point", pts.len());
     for query_pt in pts {
+        progress.next();
         let query_geo_pt = geo::Point::new(query_pt.x(), query_pt.y());
         let (sidewalk, raw_pt) = line_strings
             .iter()