Get the downloader UI to actually fetch stuff. #326

abstutil/src/abst_data.rs

@@ -32,6 +32,38 @@ impl Manifest {
     pub fn load() -> Manifest {
         crate::from_json(&include_bytes!("../../data/MANIFEST.json").to_vec()).unwrap()
     }
+
+    /// Removes entries from the Manifest to match the DataPacks that should exist locally.
+    pub fn filter(mut self, data_packs: DataPacks) -> Manifest {
+        let mut remove = Vec::new();
+        for path in self.entries.keys() {
+            // TODO Some hardcoded weird exceptions
+            if !data_packs.runtime.contains("huge_seattle")
+                && path == "data/system/seattle/scenarios/montlake/everyone_weekday.bin"
+            {
+                remove.push(path.clone());
+                continue;
+            }
+
+            let parts = path.split("/").collect::<Vec<_>>();
+            if parts[1] == "input" {
+                if data_packs.input.contains(parts[2]) {
+                    continue;
+                }
+            } else if parts[1] == "system" {
+                if data_packs.runtime.contains(parts[2]) {
+                    continue;
+                }
+            } else {
+                panic!("Wait what's {}", path);
+            }
+            remove.push(path.clone());
+        }
+        for path in remove {
+            self.entries.remove(&path).unwrap();
+        }
+        self
+    }
 }
 
 /// Player-chosen groups of files to opt into downloading
@@ -45,11 +77,8 @@ pub struct DataPacks {
 
 impl DataPacks {
     /// Load the player's config for what files to download, or create the config.
+    #[cfg(not(target_arch = "wasm32"))]
     pub fn load_or_create() -> DataPacks {
-        if cfg!(target_arch = "wasm32") {
-            panic!("DataPacks::load_or_create shouldn't be called on wasm");
-        }
-
         let path = crate::path("player/data.json");
         match crate::maybe_read_json::<DataPacks>(path.clone(), &mut Timer::throwaway()) {
             Ok(mut cfg) => {

game/Cargo.toml

@@ -17,6 +17,8 @@ osm_viewer = []
 wasm = ["console_log", "futures", "futures-channel", "js-sys", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", "widgetry/wasm-backend"]
 # Just a marker to not use localhost URLs
 wasm_s3 = []
+# A marker to use a named release from S3 instead of dev for updating files
+release_s3 = []
 
 [dependencies]
 aabb-quadtree = "0.1.0"

game/src/common/city_picker.rs

@@ -160,7 +160,14 @@ impl State<App> for CityPicker {
                     );
                 }
                 "Download more cities" => {
-                    return Transition::Replace(crate::common::updater::Picker::new(ctx));
+                    let _ = "just stop this from counting as an attribute on an expression";
+                    #[cfg(not(target_arch = "wasm32"))]
+                    {
+                        return Transition::Replace(crate::common::updater::Picker::new(
+                            ctx,
+                            self.on_load.take().unwrap(),
+                        ));
+                    }
                 }
                 path => {
                     return Transition::Replace(MapLoader::new(

game/src/common/mod.rs

@@ -25,6 +25,7 @@ mod heatmap;
 mod isochrone;
 mod minimap;
 mod navigate;
+#[cfg(not(target_arch = "wasm32"))]
 mod updater;
 mod warp;
 

game/src/common/updater.rs

@@ -1,44 +1,82 @@
 use std::collections::BTreeMap;
+use std::error::Error;
+use std::fs::File;
 
-use abstutil::{DataPacks, Manifest};
+use abstutil::{DataPacks, Manifest, Timer};
 use widgetry::{Btn, Checkbox, EventCtx, GfxCtx, Line, Outcome, Panel, State, TextExt, Widget};
 
 use crate::app::App;
-use crate::game::Transition;
+use crate::game::{PopupMsg, Transition};
+
+const LATEST_RELEASE: &str = "0.2.17";
 
 pub struct Picker {
     panel: Panel,
+    on_load: Option<Box<dyn FnOnce(&mut EventCtx, &mut App) -> Transition>>,
 }
 
 impl Picker {
-    pub fn new(ctx: &mut EventCtx) -> Box<dyn State<App>> {
+    pub fn new(
+        ctx: &mut EventCtx,
+        on_load: Box<dyn FnOnce(&mut EventCtx, &mut App) -> Transition>,
+    ) -> Box<dyn State<App>> {
         let manifest = Manifest::load();
         let data_packs = DataPacks::load_or_create();
 
-        let mut col = vec![Widget::row(vec![
-            Line("Download more cities").small_heading().draw(ctx),
-            Btn::close(ctx),
-        ])];
+        let mut col = vec![
+            Widget::row(vec![
+                Line("Download more cities").small_heading().draw(ctx),
+                Btn::close(ctx),
+            ]),
+            "Select the cities you want to include".draw_text(ctx),
+            Line("The file sizes shown are uncompressed; the download size will be smaller")
+                .secondary()
+                .draw(ctx),
+        ];
         for (city, bytes) in size_per_city(&manifest) {
             col.push(Widget::row(vec![
                 Checkbox::checkbox(ctx, &city, None, data_packs.runtime.contains(&city)),
-                prettyprint_bytes(bytes).draw_text(ctx),
+                prettyprint_bytes(bytes).draw_text(ctx).centered_vert(),
             ]));
         }
+        col.push(Btn::text_bg2("Sync files").build_def(ctx, None));
 
         Box::new(Picker {
             panel: Panel::new(Widget::col(col)).build(ctx),
+            on_load: Some(on_load),
         })
     }
 }
 
 impl State<App> for Picker {
-    fn event(&mut self, ctx: &mut EventCtx, _: &mut App) -> Transition {
+    fn event(&mut self, ctx: &mut EventCtx, app: &mut App) -> Transition {
         match self.panel.event(ctx) {
             Outcome::Clicked(x) => match x.as_ref() {
                 "close" => {
                     return Transition::Pop;
                 }
+                "Sync files" => {
+                    // First update the DataPacks file
+                    let mut data_packs = DataPacks::load_or_create();
+                    data_packs.runtime.clear();
+                    data_packs.runtime.insert("seattle".to_string());
+                    for (city, _) in size_per_city(&Manifest::load()) {
+                        if self.panel.is_checked(&city) {
+                            data_packs.runtime.insert(city);
+                        }
+                    }
+                    abstutil::write_json(abstutil::path("player/data.json"), &data_packs);
+
+                    let messages = ctx.loading_screen("sync files", |_, timer| sync(timer));
+                    return Transition::Multi(vec![
+                        Transition::Replace(crate::common::CityPicker::new(
+                            ctx,
+                            app,
+                            self.on_load.take().unwrap(),
+                        )),
+                        Transition::Push(PopupMsg::new(ctx, "Download complete", messages)),
+                    ]);
+                }
                 _ => unreachable!(),
             },
             _ => {}
@@ -58,7 +96,13 @@ fn size_per_city(manifest: &Manifest) -> BTreeMap<String, usize> {
     for (path, entry) in &manifest.entries {
         let parts = path.split("/").collect::<Vec<_>>();
         if parts[1] == "system" {
-            *per_city.entry(parts[2].to_string()).or_insert(0) += entry.size_bytes;
+            // The map and scenario for huge_seattle should count as a separate data pack.
+            let city = if parts.get(4) == Some(&"huge_seattle") {
+                "huge_seattle".to_string()
+            } else {
+                parts[2].to_string()
+            };
+            *per_city.entry(city).or_insert(0) += entry.size_bytes;
         }
     }
     per_city
@@ -75,3 +119,78 @@ fn prettyprint_bytes(bytes: usize) -> String {
     let mb = kb / 1024.0;
     format!("{} mb", mb as usize)
 }
+
+// TODO This only downloads files that don't exist but should. It doesn't remove or update
+// anything. Not sure if everything the updater does should also be done here.
+fn sync(timer: &mut Timer) -> Vec<String> {
+    let truth = Manifest::load().filter(DataPacks::load_or_create());
+    let version = if cfg!(feature = "release_s3") {
+        LATEST_RELEASE
+    } else {
+        "dev"
+    };
+
+    let mut files_downloaded = 0;
+    let mut bytes_downloaded = 0;
+    let mut messages = Vec::new();
+
+    timer.start_iter("sync files", truth.entries.len());
+    for (path, entry) in truth.entries {
+        timer.next();
+        let local_path = abstutil::path(path.strip_prefix("data/").unwrap());
+        if abstutil::file_exists(&local_path) {
+            continue;
+        }
+        let url = format!(
+            "http://abstreet.s3-website.us-east-2.amazonaws.com/{}/{}.gz",
+            version, path
+        );
+        timer.note(format!(
+            "Downloading {} ({} uncompressed)",
+            url,
+            prettyprint_bytes(entry.size_bytes)
+        ));
+        files_downloaded += 1;
+
+        std::fs::create_dir_all(std::path::Path::new(&local_path).parent().unwrap()).unwrap();
+        match download(&url, local_path, timer) {
+            Ok(bytes) => {
+                bytes_downloaded += bytes;
+            }
+            Err(err) => {
+                let msg = format!("Problem with {}: {}", url, err);
+                timer.error(msg.clone());
+                messages.push(msg);
+            }
+        }
+    }
+    messages.insert(
+        0,
+        format!(
+            "Downloaded {} files, total {}",
+            files_downloaded,
+            prettyprint_bytes(bytes_downloaded)
+        ),
+    );
+    messages
+}
+
+// Bytes downloaded if succesful
+fn download(url: &str, local_path: String, timer: &mut Timer) -> Result<usize, Box<dyn Error>> {
+    let mut resp = reqwest::blocking::get(url)?;
+    if !resp.status().is_success() {
+        return Err(format!("bad status: {:?}", resp.status()).into());
+    }
+    let mut buffer: Vec<u8> = Vec::new();
+    let bytes = resp.copy_to(&mut buffer)? as usize;
+
+    timer.note(format!(
+        "Decompressing {} ({})",
+        url,
+        prettyprint_bytes(bytes)
+    ));
+    let mut decoder = flate2::read::GzDecoder::new(&buffer[..]);
+    let mut out = File::create(&local_path).unwrap();
+    std::io::copy(&mut decoder, &mut out)?;
+    Ok(bytes)
+}

updater/src/main.rs

@@ -35,7 +35,7 @@ async fn main() {
 async fn download() {
     let data_packs = DataPacks::load_or_create();
     let local = generate_manifest();
-    let truth = filter_manifest(Manifest::load(), data_packs);
+    let truth = Manifest::load().filter(data_packs);
 
     // Anything local need deleting?
     for path in local.entries.keys() {
@@ -79,7 +79,7 @@ async fn download() {
 fn just_compare() {
     let data_packs = DataPacks::load_or_create();
     let local = generate_manifest();
-    let truth = filter_manifest(Manifest::load(), data_packs);
+    let truth = Manifest::load().filter(data_packs);
 
     // Anything local need deleting?
     for path in local.entries.keys() {
@@ -199,37 +199,6 @@ fn generate_manifest() -> Manifest {
     Manifest { entries: kv }
 }
 
-fn filter_manifest(mut manifest: Manifest, data_packs: DataPacks) -> Manifest {
-    let mut remove = Vec::new();
-    for path in manifest.entries.keys() {
-        // TODO Some hardcoded weird exceptions
-        if !data_packs.runtime.contains("huge_seattle")
-            && path == "data/system/seattle/scenarios/montlake/everyone_weekday.bin"
-        {
-            remove.push(path.clone());
-            continue;
-        }
-
-        let parts = path.split("/").collect::<Vec<_>>();
-        if parts[1] == "input" {
-            if data_packs.input.contains(parts[2]) {
-                continue;
-            }
-        } else if parts[1] == "system" {
-            if data_packs.runtime.contains(parts[2]) {
-                continue;
-            }
-        } else {
-            panic!("Wait what's {}", path);
-        }
-        remove.push(path.clone());
-    }
-    for path in remove {
-        manifest.entries.remove(&path).unwrap();
-    }
-    manifest
-}
-
 fn must_run_cmd(cmd: &mut Command) {
     println!("> Running {:?}", cmd);
     match cmd.status() {

widgetry/src/app_state.rs

@@ -5,7 +5,7 @@
 //! screen or menu to choose a map, then a map viewer, then maybe a state to drill down into pieces
 //! of the map.
 
-use crate::{Canvas, EventCtx, GfxCtx};
+use crate::{Canvas, Color, EventCtx, GfxCtx};
 
 /// Any data that should last the entire lifetime of the application should be stored in the struct
 /// implementing this trait.
@@ -58,8 +58,11 @@ impl<A: SharedAppState> App<A> {
                         self.shared_app_state.draw_default(g);
                     }
                     DrawBaselayer::Custom => {}
-                    // Nope, don't recurse
-                    DrawBaselayer::PreviousState => {}
+                    // Don't recurse, but at least clear the screen, because the state is usually
+                    // expecting the previous thing to happen.
+                    DrawBaselayer::PreviousState => {
+                        g.clear(Color::BLACK);
+                    }
                 }
 
                 self.states[self.states.len() - 2].draw(g, &self.shared_app_state);