From 1b4e25e12fa21fab3ea7149e33a29282054730dd Mon Sep 17 00:00:00 2001 From: Dustin Carlino Date: Sat, 7 Nov 2020 18:39:15 -0800 Subject: [PATCH] Get the downloader UI to actually fetch stuff. #326 --- abstutil/src/abst_data.rs | 37 ++++++++- game/Cargo.toml | 2 + game/src/common/city_picker.rs | 9 ++- game/src/common/mod.rs | 1 + game/src/common/updater.rs | 139 ++++++++++++++++++++++++++++++--- updater/src/main.rs | 35 +-------- widgetry/src/app_state.rs | 9 ++- 7 files changed, 181 insertions(+), 51 deletions(-) diff --git a/abstutil/src/abst_data.rs b/abstutil/src/abst_data.rs index b85509786e..daef617f3a 100644 --- a/abstutil/src/abst_data.rs +++ b/abstutil/src/abst_data.rs @@ -32,6 +32,38 @@ impl Manifest { pub fn load() -> Manifest { crate::from_json(&include_bytes!("../../data/MANIFEST.json").to_vec()).unwrap() } + + /// Removes entries from the Manifest to match the DataPacks that should exist locally. + pub fn filter(mut self, data_packs: DataPacks) -> Manifest { + let mut remove = Vec::new(); + for path in self.entries.keys() { + // TODO Some hardcoded weird exceptions + if !data_packs.runtime.contains("huge_seattle") + && path == "data/system/seattle/scenarios/montlake/everyone_weekday.bin" + { + remove.push(path.clone()); + continue; + } + + let parts = path.split("/").collect::>(); + if parts[1] == "input" { + if data_packs.input.contains(parts[2]) { + continue; + } + } else if parts[1] == "system" { + if data_packs.runtime.contains(parts[2]) { + continue; + } + } else { + panic!("Wait what's {}", path); + } + remove.push(path.clone()); + } + for path in remove { + self.entries.remove(&path).unwrap(); + } + self + } } /// Player-chosen groups of files to opt into downloading @@ -45,11 +77,8 @@ pub struct DataPacks { impl DataPacks { /// Load the player's config for what files to download, or create the config. + #[cfg(not(target_arch = "wasm32"))] pub fn load_or_create() -> DataPacks { - if cfg!(target_arch = "wasm32") { - panic!("DataPacks::load_or_create shouldn't be called on wasm"); - } - let path = crate::path("player/data.json"); match crate::maybe_read_json::(path.clone(), &mut Timer::throwaway()) { Ok(mut cfg) => { diff --git a/game/Cargo.toml b/game/Cargo.toml index 3c6e2d2d0d..aee7c3bba3 100644 --- a/game/Cargo.toml +++ b/game/Cargo.toml @@ -17,6 +17,8 @@ osm_viewer = [] wasm = ["console_log", "futures", "futures-channel", "js-sys", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", "widgetry/wasm-backend"] # Just a marker to not use localhost URLs wasm_s3 = [] +# A marker to use a named release from S3 instead of dev for updating files +release_s3 = [] [dependencies] aabb-quadtree = "0.1.0" diff --git a/game/src/common/city_picker.rs b/game/src/common/city_picker.rs index 4df9bd42f6..813200f134 100644 --- a/game/src/common/city_picker.rs +++ b/game/src/common/city_picker.rs @@ -160,7 +160,14 @@ impl State for CityPicker { ); } "Download more cities" => { - return Transition::Replace(crate::common::updater::Picker::new(ctx)); + let _ = "just stop this from counting as an attribute on an expression"; + #[cfg(not(target_arch = "wasm32"))] + { + return Transition::Replace(crate::common::updater::Picker::new( + ctx, + self.on_load.take().unwrap(), + )); + } } path => { return Transition::Replace(MapLoader::new( diff --git a/game/src/common/mod.rs b/game/src/common/mod.rs index eecbd4aa2d..b60776d89f 100644 --- a/game/src/common/mod.rs +++ b/game/src/common/mod.rs @@ -25,6 +25,7 @@ mod heatmap; mod isochrone; mod minimap; mod navigate; +#[cfg(not(target_arch = "wasm32"))] mod updater; mod warp; diff --git a/game/src/common/updater.rs b/game/src/common/updater.rs index b5261a3275..6c31d57c81 100644 --- a/game/src/common/updater.rs +++ b/game/src/common/updater.rs @@ -1,44 +1,82 @@ use std::collections::BTreeMap; +use std::error::Error; +use std::fs::File; -use abstutil::{DataPacks, Manifest}; +use abstutil::{DataPacks, Manifest, Timer}; use widgetry::{Btn, Checkbox, EventCtx, GfxCtx, Line, Outcome, Panel, State, TextExt, Widget}; use crate::app::App; -use crate::game::Transition; +use crate::game::{PopupMsg, Transition}; + +const LATEST_RELEASE: &str = "0.2.17"; pub struct Picker { panel: Panel, + on_load: Option Transition>>, } impl Picker { - pub fn new(ctx: &mut EventCtx) -> Box> { + pub fn new( + ctx: &mut EventCtx, + on_load: Box Transition>, + ) -> Box> { let manifest = Manifest::load(); let data_packs = DataPacks::load_or_create(); - let mut col = vec![Widget::row(vec![ - Line("Download more cities").small_heading().draw(ctx), - Btn::close(ctx), - ])]; + let mut col = vec![ + Widget::row(vec![ + Line("Download more cities").small_heading().draw(ctx), + Btn::close(ctx), + ]), + "Select the cities you want to include".draw_text(ctx), + Line("The file sizes shown are uncompressed; the download size will be smaller") + .secondary() + .draw(ctx), + ]; for (city, bytes) in size_per_city(&manifest) { col.push(Widget::row(vec![ Checkbox::checkbox(ctx, &city, None, data_packs.runtime.contains(&city)), - prettyprint_bytes(bytes).draw_text(ctx), + prettyprint_bytes(bytes).draw_text(ctx).centered_vert(), ])); } + col.push(Btn::text_bg2("Sync files").build_def(ctx, None)); Box::new(Picker { panel: Panel::new(Widget::col(col)).build(ctx), + on_load: Some(on_load), }) } } impl State for Picker { - fn event(&mut self, ctx: &mut EventCtx, _: &mut App) -> Transition { + fn event(&mut self, ctx: &mut EventCtx, app: &mut App) -> Transition { match self.panel.event(ctx) { Outcome::Clicked(x) => match x.as_ref() { "close" => { return Transition::Pop; } + "Sync files" => { + // First update the DataPacks file + let mut data_packs = DataPacks::load_or_create(); + data_packs.runtime.clear(); + data_packs.runtime.insert("seattle".to_string()); + for (city, _) in size_per_city(&Manifest::load()) { + if self.panel.is_checked(&city) { + data_packs.runtime.insert(city); + } + } + abstutil::write_json(abstutil::path("player/data.json"), &data_packs); + + let messages = ctx.loading_screen("sync files", |_, timer| sync(timer)); + return Transition::Multi(vec![ + Transition::Replace(crate::common::CityPicker::new( + ctx, + app, + self.on_load.take().unwrap(), + )), + Transition::Push(PopupMsg::new(ctx, "Download complete", messages)), + ]); + } _ => unreachable!(), }, _ => {} @@ -58,7 +96,13 @@ fn size_per_city(manifest: &Manifest) -> BTreeMap { for (path, entry) in &manifest.entries { let parts = path.split("/").collect::>(); if parts[1] == "system" { - *per_city.entry(parts[2].to_string()).or_insert(0) += entry.size_bytes; + // The map and scenario for huge_seattle should count as a separate data pack. + let city = if parts.get(4) == Some(&"huge_seattle") { + "huge_seattle".to_string() + } else { + parts[2].to_string() + }; + *per_city.entry(city).or_insert(0) += entry.size_bytes; } } per_city @@ -75,3 +119,78 @@ fn prettyprint_bytes(bytes: usize) -> String { let mb = kb / 1024.0; format!("{} mb", mb as usize) } + +// TODO This only downloads files that don't exist but should. It doesn't remove or update +// anything. Not sure if everything the updater does should also be done here. +fn sync(timer: &mut Timer) -> Vec { + let truth = Manifest::load().filter(DataPacks::load_or_create()); + let version = if cfg!(feature = "release_s3") { + LATEST_RELEASE + } else { + "dev" + }; + + let mut files_downloaded = 0; + let mut bytes_downloaded = 0; + let mut messages = Vec::new(); + + timer.start_iter("sync files", truth.entries.len()); + for (path, entry) in truth.entries { + timer.next(); + let local_path = abstutil::path(path.strip_prefix("data/").unwrap()); + if abstutil::file_exists(&local_path) { + continue; + } + let url = format!( + "http://abstreet.s3-website.us-east-2.amazonaws.com/{}/{}.gz", + version, path + ); + timer.note(format!( + "Downloading {} ({} uncompressed)", + url, + prettyprint_bytes(entry.size_bytes) + )); + files_downloaded += 1; + + std::fs::create_dir_all(std::path::Path::new(&local_path).parent().unwrap()).unwrap(); + match download(&url, local_path, timer) { + Ok(bytes) => { + bytes_downloaded += bytes; + } + Err(err) => { + let msg = format!("Problem with {}: {}", url, err); + timer.error(msg.clone()); + messages.push(msg); + } + } + } + messages.insert( + 0, + format!( + "Downloaded {} files, total {}", + files_downloaded, + prettyprint_bytes(bytes_downloaded) + ), + ); + messages +} + +// Bytes downloaded if succesful +fn download(url: &str, local_path: String, timer: &mut Timer) -> Result> { + let mut resp = reqwest::blocking::get(url)?; + if !resp.status().is_success() { + return Err(format!("bad status: {:?}", resp.status()).into()); + } + let mut buffer: Vec = Vec::new(); + let bytes = resp.copy_to(&mut buffer)? as usize; + + timer.note(format!( + "Decompressing {} ({})", + url, + prettyprint_bytes(bytes) + )); + let mut decoder = flate2::read::GzDecoder::new(&buffer[..]); + let mut out = File::create(&local_path).unwrap(); + std::io::copy(&mut decoder, &mut out)?; + Ok(bytes) +} diff --git a/updater/src/main.rs b/updater/src/main.rs index 16961979ff..7c1def2e32 100644 --- a/updater/src/main.rs +++ b/updater/src/main.rs @@ -35,7 +35,7 @@ async fn main() { async fn download() { let data_packs = DataPacks::load_or_create(); let local = generate_manifest(); - let truth = filter_manifest(Manifest::load(), data_packs); + let truth = Manifest::load().filter(data_packs); // Anything local need deleting? for path in local.entries.keys() { @@ -79,7 +79,7 @@ async fn download() { fn just_compare() { let data_packs = DataPacks::load_or_create(); let local = generate_manifest(); - let truth = filter_manifest(Manifest::load(), data_packs); + let truth = Manifest::load().filter(data_packs); // Anything local need deleting? for path in local.entries.keys() { @@ -199,37 +199,6 @@ fn generate_manifest() -> Manifest { Manifest { entries: kv } } -fn filter_manifest(mut manifest: Manifest, data_packs: DataPacks) -> Manifest { - let mut remove = Vec::new(); - for path in manifest.entries.keys() { - // TODO Some hardcoded weird exceptions - if !data_packs.runtime.contains("huge_seattle") - && path == "data/system/seattle/scenarios/montlake/everyone_weekday.bin" - { - remove.push(path.clone()); - continue; - } - - let parts = path.split("/").collect::>(); - if parts[1] == "input" { - if data_packs.input.contains(parts[2]) { - continue; - } - } else if parts[1] == "system" { - if data_packs.runtime.contains(parts[2]) { - continue; - } - } else { - panic!("Wait what's {}", path); - } - remove.push(path.clone()); - } - for path in remove { - manifest.entries.remove(&path).unwrap(); - } - manifest -} - fn must_run_cmd(cmd: &mut Command) { println!("> Running {:?}", cmd); match cmd.status() { diff --git a/widgetry/src/app_state.rs b/widgetry/src/app_state.rs index 9f13acbcab..c00c8d2230 100644 --- a/widgetry/src/app_state.rs +++ b/widgetry/src/app_state.rs @@ -5,7 +5,7 @@ //! screen or menu to choose a map, then a map viewer, then maybe a state to drill down into pieces //! of the map. -use crate::{Canvas, EventCtx, GfxCtx}; +use crate::{Canvas, Color, EventCtx, GfxCtx}; /// Any data that should last the entire lifetime of the application should be stored in the struct /// implementing this trait. @@ -58,8 +58,11 @@ impl App { self.shared_app_state.draw_default(g); } DrawBaselayer::Custom => {} - // Nope, don't recurse - DrawBaselayer::PreviousState => {} + // Don't recurse, but at least clear the screen, because the state is usually + // expecting the previous thing to happen. + DrawBaselayer::PreviousState => { + g.clear(Color::BLACK); + } } self.states[self.states.len() - 2].draw(g, &self.shared_app_state);