From 67268f5461086103e16733c3a628268b60f9374d Mon Sep 17 00:00:00 2001 From: Dustin Carlino Date: Mon, 5 Oct 2020 12:20:52 -0700 Subject: [PATCH] Organize abstutil a little better. Split out modules for generating the data/ paths and for working with serde. --- abstutil/src/abst_paths.rs | 138 ++++++++++++++++++++++++++ abstutil/src/io.rs | 98 +------------------ abstutil/src/lib.rs | 194 +++---------------------------------- abstutil/src/serde.rs | 87 +++++++++++++++++ abstutil/src/time.rs | 16 +-- abstutil/src/utils.rs | 55 +++++++++++ 6 files changed, 294 insertions(+), 294 deletions(-) create mode 100644 abstutil/src/abst_paths.rs create mode 100644 abstutil/src/serde.rs create mode 100644 abstutil/src/utils.rs diff --git a/abstutil/src/abst_paths.rs b/abstutil/src/abst_paths.rs new file mode 100644 index 0000000000..bd16944116 --- /dev/null +++ b/abstutil/src/abst_paths.rs @@ -0,0 +1,138 @@ +// Generate paths for different A/B Street files + +use crate::file_exists; + +lazy_static::lazy_static! { + static ref ROOT_DIR: String = { + // If you're packaging for a release and need the data directory to be in some fixed + // location: ABST_DATA_DIR=/some/path cargo build ... + if let Some(dir) = option_env!("ABST_DATA_DIR") { + dir.trim_end_matches('/').to_string() + } else if cfg!(target_arch = "wasm32") { + "../data".to_string() + } else if file_exists("data/".to_string()) { + "data".to_string() + } else if file_exists("../data/".to_string()) { + "../data".to_string() + } else if file_exists("../../data/".to_string()) { + "../../data".to_string() + } else { + panic!("Can't find the data/ directory"); + } + }; + + static ref ROOT_PLAYER_DIR: String = { + // If you're packaging for a release and want the player's local data directory to be + // $HOME/.abstreet, set ABST_PLAYER_HOME_DIR=1 + if option_env!("ABST_PLAYER_HOME_DIR").is_some() { + match std::env::var("HOME") { + Ok(dir) => format!("{}/.abstreet", dir.trim_end_matches('/')), + Err(err) => panic!("This build of A/B Street stores player data in $HOME/.abstreet, but $HOME isn't set: {}", err), + } + } else if cfg!(target_arch = "wasm32") { + "../data".to_string() + } else if file_exists("data/".to_string()) { + "data".to_string() + } else if file_exists("../data/".to_string()) { + "../data".to_string() + } else if file_exists("../../data/".to_string()) { + "../../data".to_string() + } else { + panic!("Can't find the data/ directory"); + } + }; +} + +pub fn path>(p: I) -> String { + let p = p.into(); + if p.starts_with("player/") { + format!("{}/{}", *ROOT_PLAYER_DIR, p) + } else { + format!("{}/{}", *ROOT_DIR, p) + } +} + +// System data (Players can't edit, needed at runtime) + +pub fn path_map(map_name: &str) -> String { + path(format!("system/maps/{}.bin", map_name)) +} +pub fn path_all_maps() -> String { + path("system/maps") +} + +pub fn path_prebaked_results(map_name: &str, scenario_name: &str) -> String { + path(format!( + "system/prebaked_results/{}/{}.bin", + map_name, scenario_name + )) +} + +pub fn path_scenario(map_name: &str, scenario_name: &str) -> String { + // TODO Getting complicated. Sometimes we're trying to load, so we should look for .bin, then + // .json. But when we're writing a custom scenario, we actually want to write a .bin. + let bin = path(format!( + "system/scenarios/{}/{}.bin", + map_name, scenario_name + )); + let json = path(format!( + "system/scenarios/{}/{}.json", + map_name, scenario_name + )); + if file_exists(&bin) { + return bin; + } + if file_exists(&json) { + return json; + } + bin +} +pub fn path_all_scenarios(map_name: &str) -> String { + path(format!("system/scenarios/{}", map_name)) +} + +pub fn path_synthetic_map(map_name: &str) -> String { + path(format!("system/synthetic_maps/{}.json", map_name)) +} +pub fn path_all_synthetic_maps() -> String { + path("system/synthetic_maps") +} + +// Player data (Players edit this) + +pub fn path_camera_state(map_name: &str) -> String { + path(format!("player/camera_state/{}.json", map_name)) +} + +pub fn path_edits(map_name: &str, edits_name: &str) -> String { + path(format!("player/edits/{}/{}.json", map_name, edits_name)) +} +pub fn path_all_edits(map_name: &str) -> String { + path(format!("player/edits/{}", map_name)) +} + +pub fn path_save(map_name: &str, edits_name: &str, run_name: &str, time: String) -> String { + path(format!( + "player/saves/{}/{}_{}/{}.bin", + map_name, edits_name, run_name, time + )) +} +pub fn path_all_saves(map_name: &str, edits_name: &str, run_name: &str) -> String { + path(format!( + "player/saves/{}/{}_{}", + map_name, edits_name, run_name + )) +} + +// Input data (For developers to build maps, not needed at runtime) + +pub fn path_popdat() -> String { + path("input/seattle/popdat.bin") +} + +pub fn path_raw_map(map_name: &str) -> String { + path(format!("input/raw_maps/{}.bin", map_name)) +} +pub fn path_all_raw_maps() -> String { + path("input/raw_maps") +} diff --git a/abstutil/src/io.rs b/abstutil/src/io.rs index 8b3933ba83..5d9bf5acd5 100644 --- a/abstutil/src/io.rs +++ b/abstutil/src/io.rs @@ -1,14 +1,9 @@ use crate::time::{clear_current_line, prettyprint_time}; -use crate::{elapsed_seconds, prettyprint_usize, MultiMap, Timer, PROGRESS_FREQUENCY_SECONDS}; -use bincode; +use crate::{elapsed_seconds, prettyprint_usize, to_json, Timer, PROGRESS_FREQUENCY_SECONDS}; use instant::Instant; use serde::de::DeserializeOwned; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use serde_json; -use std; -use std::cmp::Ord; +use serde::Serialize; use std::collections::{BTreeMap, BTreeSet}; -use std::convert::TryFrom; use std::fs::File; use std::io::{stdout, BufReader, BufWriter, Error, ErrorKind, Read, Write}; use std::path::Path; @@ -16,10 +11,6 @@ use std::path::Path; #[cfg(target_arch = "wasm32")] static SYSTEM_DATA: include_dir::Dir = include_dir::include_dir!("../data/system"); -pub fn to_json(obj: &T) -> String { - serde_json::to_string_pretty(obj).unwrap() -} - // TODO Idea: Have a wrapper type DotJSON(...) and DotBin(...) to distinguish raw path strings fn maybe_write_json(path: &str, obj: &T) -> Result<(), Error> { if !path.ends_with(".json") { @@ -87,10 +78,6 @@ pub fn read_json(path: String, timer: &mut Timer) -> T { } } -pub fn from_json(raw: &Vec) -> Result { - serde_json::from_slice(raw).map_err(|err| Error::new(ErrorKind::Other, err)) -} - fn maybe_write_binary(path: &str, obj: &T) -> Result<(), Error> { if !path.ends_with(".bin") { panic!("write_binary needs {} to end with .bin", path); @@ -103,10 +90,6 @@ fn maybe_write_binary(path: &str, obj: &T) -> Result<(), Error> { bincode::serialize_into(file, obj).map_err(|err| Error::new(ErrorKind::Other, err)) } -pub fn serialized_size_bytes(obj: &T) -> usize { - bincode::serialized_size(obj).unwrap() as usize -} - #[cfg(not(target_arch = "wasm32"))] pub fn write_binary(path: String, obj: &T) { if let Err(err) = maybe_write_binary(&path, obj) { @@ -164,74 +147,6 @@ pub fn read_object(path: String, timer: &mut Timer) -> Resu } } -// For BTreeMaps with struct keys. See https://github.com/serde-rs/json/issues/402. - -pub fn serialize_btreemap( - map: &BTreeMap, - s: S, -) -> Result { - map.iter().collect::>().serialize(s) -} - -pub fn deserialize_btreemap< - 'de, - D: Deserializer<'de>, - K: Deserialize<'de> + Ord, - V: Deserialize<'de>, ->( - d: D, -) -> Result, D::Error> { - let vec = >::deserialize(d)?; - let mut map = BTreeMap::new(); - for (k, v) in vec { - map.insert(k, v); - } - Ok(map) -} - -pub fn serialize_multimap< - S: Serializer, - K: Serialize + Eq + Ord + Clone, - V: Serialize + Eq + Ord + Clone, ->( - map: &MultiMap, - s: S, -) -> Result { - // TODO maybe need to sort to have deterministic output - map.raw_map().iter().collect::>().serialize(s) -} - -pub fn deserialize_multimap< - 'de, - D: Deserializer<'de>, - K: Deserialize<'de> + Eq + Ord + Clone, - V: Deserialize<'de> + Eq + Ord + Clone, ->( - d: D, -) -> Result, D::Error> { - let vec = )>>::deserialize(d)?; - let mut map = MultiMap::new(); - for (key, values) in vec { - for value in values { - map.insert(key.clone(), value); - } - } - Ok(map) -} - -pub fn serialize_usize(x: &usize, s: S) -> Result { - if let Ok(x) = u32::try_from(*x) { - x.serialize(s) - } else { - Err(serde::ser::Error::custom(format!("{} can't fit in u32", x))) - } -} - -pub fn deserialize_usize<'de, D: Deserializer<'de>>(d: D) -> Result { - let x = ::deserialize(d)?; - Ok(x as usize) -} - // Just list all things from a directory, return sorted by name, with file extension removed. #[cfg(not(target_arch = "wasm32"))] pub fn list_all_objects(dir: String) -> Vec { @@ -436,15 +351,6 @@ pub fn list_dir(dir: &std::path::Path) -> Vec { files } -pub fn basename(path: &str) -> String { - Path::new(path) - .file_stem() - .unwrap() - .to_os_string() - .into_string() - .unwrap() -} - #[cfg(not(target_arch = "wasm32"))] pub fn file_exists>(path: I) -> bool { Path::new(&path.into()).exists() diff --git a/abstutil/src/lib.rs b/abstutil/src/lib.rs index 7d459630dc..5c1430f2a1 100644 --- a/abstutil/src/lib.rs +++ b/abstutil/src/lib.rs @@ -5,193 +5,21 @@ // - A/B Street-specific filesystem paths // - true utility functions (collections, prettyprinting, CLI parsing +mod abst_paths; mod cli; mod collections; mod io; +mod serde; mod time; +mod utils; -pub use crate::cli::CmdArgs; -pub use crate::collections::{ - contains_duplicates, retain_btreemap, retain_btreeset, wraparound_get, Counter, MultiMap, Tags, - VecMap, -}; -pub use crate::io::{ - basename, delete_file, deserialize_btreemap, deserialize_multimap, deserialize_usize, - file_exists, find_next_file, find_prev_file, from_json, list_all_objects, list_dir, - load_all_objects, maybe_read_binary, maybe_read_json, read_binary, read_json, read_object, - serialize_btreemap, serialize_multimap, serialize_usize, serialized_size_bytes, slurp_file, - to_json, write_binary, write_json, FileWithProgress, -}; -pub use crate::time::{ - elapsed_seconds, prettyprint_usize, start_profiler, stop_profiler, Parallelism, Profiler, - Timer, TimerSink, -}; -use std::collections::BTreeSet; -use std::fmt::Write; +// I'm not generally a fan of wildcard exports, but they're more maintable here. +pub use crate::serde::*; +pub use abst_paths::*; +pub use cli::*; +pub use collections::*; +pub use io::*; +pub use time::*; +pub use utils::*; const PROGRESS_FREQUENCY_SECONDS: f64 = 0.2; - -pub fn clamp(x: f64, min: f64, max: f64) -> f64 { - if x < min { - min - } else if x > max { - max - } else { - x - } -} - -pub fn plain_list_names(names: BTreeSet) -> String { - let mut s = String::new(); - let len = names.len(); - for (idx, n) in names.into_iter().enumerate() { - if idx != 0 { - if idx == len - 1 { - if len == 2 { - write!(s, " and ").unwrap(); - } else { - write!(s, ", and ").unwrap(); - } - } else { - write!(s, ", ").unwrap(); - } - } - write!(s, "{}", n).unwrap(); - } - s -} - -lazy_static::lazy_static! { - static ref ROOT_DIR: String = { - // If you're packaging for a release and need the data directory to be in some fixed - // location: ABST_DATA_DIR=/some/path cargo build ... - if let Some(dir) = option_env!("ABST_DATA_DIR") { - dir.trim_end_matches('/').to_string() - } else if cfg!(target_arch = "wasm32") { - "../data".to_string() - } else if file_exists("data/".to_string()) { - "data".to_string() - } else if file_exists("../data/".to_string()) { - "../data".to_string() - } else if file_exists("../../data/".to_string()) { - "../../data".to_string() - } else { - panic!("Can't find the data/ directory"); - } - }; - - static ref ROOT_PLAYER_DIR: String = { - // If you're packaging for a release and want the player's local data directory to be - // $HOME/.abstreet, set ABST_PLAYER_HOME_DIR=1 - if option_env!("ABST_PLAYER_HOME_DIR").is_some() { - match std::env::var("HOME") { - Ok(dir) => format!("{}/.abstreet", dir.trim_end_matches('/')), - Err(err) => panic!("This build of A/B Street stores player data in $HOME/.abstreet, but $HOME isn't set: {}", err), - } - } else if cfg!(target_arch = "wasm32") { - "../data".to_string() - } else if file_exists("data/".to_string()) { - "data".to_string() - } else if file_exists("../data/".to_string()) { - "../data".to_string() - } else if file_exists("../../data/".to_string()) { - "../../data".to_string() - } else { - panic!("Can't find the data/ directory"); - } - }; -} - -pub fn path>(p: I) -> String { - let p = p.into(); - if p.starts_with("player/") { - format!("{}/{}", *ROOT_PLAYER_DIR, p) - } else { - format!("{}/{}", *ROOT_DIR, p) - } -} - -// System data (Players can't edit, needed at runtime) - -pub fn path_map(map_name: &str) -> String { - path(format!("system/maps/{}.bin", map_name)) -} -pub fn path_all_maps() -> String { - path("system/maps") -} - -pub fn path_prebaked_results(map_name: &str, scenario_name: &str) -> String { - path(format!( - "system/prebaked_results/{}/{}.bin", - map_name, scenario_name - )) -} - -pub fn path_scenario(map_name: &str, scenario_name: &str) -> String { - // TODO Getting complicated. Sometimes we're trying to load, so we should look for .bin, then - // .json. But when we're writing a custom scenario, we actually want to write a .bin. - let bin = path(format!( - "system/scenarios/{}/{}.bin", - map_name, scenario_name - )); - let json = path(format!( - "system/scenarios/{}/{}.json", - map_name, scenario_name - )); - if file_exists(&bin) { - return bin; - } - if file_exists(&json) { - return json; - } - bin -} -pub fn path_all_scenarios(map_name: &str) -> String { - path(format!("system/scenarios/{}", map_name)) -} - -pub fn path_synthetic_map(map_name: &str) -> String { - path(format!("system/synthetic_maps/{}.json", map_name)) -} -pub fn path_all_synthetic_maps() -> String { - path("system/synthetic_maps") -} - -// Player data (Players edit this) - -pub fn path_camera_state(map_name: &str) -> String { - path(format!("player/camera_state/{}.json", map_name)) -} - -pub fn path_edits(map_name: &str, edits_name: &str) -> String { - path(format!("player/edits/{}/{}.json", map_name, edits_name)) -} -pub fn path_all_edits(map_name: &str) -> String { - path(format!("player/edits/{}", map_name)) -} - -pub fn path_save(map_name: &str, edits_name: &str, run_name: &str, time: String) -> String { - path(format!( - "player/saves/{}/{}_{}/{}.bin", - map_name, edits_name, run_name, time - )) -} -pub fn path_all_saves(map_name: &str, edits_name: &str, run_name: &str) -> String { - path(format!( - "player/saves/{}/{}_{}", - map_name, edits_name, run_name - )) -} - -// Input data (For developers to build maps, not needed at runtime) - -pub fn path_popdat() -> String { - path("input/seattle/popdat.bin") -} - -pub fn path_raw_map(map_name: &str) -> String { - path(format!("input/raw_maps/{}.bin", map_name)) -} -pub fn path_all_raw_maps() -> String { - path("input/raw_maps") -} diff --git a/abstutil/src/serde.rs b/abstutil/src/serde.rs new file mode 100644 index 0000000000..4220c8dea2 --- /dev/null +++ b/abstutil/src/serde.rs @@ -0,0 +1,87 @@ +use crate::MultiMap; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use std::cmp::Ord; +use std::collections::BTreeMap; +use std::convert::TryFrom; +use std::io::{Error, ErrorKind}; + +pub fn to_json(obj: &T) -> String { + serde_json::to_string_pretty(obj).unwrap() +} + +pub fn from_json(raw: &Vec) -> Result { + serde_json::from_slice(raw).map_err(|err| Error::new(ErrorKind::Other, err)) +} + +pub fn serialized_size_bytes(obj: &T) -> usize { + bincode::serialized_size(obj).unwrap() as usize +} + +// For BTreeMaps with struct keys. See https://github.com/serde-rs/json/issues/402. + +pub fn serialize_btreemap( + map: &BTreeMap, + s: S, +) -> Result { + map.iter().collect::>().serialize(s) +} + +pub fn deserialize_btreemap< + 'de, + D: Deserializer<'de>, + K: Deserialize<'de> + Ord, + V: Deserialize<'de>, +>( + d: D, +) -> Result, D::Error> { + let vec = >::deserialize(d)?; + let mut map = BTreeMap::new(); + for (k, v) in vec { + map.insert(k, v); + } + Ok(map) +} + +pub fn serialize_multimap< + S: Serializer, + K: Serialize + Eq + Ord + Clone, + V: Serialize + Eq + Ord + Clone, +>( + map: &MultiMap, + s: S, +) -> Result { + // TODO maybe need to sort to have deterministic output + map.raw_map().iter().collect::>().serialize(s) +} + +pub fn deserialize_multimap< + 'de, + D: Deserializer<'de>, + K: Deserialize<'de> + Eq + Ord + Clone, + V: Deserialize<'de> + Eq + Ord + Clone, +>( + d: D, +) -> Result, D::Error> { + let vec = )>>::deserialize(d)?; + let mut map = MultiMap::new(); + for (key, values) in vec { + for value in values { + map.insert(key.clone(), value); + } + } + Ok(map) +} + +pub fn serialize_usize(x: &usize, s: S) -> Result { + if let Ok(x) = u32::try_from(*x) { + x.serialize(s) + } else { + Err(serde::ser::Error::custom(format!("{} can't fit in u32", x))) + } +} + +pub fn deserialize_usize<'de, D: Deserializer<'de>>(d: D) -> Result { + let x = ::deserialize(d)?; + Ok(x as usize) +} diff --git a/abstutil/src/time.rs b/abstutil/src/time.rs index eaffbd3f47..e4e9507cbd 100644 --- a/abstutil/src/time.rs +++ b/abstutil/src/time.rs @@ -1,4 +1,4 @@ -use crate::PROGRESS_FREQUENCY_SECONDS; +use crate::{prettyprint_usize, PROGRESS_FREQUENCY_SECONDS}; use instant::Instant; use std::collections::HashMap; use std::fs::File; @@ -560,20 +560,6 @@ impl Profiler { } } -pub fn prettyprint_usize(x: usize) -> String { - let num = format!("{}", x); - let mut result = String::new(); - let mut i = num.len(); - for c in num.chars() { - result.push(c); - i -= 1; - if i > 0 && i % 3 == 0 { - result.push(','); - } - } - result -} - pub fn prettyprint_time(seconds: f64) -> String { format!("{:.4}s", seconds) } diff --git a/abstutil/src/utils.rs b/abstutil/src/utils.rs new file mode 100644 index 0000000000..8b39186fab --- /dev/null +++ b/abstutil/src/utils.rs @@ -0,0 +1,55 @@ +use std::collections::BTreeSet; +use std::fmt::Write; + +pub fn clamp(x: f64, min: f64, max: f64) -> f64 { + if x < min { + min + } else if x > max { + max + } else { + x + } +} + +pub fn plain_list_names(names: BTreeSet) -> String { + let mut s = String::new(); + let len = names.len(); + for (idx, n) in names.into_iter().enumerate() { + if idx != 0 { + if idx == len - 1 { + if len == 2 { + write!(s, " and ").unwrap(); + } else { + write!(s, ", and ").unwrap(); + } + } else { + write!(s, ", ").unwrap(); + } + } + write!(s, "{}", n).unwrap(); + } + s +} + +pub fn prettyprint_usize(x: usize) -> String { + let num = format!("{}", x); + let mut result = String::new(); + let mut i = num.len(); + for c in num.chars() { + result.push(c); + i -= 1; + if i > 0 && i % 3 == 0 { + result.push(','); + } + } + result +} + +pub fn basename(path: &str) -> String { + std::path::Path::new(path) + .file_stem() + .unwrap() + .to_os_string() + .into_string() + .unwrap() +}