Organize abstutil a little better. Split out modules for generating the data/ paths and for working with serde.

This commit is contained in:
Dustin Carlino 2020-10-05 12:20:52 -07:00
parent 178404bd0e
commit 67268f5461
6 changed files with 294 additions and 294 deletions

138
abstutil/src/abst_paths.rs Normal file
View File

@ -0,0 +1,138 @@
// Generate paths for different A/B Street files
use crate::file_exists;
lazy_static::lazy_static! {
static ref ROOT_DIR: String = {
// If you're packaging for a release and need the data directory to be in some fixed
// location: ABST_DATA_DIR=/some/path cargo build ...
if let Some(dir) = option_env!("ABST_DATA_DIR") {
dir.trim_end_matches('/').to_string()
} else if cfg!(target_arch = "wasm32") {
"../data".to_string()
} else if file_exists("data/".to_string()) {
"data".to_string()
} else if file_exists("../data/".to_string()) {
"../data".to_string()
} else if file_exists("../../data/".to_string()) {
"../../data".to_string()
} else {
panic!("Can't find the data/ directory");
}
};
static ref ROOT_PLAYER_DIR: String = {
// If you're packaging for a release and want the player's local data directory to be
// $HOME/.abstreet, set ABST_PLAYER_HOME_DIR=1
if option_env!("ABST_PLAYER_HOME_DIR").is_some() {
match std::env::var("HOME") {
Ok(dir) => format!("{}/.abstreet", dir.trim_end_matches('/')),
Err(err) => panic!("This build of A/B Street stores player data in $HOME/.abstreet, but $HOME isn't set: {}", err),
}
} else if cfg!(target_arch = "wasm32") {
"../data".to_string()
} else if file_exists("data/".to_string()) {
"data".to_string()
} else if file_exists("../data/".to_string()) {
"../data".to_string()
} else if file_exists("../../data/".to_string()) {
"../../data".to_string()
} else {
panic!("Can't find the data/ directory");
}
};
}
pub fn path<I: Into<String>>(p: I) -> String {
let p = p.into();
if p.starts_with("player/") {
format!("{}/{}", *ROOT_PLAYER_DIR, p)
} else {
format!("{}/{}", *ROOT_DIR, p)
}
}
// System data (Players can't edit, needed at runtime)
pub fn path_map(map_name: &str) -> String {
path(format!("system/maps/{}.bin", map_name))
}
pub fn path_all_maps() -> String {
path("system/maps")
}
pub fn path_prebaked_results(map_name: &str, scenario_name: &str) -> String {
path(format!(
"system/prebaked_results/{}/{}.bin",
map_name, scenario_name
))
}
pub fn path_scenario(map_name: &str, scenario_name: &str) -> String {
// TODO Getting complicated. Sometimes we're trying to load, so we should look for .bin, then
// .json. But when we're writing a custom scenario, we actually want to write a .bin.
let bin = path(format!(
"system/scenarios/{}/{}.bin",
map_name, scenario_name
));
let json = path(format!(
"system/scenarios/{}/{}.json",
map_name, scenario_name
));
if file_exists(&bin) {
return bin;
}
if file_exists(&json) {
return json;
}
bin
}
pub fn path_all_scenarios(map_name: &str) -> String {
path(format!("system/scenarios/{}", map_name))
}
pub fn path_synthetic_map(map_name: &str) -> String {
path(format!("system/synthetic_maps/{}.json", map_name))
}
pub fn path_all_synthetic_maps() -> String {
path("system/synthetic_maps")
}
// Player data (Players edit this)
pub fn path_camera_state(map_name: &str) -> String {
path(format!("player/camera_state/{}.json", map_name))
}
pub fn path_edits(map_name: &str, edits_name: &str) -> String {
path(format!("player/edits/{}/{}.json", map_name, edits_name))
}
pub fn path_all_edits(map_name: &str) -> String {
path(format!("player/edits/{}", map_name))
}
pub fn path_save(map_name: &str, edits_name: &str, run_name: &str, time: String) -> String {
path(format!(
"player/saves/{}/{}_{}/{}.bin",
map_name, edits_name, run_name, time
))
}
pub fn path_all_saves(map_name: &str, edits_name: &str, run_name: &str) -> String {
path(format!(
"player/saves/{}/{}_{}",
map_name, edits_name, run_name
))
}
// Input data (For developers to build maps, not needed at runtime)
pub fn path_popdat() -> String {
path("input/seattle/popdat.bin")
}
pub fn path_raw_map(map_name: &str) -> String {
path(format!("input/raw_maps/{}.bin", map_name))
}
pub fn path_all_raw_maps() -> String {
path("input/raw_maps")
}

View File

@ -1,14 +1,9 @@
use crate::time::{clear_current_line, prettyprint_time};
use crate::{elapsed_seconds, prettyprint_usize, MultiMap, Timer, PROGRESS_FREQUENCY_SECONDS};
use bincode;
use crate::{elapsed_seconds, prettyprint_usize, to_json, Timer, PROGRESS_FREQUENCY_SECONDS};
use instant::Instant;
use serde::de::DeserializeOwned;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde_json;
use std;
use std::cmp::Ord;
use serde::Serialize;
use std::collections::{BTreeMap, BTreeSet};
use std::convert::TryFrom;
use std::fs::File;
use std::io::{stdout, BufReader, BufWriter, Error, ErrorKind, Read, Write};
use std::path::Path;
@ -16,10 +11,6 @@ use std::path::Path;
#[cfg(target_arch = "wasm32")]
static SYSTEM_DATA: include_dir::Dir = include_dir::include_dir!("../data/system");
pub fn to_json<T: Serialize>(obj: &T) -> String {
serde_json::to_string_pretty(obj).unwrap()
}
// TODO Idea: Have a wrapper type DotJSON(...) and DotBin(...) to distinguish raw path strings
fn maybe_write_json<T: Serialize>(path: &str, obj: &T) -> Result<(), Error> {
if !path.ends_with(".json") {
@ -87,10 +78,6 @@ pub fn read_json<T: DeserializeOwned>(path: String, timer: &mut Timer) -> T {
}
}
pub fn from_json<T: DeserializeOwned>(raw: &Vec<u8>) -> Result<T, Error> {
serde_json::from_slice(raw).map_err(|err| Error::new(ErrorKind::Other, err))
}
fn maybe_write_binary<T: Serialize>(path: &str, obj: &T) -> Result<(), Error> {
if !path.ends_with(".bin") {
panic!("write_binary needs {} to end with .bin", path);
@ -103,10 +90,6 @@ fn maybe_write_binary<T: Serialize>(path: &str, obj: &T) -> Result<(), Error> {
bincode::serialize_into(file, obj).map_err(|err| Error::new(ErrorKind::Other, err))
}
pub fn serialized_size_bytes<T: Serialize>(obj: &T) -> usize {
bincode::serialized_size(obj).unwrap() as usize
}
#[cfg(not(target_arch = "wasm32"))]
pub fn write_binary<T: Serialize>(path: String, obj: &T) {
if let Err(err) = maybe_write_binary(&path, obj) {
@ -164,74 +147,6 @@ pub fn read_object<T: DeserializeOwned>(path: String, timer: &mut Timer) -> Resu
}
}
// For BTreeMaps with struct keys. See https://github.com/serde-rs/json/issues/402.
pub fn serialize_btreemap<S: Serializer, K: Serialize, V: Serialize>(
map: &BTreeMap<K, V>,
s: S,
) -> Result<S::Ok, S::Error> {
map.iter().collect::<Vec<(_, _)>>().serialize(s)
}
pub fn deserialize_btreemap<
'de,
D: Deserializer<'de>,
K: Deserialize<'de> + Ord,
V: Deserialize<'de>,
>(
d: D,
) -> Result<BTreeMap<K, V>, D::Error> {
let vec = <Vec<(K, V)>>::deserialize(d)?;
let mut map = BTreeMap::new();
for (k, v) in vec {
map.insert(k, v);
}
Ok(map)
}
pub fn serialize_multimap<
S: Serializer,
K: Serialize + Eq + Ord + Clone,
V: Serialize + Eq + Ord + Clone,
>(
map: &MultiMap<K, V>,
s: S,
) -> Result<S::Ok, S::Error> {
// TODO maybe need to sort to have deterministic output
map.raw_map().iter().collect::<Vec<(_, _)>>().serialize(s)
}
pub fn deserialize_multimap<
'de,
D: Deserializer<'de>,
K: Deserialize<'de> + Eq + Ord + Clone,
V: Deserialize<'de> + Eq + Ord + Clone,
>(
d: D,
) -> Result<MultiMap<K, V>, D::Error> {
let vec = <Vec<(K, Vec<V>)>>::deserialize(d)?;
let mut map = MultiMap::new();
for (key, values) in vec {
for value in values {
map.insert(key.clone(), value);
}
}
Ok(map)
}
pub fn serialize_usize<S: Serializer>(x: &usize, s: S) -> Result<S::Ok, S::Error> {
if let Ok(x) = u32::try_from(*x) {
x.serialize(s)
} else {
Err(serde::ser::Error::custom(format!("{} can't fit in u32", x)))
}
}
pub fn deserialize_usize<'de, D: Deserializer<'de>>(d: D) -> Result<usize, D::Error> {
let x = <u32>::deserialize(d)?;
Ok(x as usize)
}
// Just list all things from a directory, return sorted by name, with file extension removed.
#[cfg(not(target_arch = "wasm32"))]
pub fn list_all_objects(dir: String) -> Vec<String> {
@ -436,15 +351,6 @@ pub fn list_dir(dir: &std::path::Path) -> Vec<String> {
files
}
pub fn basename(path: &str) -> String {
Path::new(path)
.file_stem()
.unwrap()
.to_os_string()
.into_string()
.unwrap()
}
#[cfg(not(target_arch = "wasm32"))]
pub fn file_exists<I: Into<String>>(path: I) -> bool {
Path::new(&path.into()).exists()

View File

@ -5,193 +5,21 @@
// - A/B Street-specific filesystem paths
// - true utility functions (collections, prettyprinting, CLI parsing
mod abst_paths;
mod cli;
mod collections;
mod io;
mod serde;
mod time;
mod utils;
pub use crate::cli::CmdArgs;
pub use crate::collections::{
contains_duplicates, retain_btreemap, retain_btreeset, wraparound_get, Counter, MultiMap, Tags,
VecMap,
};
pub use crate::io::{
basename, delete_file, deserialize_btreemap, deserialize_multimap, deserialize_usize,
file_exists, find_next_file, find_prev_file, from_json, list_all_objects, list_dir,
load_all_objects, maybe_read_binary, maybe_read_json, read_binary, read_json, read_object,
serialize_btreemap, serialize_multimap, serialize_usize, serialized_size_bytes, slurp_file,
to_json, write_binary, write_json, FileWithProgress,
};
pub use crate::time::{
elapsed_seconds, prettyprint_usize, start_profiler, stop_profiler, Parallelism, Profiler,
Timer, TimerSink,
};
use std::collections::BTreeSet;
use std::fmt::Write;
// I'm not generally a fan of wildcard exports, but they're more maintable here.
pub use crate::serde::*;
pub use abst_paths::*;
pub use cli::*;
pub use collections::*;
pub use io::*;
pub use time::*;
pub use utils::*;
const PROGRESS_FREQUENCY_SECONDS: f64 = 0.2;
pub fn clamp(x: f64, min: f64, max: f64) -> f64 {
if x < min {
min
} else if x > max {
max
} else {
x
}
}
pub fn plain_list_names(names: BTreeSet<String>) -> String {
let mut s = String::new();
let len = names.len();
for (idx, n) in names.into_iter().enumerate() {
if idx != 0 {
if idx == len - 1 {
if len == 2 {
write!(s, " and ").unwrap();
} else {
write!(s, ", and ").unwrap();
}
} else {
write!(s, ", ").unwrap();
}
}
write!(s, "{}", n).unwrap();
}
s
}
lazy_static::lazy_static! {
static ref ROOT_DIR: String = {
// If you're packaging for a release and need the data directory to be in some fixed
// location: ABST_DATA_DIR=/some/path cargo build ...
if let Some(dir) = option_env!("ABST_DATA_DIR") {
dir.trim_end_matches('/').to_string()
} else if cfg!(target_arch = "wasm32") {
"../data".to_string()
} else if file_exists("data/".to_string()) {
"data".to_string()
} else if file_exists("../data/".to_string()) {
"../data".to_string()
} else if file_exists("../../data/".to_string()) {
"../../data".to_string()
} else {
panic!("Can't find the data/ directory");
}
};
static ref ROOT_PLAYER_DIR: String = {
// If you're packaging for a release and want the player's local data directory to be
// $HOME/.abstreet, set ABST_PLAYER_HOME_DIR=1
if option_env!("ABST_PLAYER_HOME_DIR").is_some() {
match std::env::var("HOME") {
Ok(dir) => format!("{}/.abstreet", dir.trim_end_matches('/')),
Err(err) => panic!("This build of A/B Street stores player data in $HOME/.abstreet, but $HOME isn't set: {}", err),
}
} else if cfg!(target_arch = "wasm32") {
"../data".to_string()
} else if file_exists("data/".to_string()) {
"data".to_string()
} else if file_exists("../data/".to_string()) {
"../data".to_string()
} else if file_exists("../../data/".to_string()) {
"../../data".to_string()
} else {
panic!("Can't find the data/ directory");
}
};
}
pub fn path<I: Into<String>>(p: I) -> String {
let p = p.into();
if p.starts_with("player/") {
format!("{}/{}", *ROOT_PLAYER_DIR, p)
} else {
format!("{}/{}", *ROOT_DIR, p)
}
}
// System data (Players can't edit, needed at runtime)
pub fn path_map(map_name: &str) -> String {
path(format!("system/maps/{}.bin", map_name))
}
pub fn path_all_maps() -> String {
path("system/maps")
}
pub fn path_prebaked_results(map_name: &str, scenario_name: &str) -> String {
path(format!(
"system/prebaked_results/{}/{}.bin",
map_name, scenario_name
))
}
pub fn path_scenario(map_name: &str, scenario_name: &str) -> String {
// TODO Getting complicated. Sometimes we're trying to load, so we should look for .bin, then
// .json. But when we're writing a custom scenario, we actually want to write a .bin.
let bin = path(format!(
"system/scenarios/{}/{}.bin",
map_name, scenario_name
));
let json = path(format!(
"system/scenarios/{}/{}.json",
map_name, scenario_name
));
if file_exists(&bin) {
return bin;
}
if file_exists(&json) {
return json;
}
bin
}
pub fn path_all_scenarios(map_name: &str) -> String {
path(format!("system/scenarios/{}", map_name))
}
pub fn path_synthetic_map(map_name: &str) -> String {
path(format!("system/synthetic_maps/{}.json", map_name))
}
pub fn path_all_synthetic_maps() -> String {
path("system/synthetic_maps")
}
// Player data (Players edit this)
pub fn path_camera_state(map_name: &str) -> String {
path(format!("player/camera_state/{}.json", map_name))
}
pub fn path_edits(map_name: &str, edits_name: &str) -> String {
path(format!("player/edits/{}/{}.json", map_name, edits_name))
}
pub fn path_all_edits(map_name: &str) -> String {
path(format!("player/edits/{}", map_name))
}
pub fn path_save(map_name: &str, edits_name: &str, run_name: &str, time: String) -> String {
path(format!(
"player/saves/{}/{}_{}/{}.bin",
map_name, edits_name, run_name, time
))
}
pub fn path_all_saves(map_name: &str, edits_name: &str, run_name: &str) -> String {
path(format!(
"player/saves/{}/{}_{}",
map_name, edits_name, run_name
))
}
// Input data (For developers to build maps, not needed at runtime)
pub fn path_popdat() -> String {
path("input/seattle/popdat.bin")
}
pub fn path_raw_map(map_name: &str) -> String {
path(format!("input/raw_maps/{}.bin", map_name))
}
pub fn path_all_raw_maps() -> String {
path("input/raw_maps")
}

87
abstutil/src/serde.rs Normal file
View File

@ -0,0 +1,87 @@
use crate::MultiMap;
use serde::de::DeserializeOwned;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use std::cmp::Ord;
use std::collections::BTreeMap;
use std::convert::TryFrom;
use std::io::{Error, ErrorKind};
pub fn to_json<T: Serialize>(obj: &T) -> String {
serde_json::to_string_pretty(obj).unwrap()
}
pub fn from_json<T: DeserializeOwned>(raw: &Vec<u8>) -> Result<T, Error> {
serde_json::from_slice(raw).map_err(|err| Error::new(ErrorKind::Other, err))
}
pub fn serialized_size_bytes<T: Serialize>(obj: &T) -> usize {
bincode::serialized_size(obj).unwrap() as usize
}
// For BTreeMaps with struct keys. See https://github.com/serde-rs/json/issues/402.
pub fn serialize_btreemap<S: Serializer, K: Serialize, V: Serialize>(
map: &BTreeMap<K, V>,
s: S,
) -> Result<S::Ok, S::Error> {
map.iter().collect::<Vec<(_, _)>>().serialize(s)
}
pub fn deserialize_btreemap<
'de,
D: Deserializer<'de>,
K: Deserialize<'de> + Ord,
V: Deserialize<'de>,
>(
d: D,
) -> Result<BTreeMap<K, V>, D::Error> {
let vec = <Vec<(K, V)>>::deserialize(d)?;
let mut map = BTreeMap::new();
for (k, v) in vec {
map.insert(k, v);
}
Ok(map)
}
pub fn serialize_multimap<
S: Serializer,
K: Serialize + Eq + Ord + Clone,
V: Serialize + Eq + Ord + Clone,
>(
map: &MultiMap<K, V>,
s: S,
) -> Result<S::Ok, S::Error> {
// TODO maybe need to sort to have deterministic output
map.raw_map().iter().collect::<Vec<(_, _)>>().serialize(s)
}
pub fn deserialize_multimap<
'de,
D: Deserializer<'de>,
K: Deserialize<'de> + Eq + Ord + Clone,
V: Deserialize<'de> + Eq + Ord + Clone,
>(
d: D,
) -> Result<MultiMap<K, V>, D::Error> {
let vec = <Vec<(K, Vec<V>)>>::deserialize(d)?;
let mut map = MultiMap::new();
for (key, values) in vec {
for value in values {
map.insert(key.clone(), value);
}
}
Ok(map)
}
pub fn serialize_usize<S: Serializer>(x: &usize, s: S) -> Result<S::Ok, S::Error> {
if let Ok(x) = u32::try_from(*x) {
x.serialize(s)
} else {
Err(serde::ser::Error::custom(format!("{} can't fit in u32", x)))
}
}
pub fn deserialize_usize<'de, D: Deserializer<'de>>(d: D) -> Result<usize, D::Error> {
let x = <u32>::deserialize(d)?;
Ok(x as usize)
}

View File

@ -1,4 +1,4 @@
use crate::PROGRESS_FREQUENCY_SECONDS;
use crate::{prettyprint_usize, PROGRESS_FREQUENCY_SECONDS};
use instant::Instant;
use std::collections::HashMap;
use std::fs::File;
@ -560,20 +560,6 @@ impl Profiler {
}
}
pub fn prettyprint_usize(x: usize) -> String {
let num = format!("{}", x);
let mut result = String::new();
let mut i = num.len();
for c in num.chars() {
result.push(c);
i -= 1;
if i > 0 && i % 3 == 0 {
result.push(',');
}
}
result
}
pub fn prettyprint_time(seconds: f64) -> String {
format!("{:.4}s", seconds)
}

55
abstutil/src/utils.rs Normal file
View File

@ -0,0 +1,55 @@
use std::collections::BTreeSet;
use std::fmt::Write;
pub fn clamp(x: f64, min: f64, max: f64) -> f64 {
if x < min {
min
} else if x > max {
max
} else {
x
}
}
pub fn plain_list_names(names: BTreeSet<String>) -> String {
let mut s = String::new();
let len = names.len();
for (idx, n) in names.into_iter().enumerate() {
if idx != 0 {
if idx == len - 1 {
if len == 2 {
write!(s, " and ").unwrap();
} else {
write!(s, ", and ").unwrap();
}
} else {
write!(s, ", ").unwrap();
}
}
write!(s, "{}", n).unwrap();
}
s
}
pub fn prettyprint_usize(x: usize) -> String {
let num = format!("{}", x);
let mut result = String::new();
let mut i = num.len();
for c in num.chars() {
result.push(c);
i -= 1;
if i > 0 && i % 3 == 0 {
result.push(',');
}
}
result
}
pub fn basename(path: &str) -> String {
std::path::Path::new(path)
.file_stem()
.unwrap()
.to_os_string()
.into_string()
.unwrap()
}