From 72c68aa32073a40f71f6159986b759c6e95edc23 Mon Sep 17 00:00:00 2001 From: Dustin Carlino Date: Tue, 3 Dec 2019 11:42:02 -0800 Subject: [PATCH] remove old census popdat stuff --- docs/onboarding.md | 3 +- game/src/mission/dataviz.rs | 307 ------------------------------------ game/src/mission/mod.rs | 4 - import.sh | 18 --- kml/Cargo.toml | 1 - kml/src/lib.rs | 45 +----- popdat/src/lib.rs | 162 ------------------- popdat/src/main.rs | 5 +- precompute.sh | 32 ++-- 9 files changed, 21 insertions(+), 556 deletions(-) delete mode 100644 game/src/mission/dataviz.rs diff --git a/docs/onboarding.md b/docs/onboarding.md index e73ea90a26..630e2a7609 100644 --- a/docs/onboarding.md +++ b/docs/onboarding.md @@ -19,8 +19,7 @@ Constructing the map: intermediate map format into the final format - `precompute`: small tool to run the second stage of map conversion and write final output -- `popdat`: importing extra census-based data specific to Seattle, optional - right now +- `popdat`: importing daily trips from PSRC's Soundcast model, specific to Seattle - `map_editor`: GUI for modifying geometry of maps and creating maps from scratch diff --git a/game/src/mission/dataviz.rs b/game/src/mission/dataviz.rs deleted file mode 100644 index 03a288837e..0000000000 --- a/game/src/mission/dataviz.rs +++ /dev/null @@ -1,307 +0,0 @@ -use crate::common::CommonState; -use crate::game::{State, Transition}; -use crate::helpers::{rotating_color_total, ID}; -use crate::ui::UI; -use abstutil::{prettyprint_usize, Timer}; -use ezgui::{ - hotkey, Color, EventCtx, GfxCtx, HorizontalAlignment, Key, Line, ModalMenu, Text, - VerticalAlignment, -}; -use geom::{Distance, Polygon, Pt2D}; -use popdat::{Estimate, PopDat}; -use std::collections::BTreeMap; - -pub struct DataVisualizer { - menu: ModalMenu, - popdat: PopDat, - tracts: BTreeMap, - - // Table if false - show_bars: bool, - // TODO Urgh. 0, 1, or 2. - current_dataset: usize, - current_tract: Option, -} - -struct Tract { - polygon: Polygon, - color: Color, - - num_bldgs: usize, - num_parking_spots: usize, - total_owned_cars: usize, -} - -impl DataVisualizer { - pub fn new(ctx: &mut EventCtx, ui: &UI) -> DataVisualizer { - let (popdat, tracts) = ctx.loading_screen("initialize popdat", |_, mut timer| { - let popdat: PopDat = abstutil::read_binary(abstutil::path_popdat(), &mut timer); - let tracts = clip_tracts(&popdat, ui, &mut timer); - (popdat, tracts) - }); - - DataVisualizer { - menu: ModalMenu::new( - "Data Visualizer", - vec![ - (hotkey(Key::Escape), "quit"), - (hotkey(Key::Space), "toggle table/bar chart"), - (hotkey(Key::Num1), "household vehicles"), - (hotkey(Key::Num2), "commute times"), - (hotkey(Key::Num3), "commute modes"), - ], - ctx, - ), - tracts, - popdat, - show_bars: false, - current_dataset: 0, - current_tract: None, - } - } -} -impl State for DataVisualizer { - fn event(&mut self, ctx: &mut EventCtx, ui: &mut UI) -> Transition { - { - let mut txt = Text::new(); - if let Some(ref name) = self.current_tract { - txt.add_appended(vec![ - Line("Census "), - Line(name).fg(ui.cs.get("OSD name color")), - ]); - let tract = &self.tracts[name]; - txt.add(Line(format!( - "{} buildings", - prettyprint_usize(tract.num_bldgs) - ))); - txt.add(Line(format!( - "{} parking spots ", - prettyprint_usize(tract.num_parking_spots) - ))); - txt.add(Line(format!( - "{} total owned cars", - prettyprint_usize(tract.total_owned_cars) - ))); - } - self.menu.set_info(ctx, txt); - } - self.menu.event(ctx); - ctx.canvas.handle_event(ctx.input); - - // TODO Remember which dataset we're showing and don't allow reseting to the same. - if self.menu.action("quit") { - return Transition::Pop; - } else if self.current_dataset != 0 && self.menu.action("household vehicles") { - self.current_dataset = 0; - } else if self.current_dataset != 1 && self.menu.action("commute times") { - self.current_dataset = 1; - } else if self.current_dataset != 2 && self.menu.action("commute modes") { - self.current_dataset = 2; - } else if self.menu.action("toggle table/bar chart") { - self.show_bars = !self.show_bars; - } - - if ctx.redo_mouseover() { - self.current_tract = None; - if let Some(pt) = ctx.canvas.get_cursor_in_map_space() { - for (name, tract) in &self.tracts { - if tract.polygon.contains_pt(pt) { - self.current_tract = Some(name.clone()); - break; - } - } - } - } - - Transition::Keep - } - - fn draw(&self, g: &mut GfxCtx, ui: &UI) { - for (name, tract) in &self.tracts { - let color = if Some(name.clone()) == self.current_tract { - ui.cs.get("selected") - } else { - tract.color - }; - g.draw_polygon(color, &tract.polygon); - } - - self.menu.draw(g); - if let Some(ref name) = self.current_tract { - let mut osd = Text::new(); - osd.add_appended(vec![ - Line("Census "), - Line(name).fg(ui.cs.get("OSD name color")), - ]); - CommonState::draw_custom_osd(g, osd); - } else { - CommonState::draw_osd(g, ui, &None); - } - - if let Some(ref name) = self.current_tract { - let tract = &self.popdat.tracts[name]; - let kv = if self.current_dataset == 0 { - &tract.household_vehicles - } else if self.current_dataset == 1 { - &tract.commute_times - } else if self.current_dataset == 2 { - &tract.commute_modes - } else { - unreachable!() - }; - - if self.show_bars { - bar_chart(g, kv); - } else { - let mut txt = Text::new(); - for (k, v) in kv { - txt.add_appended(vec![ - Line(k).fg(Color::RED), - Line(" = "), - Line(v.to_string()).fg(Color::CYAN), - ]); - } - g.draw_blocking_text(&txt, (HorizontalAlignment::Left, VerticalAlignment::Top)); - } - } - } -} - -fn clip_tracts(popdat: &PopDat, ui: &UI, timer: &mut Timer) -> BTreeMap { - // TODO Partial clipping could be neat, except it'd be confusing to interpret totals. - let mut results = BTreeMap::new(); - timer.start_iter("clip tracts", popdat.tracts.len()); - for (name, tract) in &popdat.tracts { - timer.next(); - if let Some(pts) = ui.primary.map.get_gps_bounds().try_convert(&tract.pts) { - // TODO We should actually make sure the polygon is completely contained within the - // map's boundary. - let polygon = Polygon::new(&pts); - - // TODO Don't just use the center... - let mut num_bldgs = 0; - let mut num_parking_spots = 0; - for id in ui - .primary - .draw_map - .get_matching_objects(polygon.get_bounds()) - { - match id { - ID::Building(b) => { - if polygon.contains_pt(ui.primary.map.get_b(b).polygon.center()) { - num_bldgs += 1; - } - } - ID::Lane(l) => { - let lane = ui.primary.map.get_l(l); - if lane.is_parking() && polygon.contains_pt(lane.lane_center_pts.middle()) { - num_parking_spots += lane.number_parking_spots(); - } - } - _ => {} - } - } - - results.insert( - name.clone(), - Tract { - polygon, - // Update it after we know the total number of matching tracts. - color: Color::WHITE, - num_bldgs, - num_parking_spots, - total_owned_cars: tract.total_owned_cars(), - }, - ); - } - } - let len = results.len(); - for (idx, tract) in results.values_mut().enumerate() { - tract.color = rotating_color_total(idx, len); - } - println!( - "Clipped {} tracts from {}", - results.len(), - popdat.tracts.len() - ); - results -} - -fn bar_chart(g: &mut GfxCtx, data: &BTreeMap) { - let mut max = 0; - let mut sum = 0; - for (name, est) in data { - if name == "Total:" { - continue; - } - max = max.max(est.value); - sum += est.value; - } - - let mut labels = Text::new().no_bg(); - for (name, est) in data { - if name == "Total:" { - continue; - } - labels.add_appended(vec![ - Line(format!("{} (", name)).size(40), - Line(format!( - "{}%", - ((est.value as f64) / (sum as f64) * 100.0) as usize - )) - .fg(Color::RED), - Line(")"), - ]); - } - let txt_dims = g.text_dims(&labels); - let line_height = txt_dims.height / ((data.len() as f64) - 1.0); - labels.add(Line(format!("{} samples", prettyprint_usize(sum))).size(40)); - - // This is, uh, pixels. :P - let max_bar_width = 300.0; - - g.fork_screenspace(); - g.draw_polygon( - Color::grey(0.3), - &Polygon::rectangle_topleft( - Pt2D::new(0.0, 0.0), - Distance::meters(txt_dims.width + 1.2 * max_bar_width), - Distance::meters(txt_dims.height + line_height), - ), - ); - g.draw_blocking_text(&labels, (HorizontalAlignment::Left, VerticalAlignment::Top)); - // draw_blocking_text undoes this! Oops. - g.fork_screenspace(); - - for (idx, (name, est)) in data.iter().enumerate() { - if name == "Total:" { - continue; - } - let this_width = max_bar_width * ((est.value as f64) / (max as f64)); - g.draw_polygon( - rotating_color_total(idx, data.len() - 1), - &Polygon::rectangle_topleft( - Pt2D::new(txt_dims.width, (0.1 + (idx as f64)) * line_height), - Distance::meters(this_width), - Distance::meters(0.8 * line_height), - ), - ); - - // Error bars! - // TODO Little cap on both sides - let half_moe_width = max_bar_width * (est.moe as f64) / (max as f64) / 2.0; - g.draw_polygon( - Color::BLACK, - &Polygon::rectangle_topleft( - Pt2D::new( - txt_dims.width + this_width - half_moe_width, - (0.4 + (idx as f64)) * line_height, - ), - 2.0 * Distance::meters(half_moe_width), - 0.2 * Distance::meters(line_height), - ), - ); - } - - g.unfork(); -} diff --git a/game/src/mission/mod.rs b/game/src/mission/mod.rs index 3eb8fde46e..83729d6870 100644 --- a/game/src/mission/mod.rs +++ b/game/src/mission/mod.rs @@ -1,5 +1,4 @@ mod all_trips; -mod dataviz; mod individ_trips; mod neighborhood; mod scenario; @@ -21,7 +20,6 @@ impl MissionEditMode { menu: ModalMenu::new( "Mission Edit Mode", vec![ - (hotkey(Key::D), "visualize population data"), (hotkey(Key::T), "visualize individual PSRC trips"), (hotkey(Key::A), "visualize all PSRC trips"), (hotkey(Key::N), "manage neighborhoods"), @@ -42,8 +40,6 @@ impl State for MissionEditMode { if self.menu.action("quit") { return Transition::Pop; - } else if self.menu.action("visualize population data") { - return Transition::Push(Box::new(dataviz::DataVisualizer::new(ctx, ui))); } else if self.menu.action("visualize individual PSRC trips") { return Transition::Push(Box::new(individ_trips::TripsVisualizer::new(ctx, ui))); } else if self.menu.action("visualize all PSRC trips") { diff --git a/import.sh b/import.sh index 5904284786..a0bc108e23 100755 --- a/import.sh +++ b/import.sh @@ -86,24 +86,6 @@ if [ ! -f data/shapes/sidewalks.bin ]; then cd .. fi -if [ ! -f data/input/household_vehicles.kml ]; then - # From https://gis-kingcounty.opendata.arcgis.com/datasets/acs-household-size-by-vehicles-available-acs-b08201-householdvehicles - get_if_needed https://opendata.arcgis.com/datasets/7842d815523c4f1b9564e0301e2eafa4_2372.kml data/input/household_vehicles.kml; - get_if_needed https://www.arcgis.com/sharing/rest/content/items/7842d815523c4f1b9564e0301e2eafa4/info/metadata/metadata.xml data/input/household_vehicles.xml; -fi - -if [ ! -f data/input/commute_time.kml ]; then - # From https://gis-kingcounty.opendata.arcgis.com/datasets/acs-travel-time-to-work-acs-b08303-traveltime - get_if_needed https://opendata.arcgis.com/datasets/9b5fd85861a04c5ab8b7407c7b58da7c_2375.kml data/input/commute_time.kml; - get_if_needed https://www.arcgis.com/sharing/rest/content/items/9b5fd85861a04c5ab8b7407c7b58da7c/info/metadata/metadata.xml data/input/commute_time.xml; -fi - -if [ ! -f data/input/commute_mode.kml ]; then - # From https://gis-kingcounty.opendata.arcgis.com/datasets/acs-means-of-transportation-to-work-acs-b08301-transportation - get_if_needed https://opendata.arcgis.com/datasets/1da9717ca5ff4505826aba40a7ac0a58_2374.kml data/input/commute_mode.kml; - get_if_needed https://www.arcgis.com/sharing/rest/content/items/1da9717ca5ff4505826aba40a7ac0a58/info/metadata/metadata.xml data/input/commute_mode.xml; -fi - if [ ! -f data/input/offstreet_parking.kml ]; then # From https://data.seattle.gov/Transportation/Public-Garages-or-Parking-Lots/xefx-khzm get_if_needed http://data-seattlecitygis.opendata.arcgis.com/datasets/8e52dfde6d5d45948f7a90654c8d50cd_0.kml data/input/offstreet_parking.kml; diff --git a/kml/Cargo.toml b/kml/Cargo.toml index 9f144b5286..c04eb9e63c 100644 --- a/kml/Cargo.toml +++ b/kml/Cargo.toml @@ -10,4 +10,3 @@ geom = { path = "../geom" } quick-xml = "0.13.3" serde = "1.0.98" serde_derive = "1.0.98" -xmltree = "0.8.0" diff --git a/kml/src/lib.rs b/kml/src/lib.rs index 22e8ab3ad1..4be5629c2f 100644 --- a/kml/src/lib.rs +++ b/kml/src/lib.rs @@ -4,8 +4,6 @@ use quick_xml::events::Event; use quick_xml::Reader; use serde_derive::{Deserialize, Serialize}; use std::collections::BTreeMap; -use std::{fs, io}; -use xmltree::Element; #[derive(Serialize, Deserialize)] pub struct ExtraShapes { @@ -22,11 +20,11 @@ pub fn load( path: &str, gps_bounds: &GPSBounds, timer: &mut Timer, -) -> Result { +) -> Result { println!("Opening {}", path); let (f, done) = FileWithProgress::new(path)?; // TODO FileWithProgress should implement BufRead, so we don't have to double wrap like this - let mut reader = Reader::from_reader(io::BufReader::new(f)); + let mut reader = Reader::from_reader(std::io::BufReader::new(f)); reader.trim_text(true); let mut buf = Vec::new(); @@ -102,11 +100,7 @@ pub fn load( ); done(timer); - let mut shapes = ExtraShapes { shapes }; - if fix_field_names(path, &mut shapes).is_none() { - timer.warn(format!("Applying extra XML metadata for {} failed", path)); - } - Ok(shapes) + Ok(ExtraShapes { shapes }) } fn parse_pt(input: &str, gps_bounds: &GPSBounds) -> Option { @@ -124,36 +118,3 @@ fn parse_pt(input: &str, gps_bounds: &GPSBounds) -> Option { None } } - -fn fix_field_names(orig_path: &str, shapes: &mut ExtraShapes) -> Option<()> { - let new_path = orig_path.replace(".kml", ".xml"); - if !std::path::Path::new(&new_path).exists() { - return None; - } - println!("Loading extra metadata from {}", new_path); - let root = Element::parse(fs::read_to_string(new_path).ok()?.as_bytes()).ok()?; - - let mut rename = BTreeMap::new(); - for attr in &root.get_child("eainfo")?.get_child("detailed")?.children { - if attr.name != "attr" { - continue; - } - let key = attr.get_child("attrlabl")?.text.clone()?; - let value = attr.get_child("attrdef")?.text.clone()?; - rename.insert(key, value); - } - - for shp in shapes.shapes.iter_mut() { - let mut attribs = BTreeMap::new(); - for (k, v) in &shp.attributes { - if let Some(new_key) = rename.get(k) { - attribs.insert(new_key.clone(), v.clone()); - } else { - attribs.insert(k.clone(), v.clone()); - } - } - shp.attributes = attribs; - } - - Some(()) -} diff --git a/popdat/src/lib.rs b/popdat/src/lib.rs index 3f35f3ee81..7ae1f1fd80 100644 --- a/popdat/src/lib.rs +++ b/popdat/src/lib.rs @@ -1,174 +1,12 @@ pub mod psrc; mod trips; -use abstutil::Timer; -use geom::{GPSBounds, LonLat}; use serde_derive::{Deserialize, Serialize}; use std::collections::BTreeMap; -use std::fmt; pub use trips::{clip_trips, trips_to_scenario, Trip, TripEndpt}; #[derive(Serialize, Deserialize)] pub struct PopDat { - // Keyed by census tract label - // Invariant: Every tract has all data filled out. - pub tracts: BTreeMap, - pub trips: Vec, pub parcels: BTreeMap, } - -#[derive(Serialize, Deserialize)] -pub struct TractData { - pub pts: Vec, - pub household_vehicles: BTreeMap, - pub commute_times: BTreeMap, - pub commute_modes: BTreeMap, -} - -#[derive(Serialize, Deserialize)] -pub struct Estimate { - pub value: usize, - // margin of error, 90% confidence - pub moe: usize, -} - -impl PopDat { - pub fn import_all(timer: &mut Timer) -> PopDat { - let mut dat = PopDat { - tracts: BTreeMap::new(), - trips: Vec::new(), - parcels: BTreeMap::new(), - }; - let fields: Vec<( - &str, - Box)>, - )> = vec![ - ( - "../data/input/household_vehicles.kml", - Box::new(|tract, map| { - tract.household_vehicles = map; - }), - ), - ( - "../data/input/commute_time.kml", - Box::new(|tract, map| { - tract.commute_times = map; - }), - ), - ( - "../data/input/commute_mode.kml", - Box::new(|tract, map| { - tract.commute_modes = map; - }), - ), - ]; - for (path, setter) in fields { - for mut shape in kml::load(path, &GPSBounds::seattle_bounds(), timer) - .expect(&format!("couldn't load {}", path)) - .shapes - { - let name = shape.attributes.remove("TRACT_LBL").unwrap(); - - if let Some(ref tract) = dat.tracts.get(&name) { - assert_eq!(shape.points, tract.pts); - } else { - dat.tracts.insert( - name.clone(), - TractData { - pts: shape.points, - household_vehicles: BTreeMap::new(), - commute_times: BTreeMap::new(), - commute_modes: BTreeMap::new(), - }, - ); - } - - setter( - dat.tracts.get_mut(&name).unwrap(), - group_attribs(shape.attributes), - ); - } - } - - for (name, tract) in &dat.tracts { - if tract.household_vehicles.is_empty() - || tract.commute_times.is_empty() - || tract.commute_modes.is_empty() - { - panic!("{} is missing data", name); - } - } - - dat - } -} - -impl fmt::Display for Estimate { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{} ± {}", self.value, self.moe) - } -} - -fn group_attribs(mut attribs: BTreeMap) -> BTreeMap { - // Remove useless stuff - attribs.remove("Internal feature number."); - attribs.remove("GEO_ID_TRT"); - - let mut estimates = BTreeMap::new(); - let mut moes = BTreeMap::new(); - for (k, v) in attribs { - // These fields in the household_vehicles dataset aren't interesting. - if k.contains("person hsehold") { - continue; - } - - let value = v - .parse::() - .unwrap_or_else(|_| panic!("Unknown value {}={}", k, v)); - - if k.starts_with("E1216 - ") { - estimates.insert(k["E1216 - ".len()..k.len()].to_string(), value); - } else if k.starts_with("M121616 - ") { - moes.insert(k["M121616 - ".len()..k.len()].to_string(), value); - } else { - panic!("Unknown key {}={}", k, v); - } - } - - // If the length is the same but some keys differ, the lookup in moes below will blow up. - if estimates.len() != moes.len() { - panic!("estimates and margins of error have different keys, probably"); - } - estimates - .into_iter() - .map(|(key, e)| { - ( - key.clone(), - Estimate { - value: e, - moe: moes[&key], - }, - ) - }) - .collect() -} - -impl TractData { - // Nontrivial summary - pub fn total_owned_cars(&self) -> usize { - let mut sum = 0; - for (name, est) in &self.household_vehicles { - match name.as_str() { - "1 vehicle avail." => sum += est.value, - "2 vehicles avail." => sum += 2 * est.value, - "3 vehicles avail." => sum += 3 * est.value, - // Many more than 4 seems unrealistic - "4 or more vehicles avail." => sum += 4 * est.value, - "No vehicle avail." | "Total:" => {} - _ => panic!("Unknown household_vehicles key {}", name), - } - } - sum - } -} diff --git a/popdat/src/main.rs b/popdat/src/main.rs index 5484794aed..a0fa18b89e 100644 --- a/popdat/src/main.rs +++ b/popdat/src/main.rs @@ -1,14 +1,11 @@ fn main() { let mut timer = abstutil::Timer::new("creating popdat"); - let mut popdat = popdat::PopDat::import_all(&mut timer); - let (trips, parcels) = popdat::psrc::import_trips( "../data/input/parcels_urbansim.txt", "../data/input/trips_2014.csv", &mut timer, ) .unwrap(); - popdat.trips = trips; - popdat.parcels = parcels; + let popdat = popdat::PopDat { trips, parcels }; abstutil::write_binary(abstutil::path_popdat(), &popdat); } diff --git a/precompute.sh b/precompute.sh index 625ba73780..6398a4152e 100755 --- a/precompute.sh +++ b/precompute.sh @@ -2,6 +2,22 @@ set -e +mkdir -p data/maps/ + +# Need this first +if [ ! -f data/shapes/popdat.bin ]; then + # We probably don't have this map yet. + if [ ! -f data/maps/huge_seattle.bin ]; then + cd precompute; + RUST_BACKTRACE=1 cargo run --release ../data/raw_maps/huge_seattle.bin --disable_psrc_scenarios; + cd ..; + fi + + cd popdat; + cargo run --release; + cd ..; +fi + release_mode="" psrc_scenarios="" no_fixes="" @@ -21,22 +37,6 @@ for arg in "$@"; do fi done -mkdir -p data/maps/ - -# Need this first -if [ ! -f data/shapes/popdat.bin ]; then - # We probably don't have this map yet. - if [ ! -f data/maps/huge_seattle.bin ]; then - cd precompute; - RUST_BACKTRACE=1 cargo run --release ../data/raw_maps/huge_seattle.bin --disable_psrc_scenarios; - cd ..; - fi - - cd popdat; - cargo run --release; - cd ..; -fi - for map_path in `ls data/raw_maps/`; do map=`basename $map_path .bin`; echo "Precomputing $map";