a-b-street/abstreet
1
1
Fork 0
mirror of https://github.com/a-b-street/abstreet.git synced 2024-12-24 23:15:24 +03:00
Code Issues Projects Releases Wiki Activity

remove old census popdat stuff

Browse Source
This commit is contained in:
Dustin Carlino 2019-12-03 11:42:02 -08:00
parent 6695f72847
commit 72c68aa320
9 changed files with 21 additions and 556 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
docs/onboarding.md
View File

@ -19,8 +19,7 @@ Constructing the map:
intermediate map format into the final format
- `precompute`: small tool to run the second stage of map conversion and write
final output
- `popdat`: importing extra census-based data specific to Seattle, optional
right now
- `popdat`: importing daily trips from PSRC's Soundcast model, specific to Seattle
- `map_editor`: GUI for modifying geometry of maps and creating maps from
scratch

307
game/src/mission/dataviz.rs
View File

@ -1,307 +0,0 @@
use crate::common::CommonState;
use crate::game::{State, Transition};
use crate::helpers::{rotating_color_total, ID};
use crate::ui::UI;
use abstutil::{prettyprint_usize, Timer};
use ezgui::{
hotkey, Color, EventCtx, GfxCtx, HorizontalAlignment, Key, Line, ModalMenu, Text,
VerticalAlignment,
};
use geom::{Distance, Polygon, Pt2D};
use popdat::{Estimate, PopDat};
use std::collections::BTreeMap;
pub struct DataVisualizer {
menu: ModalMenu,
popdat: PopDat,
tracts: BTreeMap<String, Tract>,
// Table if false
show_bars: bool,
// TODO Urgh. 0, 1, or 2.
current_dataset: usize,
current_tract: Option<String>,
}
struct Tract {
polygon: Polygon,
color: Color,
num_bldgs: usize,
num_parking_spots: usize,
total_owned_cars: usize,
}
impl DataVisualizer {
pub fn new(ctx: &mut EventCtx, ui: &UI) -> DataVisualizer {
let (popdat, tracts) = ctx.loading_screen("initialize popdat", |_, mut timer| {
let popdat: PopDat = abstutil::read_binary(abstutil::path_popdat(), &mut timer);
let tracts = clip_tracts(&popdat, ui, &mut timer);
(popdat, tracts)
});
DataVisualizer {
menu: ModalMenu::new(
"Data Visualizer",
vec![
(hotkey(Key::Escape), "quit"),
(hotkey(Key::Space), "toggle table/bar chart"),
(hotkey(Key::Num1), "household vehicles"),
(hotkey(Key::Num2), "commute times"),
(hotkey(Key::Num3), "commute modes"),
],
ctx,
),
tracts,
popdat,
show_bars: false,
current_dataset: 0,
current_tract: None,
}
}
}
impl State for DataVisualizer {
fn event(&mut self, ctx: &mut EventCtx, ui: &mut UI) -> Transition {
{
let mut txt = Text::new();
if let Some(ref name) = self.current_tract {
txt.add_appended(vec![
Line("Census "),
Line(name).fg(ui.cs.get("OSD name color")),
]);
let tract = &self.tracts[name];
txt.add(Line(format!(
"{} buildings",
prettyprint_usize(tract.num_bldgs)
)));
txt.add(Line(format!(
"{} parking spots ",
prettyprint_usize(tract.num_parking_spots)
)));
txt.add(Line(format!(
"{} total owned cars",
prettyprint_usize(tract.total_owned_cars)
)));
}
self.menu.set_info(ctx, txt);
}
self.menu.event(ctx);
ctx.canvas.handle_event(ctx.input);
// TODO Remember which dataset we're showing and don't allow reseting to the same.
if self.menu.action("quit") {
return Transition::Pop;
} else if self.current_dataset != 0 && self.menu.action("household vehicles") {
self.current_dataset = 0;
} else if self.current_dataset != 1 && self.menu.action("commute times") {
self.current_dataset = 1;
} else if self.current_dataset != 2 && self.menu.action("commute modes") {
self.current_dataset = 2;
} else if self.menu.action("toggle table/bar chart") {
self.show_bars = !self.show_bars;
}
if ctx.redo_mouseover() {
self.current_tract = None;
if let Some(pt) = ctx.canvas.get_cursor_in_map_space() {
for (name, tract) in &self.tracts {
if tract.polygon.contains_pt(pt) {
self.current_tract = Some(name.clone());
break;
}
}
}
}
Transition::Keep
}
fn draw(&self, g: &mut GfxCtx, ui: &UI) {
for (name, tract) in &self.tracts {
let color = if Some(name.clone()) == self.current_tract {
ui.cs.get("selected")
} else {
tract.color
};
g.draw_polygon(color, &tract.polygon);
}
self.menu.draw(g);
if let Some(ref name) = self.current_tract {
let mut osd = Text::new();
osd.add_appended(vec![
Line("Census "),
Line(name).fg(ui.cs.get("OSD name color")),
]);
CommonState::draw_custom_osd(g, osd);
} else {
CommonState::draw_osd(g, ui, &None);
}
if let Some(ref name) = self.current_tract {
let tract = &self.popdat.tracts[name];
let kv = if self.current_dataset == 0 {
&tract.household_vehicles
} else if self.current_dataset == 1 {
&tract.commute_times
} else if self.current_dataset == 2 {
&tract.commute_modes
} else {
unreachable!()
};
if self.show_bars {
bar_chart(g, kv);
} else {
let mut txt = Text::new();
for (k, v) in kv {
txt.add_appended(vec![
Line(k).fg(Color::RED),
Line(" = "),
Line(v.to_string()).fg(Color::CYAN),
]);
}
g.draw_blocking_text(&txt, (HorizontalAlignment::Left, VerticalAlignment::Top));
}
}
}
}
fn clip_tracts(popdat: &PopDat, ui: &UI, timer: &mut Timer) -> BTreeMap<String, Tract> {
// TODO Partial clipping could be neat, except it'd be confusing to interpret totals.
let mut results = BTreeMap::new();
timer.start_iter("clip tracts", popdat.tracts.len());
for (name, tract) in &popdat.tracts {
timer.next();
if let Some(pts) = ui.primary.map.get_gps_bounds().try_convert(&tract.pts) {
// TODO We should actually make sure the polygon is completely contained within the
// map's boundary.
let polygon = Polygon::new(&pts);
// TODO Don't just use the center...
let mut num_bldgs = 0;
let mut num_parking_spots = 0;
for id in ui
.primary
.draw_map
.get_matching_objects(polygon.get_bounds())
{
match id {
ID::Building(b) => {
if polygon.contains_pt(ui.primary.map.get_b(b).polygon.center()) {
num_bldgs += 1;
}
}
ID::Lane(l) => {
let lane = ui.primary.map.get_l(l);
if lane.is_parking() && polygon.contains_pt(lane.lane_center_pts.middle()) {
num_parking_spots += lane.number_parking_spots();
}
}
_ => {}
}
}
results.insert(
name.clone(),
Tract {
polygon,
// Update it after we know the total number of matching tracts.
color: Color::WHITE,
num_bldgs,
num_parking_spots,
total_owned_cars: tract.total_owned_cars(),
},
);
}
}
let len = results.len();
for (idx, tract) in results.values_mut().enumerate() {
tract.color = rotating_color_total(idx, len);
}
println!(
"Clipped {} tracts from {}",
results.len(),
popdat.tracts.len()
);
results
}
fn bar_chart(g: &mut GfxCtx, data: &BTreeMap<String, Estimate>) {
let mut max = 0;
let mut sum = 0;
for (name, est) in data {
if name == "Total:" {
continue;
}
max = max.max(est.value);
sum += est.value;
}
let mut labels = Text::new().no_bg();
for (name, est) in data {
if name == "Total:" {
continue;
}
labels.add_appended(vec![
Line(format!("{} (", name)).size(40),
Line(format!(
"{}%",
((est.value as f64) / (sum as f64) * 100.0) as usize
))
.fg(Color::RED),
Line(")"),
]);
}
let txt_dims = g.text_dims(&labels);
let line_height = txt_dims.height / ((data.len() as f64) - 1.0);
labels.add(Line(format!("{} samples", prettyprint_usize(sum))).size(40));
// This is, uh, pixels. :P
let max_bar_width = 300.0;
g.fork_screenspace();
g.draw_polygon(
Color::grey(0.3),
&Polygon::rectangle_topleft(
Pt2D::new(0.0, 0.0),
Distance::meters(txt_dims.width + 1.2 * max_bar_width),
Distance::meters(txt_dims.height + line_height),
),
);
g.draw_blocking_text(&labels, (HorizontalAlignment::Left, VerticalAlignment::Top));
// draw_blocking_text undoes this! Oops.
g.fork_screenspace();
for (idx, (name, est)) in data.iter().enumerate() {
if name == "Total:" {
continue;
}
let this_width = max_bar_width * ((est.value as f64) / (max as f64));
g.draw_polygon(
rotating_color_total(idx, data.len() - 1),
&Polygon::rectangle_topleft(
Pt2D::new(txt_dims.width, (0.1 + (idx as f64)) * line_height),
Distance::meters(this_width),
Distance::meters(0.8 * line_height),
),
);
// Error bars!
// TODO Little cap on both sides
let half_moe_width = max_bar_width * (est.moe as f64) / (max as f64) / 2.0;
g.draw_polygon(
Color::BLACK,
&Polygon::rectangle_topleft(
Pt2D::new(
txt_dims.width + this_width - half_moe_width,
(0.4 + (idx as f64)) * line_height,
),
2.0 * Distance::meters(half_moe_width),
0.2 * Distance::meters(line_height),
),
);
}
g.unfork();
}

4
game/src/mission/mod.rs
View File

@ -1,5 +1,4 @@
mod all_trips;
mod dataviz;
mod individ_trips;
mod neighborhood;
mod scenario;
@ -21,7 +20,6 @@ impl MissionEditMode {
menu: ModalMenu::new(
"Mission Edit Mode",
vec![
(hotkey(Key::D), "visualize population data"),
(hotkey(Key::T), "visualize individual PSRC trips"),
(hotkey(Key::A), "visualize all PSRC trips"),
(hotkey(Key::N), "manage neighborhoods"),
@ -42,8 +40,6 @@ impl State for MissionEditMode {
if self.menu.action("quit") {
return Transition::Pop;
} else if self.menu.action("visualize population data") {
return Transition::Push(Box::new(dataviz::DataVisualizer::new(ctx, ui)));
} else if self.menu.action("visualize individual PSRC trips") {
return Transition::Push(Box::new(individ_trips::TripsVisualizer::new(ctx, ui)));
} else if self.menu.action("visualize all PSRC trips") {

18
import.sh
View File

@ -86,24 +86,6 @@ if [ ! -f data/shapes/sidewalks.bin ]; then
cd ..
fi
if [ ! -f data/input/household_vehicles.kml ]; then
# From https://gis-kingcounty.opendata.arcgis.com/datasets/acs-household-size-by-vehicles-available-acs-b08201-householdvehicles
get_if_needed https://opendata.arcgis.com/datasets/7842d815523c4f1b9564e0301e2eafa4_2372.kml data/input/household_vehicles.kml;
get_if_needed https://www.arcgis.com/sharing/rest/content/items/7842d815523c4f1b9564e0301e2eafa4/info/metadata/metadata.xml data/input/household_vehicles.xml;
fi
if [ ! -f data/input/commute_time.kml ]; then
# From https://gis-kingcounty.opendata.arcgis.com/datasets/acs-travel-time-to-work-acs-b08303-traveltime
get_if_needed https://opendata.arcgis.com/datasets/9b5fd85861a04c5ab8b7407c7b58da7c_2375.kml data/input/commute_time.kml;
get_if_needed https://www.arcgis.com/sharing/rest/content/items/9b5fd85861a04c5ab8b7407c7b58da7c/info/metadata/metadata.xml data/input/commute_time.xml;
fi
if [ ! -f data/input/commute_mode.kml ]; then
# From https://gis-kingcounty.opendata.arcgis.com/datasets/acs-means-of-transportation-to-work-acs-b08301-transportation
get_if_needed https://opendata.arcgis.com/datasets/1da9717ca5ff4505826aba40a7ac0a58_2374.kml data/input/commute_mode.kml;
get_if_needed https://www.arcgis.com/sharing/rest/content/items/1da9717ca5ff4505826aba40a7ac0a58/info/metadata/metadata.xml data/input/commute_mode.xml;
fi
if [ ! -f data/input/offstreet_parking.kml ]; then
# From https://data.seattle.gov/Transportation/Public-Garages-or-Parking-Lots/xefx-khzm
get_if_needed http://data-seattlecitygis.opendata.arcgis.com/datasets/8e52dfde6d5d45948f7a90654c8d50cd_0.kml data/input/offstreet_parking.kml;

1
kml/Cargo.toml
View File

@ -10,4 +10,3 @@ geom = { path = "../geom" }
quick-xml = "0.13.3"
serde = "1.0.98"
serde_derive = "1.0.98"
xmltree = "0.8.0"

45
kml/src/lib.rs
View File

@ -4,8 +4,6 @@ use quick_xml::events::Event;
use quick_xml::Reader;
use serde_derive::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::{fs, io};
use xmltree::Element;
#[derive(Serialize, Deserialize)]
pub struct ExtraShapes {
@ -22,11 +20,11 @@ pub fn load(
path: &str,
gps_bounds: &GPSBounds,
timer: &mut Timer,
) -> Result<ExtraShapes, io::Error> {
) -> Result<ExtraShapes, std::io::Error> {
println!("Opening {}", path);
let (f, done) = FileWithProgress::new(path)?;
// TODO FileWithProgress should implement BufRead, so we don't have to double wrap like this
let mut reader = Reader::from_reader(io::BufReader::new(f));
let mut reader = Reader::from_reader(std::io::BufReader::new(f));
reader.trim_text(true);
let mut buf = Vec::new();
@ -102,11 +100,7 @@ pub fn load(
);
done(timer);
let mut shapes = ExtraShapes { shapes };
if fix_field_names(path, &mut shapes).is_none() {
timer.warn(format!("Applying extra XML metadata for {} failed", path));
}
Ok(shapes)
Ok(ExtraShapes { shapes })
}
fn parse_pt(input: &str, gps_bounds: &GPSBounds) -> Option<LonLat> {
@ -124,36 +118,3 @@ fn parse_pt(input: &str, gps_bounds: &GPSBounds) -> Option<LonLat> {
None
}
}
fn fix_field_names(orig_path: &str, shapes: &mut ExtraShapes) -> Option<()> {
let new_path = orig_path.replace(".kml", ".xml");
if !std::path::Path::new(&new_path).exists() {
return None;
}
println!("Loading extra metadata from {}", new_path);
let root = Element::parse(fs::read_to_string(new_path).ok()?.as_bytes()).ok()?;
let mut rename = BTreeMap::new();
for attr in &root.get_child("eainfo")?.get_child("detailed")?.children {
if attr.name != "attr" {
continue;
}
let key = attr.get_child("attrlabl")?.text.clone()?;
let value = attr.get_child("attrdef")?.text.clone()?;
rename.insert(key, value);
}
for shp in shapes.shapes.iter_mut() {
let mut attribs = BTreeMap::new();
for (k, v) in &shp.attributes {
if let Some(new_key) = rename.get(k) {
attribs.insert(new_key.clone(), v.clone());
} else {
attribs.insert(k.clone(), v.clone());
}
}
shp.attributes = attribs;
}
Some(())
}

162
popdat/src/lib.rs
View File

@ -1,174 +1,12 @@
pub mod psrc;
mod trips;
use abstutil::Timer;
use geom::{GPSBounds, LonLat};
use serde_derive::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::fmt;
pub use trips::{clip_trips, trips_to_scenario, Trip, TripEndpt};
#[derive(Serialize, Deserialize)]
pub struct PopDat {
// Keyed by census tract label
// Invariant: Every tract has all data filled out.
pub tracts: BTreeMap<String, TractData>,
pub trips: Vec<psrc::Trip>,
pub parcels: BTreeMap<i64, psrc::Parcel>,
}
#[derive(Serialize, Deserialize)]
pub struct TractData {
pub pts: Vec<LonLat>,
pub household_vehicles: BTreeMap<String, Estimate>,
pub commute_times: BTreeMap<String, Estimate>,
pub commute_modes: BTreeMap<String, Estimate>,
}
#[derive(Serialize, Deserialize)]
pub struct Estimate {
pub value: usize,
// margin of error, 90% confidence
pub moe: usize,
}
impl PopDat {
pub fn import_all(timer: &mut Timer) -> PopDat {
let mut dat = PopDat {
tracts: BTreeMap::new(),
trips: Vec::new(),
parcels: BTreeMap::new(),
};
let fields: Vec<(
&str,
Box<dyn Fn(&mut TractData, BTreeMap<String, Estimate>)>,
)> = vec![
(
"../data/input/household_vehicles.kml",
Box::new(|tract, map| {
tract.household_vehicles = map;
}),
),
(
"../data/input/commute_time.kml",
Box::new(|tract, map| {
tract.commute_times = map;
}),
),
(
"../data/input/commute_mode.kml",
Box::new(|tract, map| {
tract.commute_modes = map;
}),
),
];
for (path, setter) in fields {
for mut shape in kml::load(path, &GPSBounds::seattle_bounds(), timer)
.expect(&format!("couldn't load {}", path))
.shapes
{
let name = shape.attributes.remove("TRACT_LBL").unwrap();
if let Some(ref tract) = dat.tracts.get(&name) {
assert_eq!(shape.points, tract.pts);
} else {
dat.tracts.insert(
name.clone(),
TractData {
pts: shape.points,
household_vehicles: BTreeMap::new(),
commute_times: BTreeMap::new(),
commute_modes: BTreeMap::new(),
},
);
}
setter(
dat.tracts.get_mut(&name).unwrap(),
group_attribs(shape.attributes),
);
}
}
for (name, tract) in &dat.tracts {
if tract.household_vehicles.is_empty()
|| tract.commute_times.is_empty()
|| tract.commute_modes.is_empty()
{
panic!("{} is missing data", name);
}
}
dat
}
}
impl fmt::Display for Estimate {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{} ± {}", self.value, self.moe)
}
}
fn group_attribs(mut attribs: BTreeMap<String, String>) -> BTreeMap<String, Estimate> {
// Remove useless stuff
attribs.remove("Internal feature number.");
attribs.remove("GEO_ID_TRT");
let mut estimates = BTreeMap::new();
let mut moes = BTreeMap::new();
for (k, v) in attribs {
// These fields in the household_vehicles dataset aren't interesting.
if k.contains("person hsehold") {
continue;
}
let value = v
.parse::<usize>()
.unwrap_or_else(|_| panic!("Unknown value {}={}", k, v));
if k.starts_with("E1216 - ") {
estimates.insert(k["E1216 - ".len()..k.len()].to_string(), value);
} else if k.starts_with("M121616 - ") {
moes.insert(k["M121616 - ".len()..k.len()].to_string(), value);
} else {
panic!("Unknown key {}={}", k, v);
}
}
// If the length is the same but some keys differ, the lookup in moes below will blow up.
if estimates.len() != moes.len() {
panic!("estimates and margins of error have different keys, probably");
}
estimates
.into_iter()
.map(|(key, e)| {
(
key.clone(),
Estimate {
value: e,
moe: moes[&key],
},
)
})
.collect()
}
impl TractData {
// Nontrivial summary
pub fn total_owned_cars(&self) -> usize {
let mut sum = 0;
for (name, est) in &self.household_vehicles {
match name.as_str() {
"1 vehicle avail." => sum += est.value,
"2 vehicles avail." => sum += 2 * est.value,
"3 vehicles avail." => sum += 3 * est.value,
// Many more than 4 seems unrealistic
"4 or more vehicles avail." => sum += 4 * est.value,
"No vehicle avail." | "Total:" => {}
_ => panic!("Unknown household_vehicles key {}", name),
}
}
sum
}
}

5
popdat/src/main.rs
View File

@ -1,14 +1,11 @@
fn main() {
let mut timer = abstutil::Timer::new("creating popdat");
let mut popdat = popdat::PopDat::import_all(&mut timer);
let (trips, parcels) = popdat::psrc::import_trips(
"../data/input/parcels_urbansim.txt",
"../data/input/trips_2014.csv",
&mut timer,
)
.unwrap();
popdat.trips = trips;
popdat.parcels = parcels;
let popdat = popdat::PopDat { trips, parcels };
abstutil::write_binary(abstutil::path_popdat(), &popdat);
}

32
precompute.sh
View File

@ -2,6 +2,22 @@
set -e
mkdir -p data/maps/
# Need this first
if [ ! -f data/shapes/popdat.bin ]; then
# We probably don't have this map yet.
if [ ! -f data/maps/huge_seattle.bin ]; then
cd precompute;
RUST_BACKTRACE=1 cargo run --release ../data/raw_maps/huge_seattle.bin --disable_psrc_scenarios;
cd ..;
fi
cd popdat;
cargo run --release;
cd ..;
fi
release_mode=""
psrc_scenarios=""
no_fixes=""
@ -21,22 +37,6 @@ for arg in "$@"; do
fi
done
mkdir -p data/maps/
# Need this first
if [ ! -f data/shapes/popdat.bin ]; then
# We probably don't have this map yet.
if [ ! -f data/maps/huge_seattle.bin ]; then
cd precompute;
RUST_BACKTRACE=1 cargo run --release ../data/raw_maps/huge_seattle.bin --disable_psrc_scenarios;
cd ..;
fi
cd popdat;
cargo run --release;
cd ..;
fi
for map_path in `ls data/raw_maps/`; do
map=`basename $map_path .bin`;
echo "Precomputing $map";