From 0fa4b305219de89b1ca449e0617965976e9ff474 Mon Sep 17 00:00:00 2001 From: Dustin Carlino Date: Sat, 16 May 2020 10:26:59 -0700 Subject: [PATCH] turn soundcast parcels into extrashapes / kml to understand skipped parcels --- .gitignore | 1 + Cargo.lock | 1 - data/MANIFEST.txt | 3 +- importer/Cargo.toml | 1 - importer/src/soundcast/popdat.rs | 111 ++++++++++++------------------- 5 files changed, 46 insertions(+), 71 deletions(-) diff --git a/.gitignore b/.gitignore index bde6940d61..8f205966c4 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ data/input/seattle/N47W122.hgt data/input/seattle/offstreet_parking.bin data/input/seattle/offstreet_parking.kml data/input/seattle/osm +data/input/seattle/parcels.bin data/input/seattle/parcels_urbansim.txt data/input/seattle/popdat.bin data/input/seattle/sidewalks.bin diff --git a/Cargo.lock b/Cargo.lock index 9a489e2128..b2479f44fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1610,7 +1610,6 @@ dependencies = [ "abstutil 0.1.0", "convert_osm 0.1.0", "csv 1.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "failure 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "gdal 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "geom 0.1.0", "kml 0.1.0", diff --git a/data/MANIFEST.txt b/data/MANIFEST.txt index 3cead12716..4ce454a328 100644 --- a/data/MANIFEST.txt +++ b/data/MANIFEST.txt @@ -178,8 +178,9 @@ data/input/seattle/osm/mt_baker.osm,2cd7ba808d94318975bd73e9ced01c51,https://www data/input/seattle/osm/udistrict.osm,e5b0d978d1c74ae5c555a4ebd3ad9d70,https://www.dropbox.com/s/h66cidmzcq15pbo/udistrict.osm.zip?dl=0 data/input/seattle/osm/washington-latest.osm.pbf,363e061c2bf9aa49b48fa12ec56d0ed1,https://www.dropbox.com/s/xha9cx696czbzlf/washington-latest.osm.pbf.zip?dl=0 data/input/seattle/osm/west_seattle.osm,8f893f5a3302b3cf5a614b1d52e49b5f,https://www.dropbox.com/s/8m0vmvwg5zz3uod/west_seattle.osm.zip?dl=0 +data/input/seattle/parcels.bin,dc3eee0a4ba74a4a6ee626f2c55d5e51,https://www.dropbox.com/s/39bymqayph3easc/parcels.bin.zip?dl=0 data/input/seattle/parcels_urbansim.txt,db63d7d606e8702d12f9399e87e6a00f,https://www.dropbox.com/s/6g8rbsf200dssj3/parcels_urbansim.txt.zip?dl=0 -data/input/seattle/popdat.bin,977996fa2f2457dc399cfc76539df318,https://www.dropbox.com/s/wrpji7e14rdwszs/popdat.bin.zip?dl=0 +data/input/seattle/popdat.bin,d1fe506c5f4a65447646f0100e3e3a5a,https://www.dropbox.com/s/wrpji7e14rdwszs/popdat.bin.zip?dl=0 data/input/seattle/sidewalks.bin,034dd47ab77902dbc81c0107f13d8965,https://www.dropbox.com/s/ma9bmisijc7v7xa/sidewalks.bin.zip?dl=0 data/input/seattle/sidewalks.kml,94d385ba03ef1b57a5ba10965913ec6c,https://www.dropbox.com/s/vn8amar9xi6vbvh/sidewalks.kml.zip?dl=0 data/input/seattle/trips_2014.csv,d4a8e733045b28c0385fb81359d6df03,https://www.dropbox.com/s/5ppravwmk6bf20d/trips_2014.csv.zip?dl=0 diff --git a/importer/Cargo.toml b/importer/Cargo.toml index 96a01a0a3f..97cc852fd3 100644 --- a/importer/Cargo.toml +++ b/importer/Cargo.toml @@ -8,7 +8,6 @@ edition = "2018" abstutil = { path = "../abstutil" } convert_osm = { path = "../convert_osm" } csv = "1.0.1" -failure = "0.1.2" geom = { path = "../geom" } gdal = "0.6.0" kml = { path = "../kml" } diff --git a/importer/src/soundcast/popdat.rs b/importer/src/soundcast/popdat.rs index dd7bb74204..ef43dae532 100644 --- a/importer/src/soundcast/popdat.rs +++ b/importer/src/soundcast/popdat.rs @@ -1,59 +1,41 @@ use abstutil::{prettyprint_usize, FileWithProgress, Timer}; use geom::{Distance, Duration, FindClosest, LonLat, Pt2D, Time}; +use kml::{ExtraShape, ExtraShapes}; use map_model::Map; use serde_derive::{Deserialize, Serialize}; use sim::{OrigPersonID, TripMode}; use std::collections::{BTreeMap, HashMap, HashSet}; -use std::fs::File; -use std::io::Write; #[derive(Serialize, Deserialize)] pub struct PopDat { pub trips: Vec, - pub parcels: BTreeMap, } // Extract trip demand data from PSRC's Soundcast outputs. pub fn import_data() { let mut timer = abstutil::Timer::new("creating popdat"); - let (trips, parcels) = import_trips( - "../data/input/seattle/parcels_urbansim.txt", - "../data/input/seattle/trips_2014.csv", - &mut timer, - ) - .unwrap(); - let popdat = PopDat { trips, parcels }; + let parcels = import_parcels("../data/input/seattle/parcels_urbansim.txt", &mut timer); + let trips = import_trips(parcels, "../data/input/seattle/trips_2014.csv", &mut timer); + let popdat = PopDat { trips }; abstutil::write_binary(abstutil::path_popdat(), &popdat); } fn import_trips( - parcels_path: &str, + parcels_lookup: HashMap, trips_path: &str, timer: &mut Timer, -) -> Result<(Vec, BTreeMap), failure::Error> { - let (parcels, metadata) = import_parcels(parcels_path, timer)?; - - if false { - timer.start("recording parcel IDs"); - let mut f = File::create("parcels.csv")?; - writeln!(f, "parcel_id")?; - for id in parcels.keys() { - writeln!(f, "{}", id)?; - } - timer.stop("recording parcel IDs"); - } - +) -> Vec { let mut trips = Vec::new(); - let (reader, done) = FileWithProgress::new(trips_path)?; + let (reader, done) = FileWithProgress::new(trips_path).unwrap(); let mut total_records = 0; let mut people: HashSet = HashSet::new(); for rec in csv::Reader::from_reader(reader).deserialize() { total_records += 1; - let rec: RawTrip = rec?; + let rec: RawTrip = rec.unwrap(); - let from = parcels[&(rec.opcl as usize)].clone(); - let to = parcels[&(rec.dpcl as usize)].clone(); + let from = parcels_lookup[&(rec.opcl as usize)].clone(); + let to = parcels_lookup[&(rec.dpcl as usize)].clone(); // If both are None, then skip -- the trip doesn't start or end within huge_seattle. // If both are the same building, also skip -- that's a redundant trip. @@ -102,15 +84,12 @@ fn import_trips( trips.sort_by_key(|t| t.depart_at); - Ok((trips, metadata)) + trips } // TODO Do we also need the zone ID, or is parcel ID globally unique? -// Returns (parcel ID -> Endpoint) and (OSM building ID -> metadata) -fn import_parcels( - path: &str, - timer: &mut Timer, -) -> Result<(HashMap, BTreeMap), failure::Error> { +// Returns parcel ID -> Endpoint +fn import_parcels(path: &str, timer: &mut Timer) -> HashMap { let map = Map::new(abstutil::path_map("huge_seattle"), timer); // TODO I really just want to do polygon containment with a quadtree. FindClosest only does @@ -124,23 +103,18 @@ fn import_parcels( let mut y_coords: Vec = Vec::new(); // Dummy values let mut z_coords: Vec = Vec::new(); - // (parcel ID, number of households, number of employees, number of parking spots) + // (parcel ID, number of households, number of parking spots) let mut parcel_metadata = Vec::new(); - let (reader, done) = FileWithProgress::new(path)?; + let (reader, done) = FileWithProgress::new(path).unwrap(); for rec in csv::ReaderBuilder::new() .delimiter(b' ') .from_reader(reader) .deserialize() { - let rec: RawParcel = rec?; + let rec: RawParcel = rec.unwrap(); // Note parkdy_p and parkhr_p might overlap, so this could be double-counting. >_< - parcel_metadata.push(( - rec.parcelid, - rec.hh_p, - rec.emptot_p, - rec.parkdy_p + rec.parkhr_p, - )); + parcel_metadata.push((rec.parcelid, rec.hh_p, rec.parkdy_p + rec.parkhr_p)); x_coords.push(rec.xcoord_p); y_coords.push(rec.ycoord_p); z_coords.push(0.0); @@ -166,44 +140,53 @@ fn import_parcels( timer.stop(format!("transform {} points", parcel_metadata.len())); let bounds = map.get_gps_bounds(); + let boundary = map.get_boundary_polygon(); let mut result = HashMap::new(); - let mut metadata = BTreeMap::new(); + let mut shapes = Vec::new(); timer.start_iter("finalize parcel output", parcel_metadata.len()); - for ((x, y), (id, num_households, num_employees, offstreet_parking_spaces)) in x_coords + for ((x, y), (id, num_households, offstreet_parking_spaces)) in x_coords .into_iter() .zip(y_coords.into_iter()) .zip(parcel_metadata.into_iter()) { timer.next(); - let pt = LonLat::new(x, y); - let osm_building = if bounds.contains(pt) { + let gps = LonLat::new(x, y); + let pt = Pt2D::forcibly_from_gps(gps, bounds); + let osm_building = if bounds.contains(gps) { + if boundary.contains_pt(pt) { + let mut attributes = BTreeMap::new(); + attributes.insert("id".to_string(), id.to_string()); + attributes.insert("households".to_string(), num_households.to_string()); + attributes.insert("parking".to_string(), offstreet_parking_spaces.to_string()); + shapes.push(ExtraShape { + points: vec![gps], + attributes, + }); + } + closest_bldg - .closest_pt(Pt2D::forcibly_from_gps(pt, bounds), Distance::meters(30.0)) + .closest_pt(pt, Distance::meters(30.0)) .map(|(b, _)| b) } else { None }; - if let Some(b) = osm_building { - metadata.insert( - b, - Parcel { - num_households, - num_employees, - offstreet_parking_spaces, - }, - ); - } result.insert( id, Endpoint { - pos: pt, + pos: gps, osm_building, parcel_id: id, }, ); } timer.note(format!("{} parcels", prettyprint_usize(result.len()))); - Ok((result, metadata)) + + abstutil::write_binary( + "../data/input/seattle/parcels.bin".to_string(), + &ExtraShapes { shapes }, + ); + + result } // From https://github.com/psrc/soundcast/wiki/Outputs#trip-file-_triptsv, opurp and dpurp @@ -262,7 +245,6 @@ struct RawTrip { struct RawParcel { parcelid: usize, hh_p: usize, - emptot_p: usize, parkdy_p: usize, parkhr_p: usize, xcoord_p: f64, @@ -292,13 +274,6 @@ pub struct Endpoint { pub parcel_id: usize, } -#[derive(Serialize, Deserialize)] -pub struct Parcel { - pub num_households: usize, - pub num_employees: usize, - pub offstreet_parking_spaces: usize, -} - #[derive(Serialize, Deserialize, Debug, Clone, Copy)] pub enum Purpose { Home,