turn soundcast parcels into extrashapes / kml to understand skipped

parcels

.gitignore

@@ -14,6 +14,7 @@ data/input/seattle/N47W122.hgt
 data/input/seattle/offstreet_parking.bin
 data/input/seattle/offstreet_parking.kml
 data/input/seattle/osm
+data/input/seattle/parcels.bin
 data/input/seattle/parcels_urbansim.txt
 data/input/seattle/popdat.bin
 data/input/seattle/sidewalks.bin

Cargo.lock

@@ -1610,7 +1610,6 @@ dependencies = [
  "abstutil 0.1.0",
  "convert_osm 0.1.0",
  "csv 1.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "failure 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "gdal 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "geom 0.1.0",
  "kml 0.1.0",

data/MANIFEST.txt

@@ -178,8 +178,9 @@ data/input/seattle/osm/mt_baker.osm,2cd7ba808d94318975bd73e9ced01c51,https://www
 data/input/seattle/osm/udistrict.osm,e5b0d978d1c74ae5c555a4ebd3ad9d70,https://www.dropbox.com/s/h66cidmzcq15pbo/udistrict.osm.zip?dl=0
 data/input/seattle/osm/washington-latest.osm.pbf,363e061c2bf9aa49b48fa12ec56d0ed1,https://www.dropbox.com/s/xha9cx696czbzlf/washington-latest.osm.pbf.zip?dl=0
 data/input/seattle/osm/west_seattle.osm,8f893f5a3302b3cf5a614b1d52e49b5f,https://www.dropbox.com/s/8m0vmvwg5zz3uod/west_seattle.osm.zip?dl=0
+data/input/seattle/parcels.bin,dc3eee0a4ba74a4a6ee626f2c55d5e51,https://www.dropbox.com/s/39bymqayph3easc/parcels.bin.zip?dl=0
 data/input/seattle/parcels_urbansim.txt,db63d7d606e8702d12f9399e87e6a00f,https://www.dropbox.com/s/6g8rbsf200dssj3/parcels_urbansim.txt.zip?dl=0
-data/input/seattle/popdat.bin,977996fa2f2457dc399cfc76539df318,https://www.dropbox.com/s/wrpji7e14rdwszs/popdat.bin.zip?dl=0
+data/input/seattle/popdat.bin,d1fe506c5f4a65447646f0100e3e3a5a,https://www.dropbox.com/s/wrpji7e14rdwszs/popdat.bin.zip?dl=0
 data/input/seattle/sidewalks.bin,034dd47ab77902dbc81c0107f13d8965,https://www.dropbox.com/s/ma9bmisijc7v7xa/sidewalks.bin.zip?dl=0
 data/input/seattle/sidewalks.kml,94d385ba03ef1b57a5ba10965913ec6c,https://www.dropbox.com/s/vn8amar9xi6vbvh/sidewalks.kml.zip?dl=0
 data/input/seattle/trips_2014.csv,d4a8e733045b28c0385fb81359d6df03,https://www.dropbox.com/s/5ppravwmk6bf20d/trips_2014.csv.zip?dl=0

importer/Cargo.toml

@@ -8,7 +8,6 @@ edition = "2018"
 abstutil = { path = "../abstutil" }
 convert_osm = { path = "../convert_osm" }
 csv = "1.0.1"
-failure = "0.1.2"
 geom = { path = "../geom" }
 gdal = "0.6.0"
 kml = { path = "../kml" }

importer/src/soundcast/popdat.rs

@@ -1,59 +1,41 @@
 use abstutil::{prettyprint_usize, FileWithProgress, Timer};
 use geom::{Distance, Duration, FindClosest, LonLat, Pt2D, Time};
+use kml::{ExtraShape, ExtraShapes};
 use map_model::Map;
 use serde_derive::{Deserialize, Serialize};
 use sim::{OrigPersonID, TripMode};
 use std::collections::{BTreeMap, HashMap, HashSet};
-use std::fs::File;
-use std::io::Write;
 
 #[derive(Serialize, Deserialize)]
 pub struct PopDat {
     pub trips: Vec<OrigTrip>,
-    pub parcels: BTreeMap<i64, Parcel>,
 }
 
 // Extract trip demand data from PSRC's Soundcast outputs.
 pub fn import_data() {
     let mut timer = abstutil::Timer::new("creating popdat");
-    let (trips, parcels) = import_trips(
+    let parcels = import_parcels("../data/input/seattle/parcels_urbansim.txt", &mut timer);
-        "../data/input/seattle/parcels_urbansim.txt",
+    let trips = import_trips(parcels, "../data/input/seattle/trips_2014.csv", &mut timer);
-        "../data/input/seattle/trips_2014.csv",
+    let popdat = PopDat { trips };
-        &mut timer,
-    )
-    .unwrap();
-    let popdat = PopDat { trips, parcels };
     abstutil::write_binary(abstutil::path_popdat(), &popdat);
 }
 
 fn import_trips(
-    parcels_path: &str,
+    parcels_lookup: HashMap<usize, Endpoint>,
     trips_path: &str,
     timer: &mut Timer,
-) -> Result<(Vec<OrigTrip>, BTreeMap<i64, Parcel>), failure::Error> {
+) -> Vec<OrigTrip> {
-    let (parcels, metadata) = import_parcels(parcels_path, timer)?;
-
-    if false {
-        timer.start("recording parcel IDs");
-        let mut f = File::create("parcels.csv")?;
-        writeln!(f, "parcel_id")?;
-        for id in parcels.keys() {
-            writeln!(f, "{}", id)?;
-        }
-        timer.stop("recording parcel IDs");
-    }
-
     let mut trips = Vec::new();
-    let (reader, done) = FileWithProgress::new(trips_path)?;
+    let (reader, done) = FileWithProgress::new(trips_path).unwrap();
     let mut total_records = 0;
     let mut people: HashSet<OrigPersonID> = HashSet::new();
 
     for rec in csv::Reader::from_reader(reader).deserialize() {
         total_records += 1;
-        let rec: RawTrip = rec?;
+        let rec: RawTrip = rec.unwrap();
 
-        let from = parcels[&(rec.opcl as usize)].clone();
+        let from = parcels_lookup[&(rec.opcl as usize)].clone();
-        let to = parcels[&(rec.dpcl as usize)].clone();
+        let to = parcels_lookup[&(rec.dpcl as usize)].clone();
 
         // If both are None, then skip -- the trip doesn't start or end within huge_seattle.
         // If both are the same building, also skip -- that's a redundant trip.
@@ -102,15 +84,12 @@ fn import_trips(
 
     trips.sort_by_key(|t| t.depart_at);
 
-    Ok((trips, metadata))
+    trips
 }
 
 // TODO Do we also need the zone ID, or is parcel ID globally unique?
-// Returns (parcel ID -> Endpoint) and (OSM building ID -> metadata)
+// Returns parcel ID -> Endpoint
-fn import_parcels(
+fn import_parcels(path: &str, timer: &mut Timer) -> HashMap<usize, Endpoint> {
-    path: &str,
-    timer: &mut Timer,
-) -> Result<(HashMap<usize, Endpoint>, BTreeMap<i64, Parcel>), failure::Error> {
     let map = Map::new(abstutil::path_map("huge_seattle"), timer);
 
     // TODO I really just want to do polygon containment with a quadtree. FindClosest only does
@@ -124,23 +103,18 @@ fn import_parcels(
     let mut y_coords: Vec<f64> = Vec::new();
     // Dummy values
     let mut z_coords: Vec<f64> = Vec::new();
-    // (parcel ID, number of households, number of employees, number of parking spots)
+    // (parcel ID, number of households, number of parking spots)
     let mut parcel_metadata = Vec::new();
 
-    let (reader, done) = FileWithProgress::new(path)?;
+    let (reader, done) = FileWithProgress::new(path).unwrap();
     for rec in csv::ReaderBuilder::new()
         .delimiter(b' ')
         .from_reader(reader)
         .deserialize()
     {
-        let rec: RawParcel = rec?;
+        let rec: RawParcel = rec.unwrap();
         // Note parkdy_p and parkhr_p might overlap, so this could be double-counting. >_<
-        parcel_metadata.push((
+        parcel_metadata.push((rec.parcelid, rec.hh_p, rec.parkdy_p + rec.parkhr_p));
-            rec.parcelid,
-            rec.hh_p,
-            rec.emptot_p,
-            rec.parkdy_p + rec.parkhr_p,
-        ));
         x_coords.push(rec.xcoord_p);
         y_coords.push(rec.ycoord_p);
         z_coords.push(0.0);
@@ -166,44 +140,53 @@ fn import_parcels(
     timer.stop(format!("transform {} points", parcel_metadata.len()));
 
     let bounds = map.get_gps_bounds();
+    let boundary = map.get_boundary_polygon();
     let mut result = HashMap::new();
-    let mut metadata = BTreeMap::new();
+    let mut shapes = Vec::new();
     timer.start_iter("finalize parcel output", parcel_metadata.len());
-    for ((x, y), (id, num_households, num_employees, offstreet_parking_spaces)) in x_coords
+    for ((x, y), (id, num_households, offstreet_parking_spaces)) in x_coords
         .into_iter()
         .zip(y_coords.into_iter())
         .zip(parcel_metadata.into_iter())
     {
         timer.next();
-        let pt = LonLat::new(x, y);
+        let gps = LonLat::new(x, y);
-        let osm_building = if bounds.contains(pt) {
+        let pt = Pt2D::forcibly_from_gps(gps, bounds);
+        let osm_building = if bounds.contains(gps) {
+            if boundary.contains_pt(pt) {
+                let mut attributes = BTreeMap::new();
+                attributes.insert("id".to_string(), id.to_string());
+                attributes.insert("households".to_string(), num_households.to_string());
+                attributes.insert("parking".to_string(), offstreet_parking_spaces.to_string());
+                shapes.push(ExtraShape {
+                    points: vec![gps],
+                    attributes,
+                });
+            }
+
             closest_bldg
-                .closest_pt(Pt2D::forcibly_from_gps(pt, bounds), Distance::meters(30.0))
+                .closest_pt(pt, Distance::meters(30.0))
                 .map(|(b, _)| b)
         } else {
             None
         };
-        if let Some(b) = osm_building {
-            metadata.insert(
-                b,
-                Parcel {
-                    num_households,
-                    num_employees,
-                    offstreet_parking_spaces,
-                },
-            );
-        }
         result.insert(
             id,
             Endpoint {
-                pos: pt,
+                pos: gps,
                 osm_building,
                 parcel_id: id,
             },
         );
     }
     timer.note(format!("{} parcels", prettyprint_usize(result.len())));
-    Ok((result, metadata))
+
+    abstutil::write_binary(
+        "../data/input/seattle/parcels.bin".to_string(),
+        &ExtraShapes { shapes },
+    );
+
+    result
 }
 
 // From https://github.com/psrc/soundcast/wiki/Outputs#trip-file-_triptsv, opurp and dpurp
@@ -262,7 +245,6 @@ struct RawTrip {
 struct RawParcel {
     parcelid: usize,
     hh_p: usize,
-    emptot_p: usize,
     parkdy_p: usize,
     parkhr_p: usize,
     xcoord_p: f64,
@@ -292,13 +274,6 @@ pub struct Endpoint {
     pub parcel_id: usize,
 }
 
-#[derive(Serialize, Deserialize)]
-pub struct Parcel {
-    pub num_households: usize,
-    pub num_employees: usize,
-    pub offstreet_parking_spaces: usize,
-}
-
 #[derive(Serialize, Deserialize, Debug, Clone, Copy)]
 pub enum Purpose {
     Home,