use named csv fields in psrc parser. group trips by person in popdat layer, discovering that almost no people have multiple trips...

popdat/Cargo.toml

@@ -9,6 +9,7 @@ abstutil = { path = "../abstutil" }
 csv = "1.0.1"
 failure = "0.1.2"
 geom = { path = "../geom" }
+itertools = "0.8.0"
 kml = { path = "../kml" }
 map_model = { path = "../map_model" }
 serde = "1.0.98"

popdat/src/psrc.rs

@@ -67,51 +67,34 @@ pub fn import_trips(
 
     let mut trips = Vec::new();
     let (reader, done) = FileWithProgress::new(trips_path)?;
-    for rec in csv::Reader::from_reader(reader).records() {
-        let rec = rec?;
+    for rec in csv::Reader::from_reader(reader).deserialize() {
+        let rec: RawTrip = rec?;
 
-        // opcl
-        let from = skip_fail!(parcels.get(rec[15].trim_end_matches(".0"))).clone();
-        // dpcl
-        let to = skip_fail!(parcels.get(rec[6].trim_end_matches(".0"))).clone();
+        let from = skip_fail!(parcels.get(&(rec.opcl as usize))).clone();
+        let to = skip_fail!(parcels.get(&(rec.dpcl as usize))).clone();
 
         if from.osm_building == to.osm_building {
             // TODO Plumb along pass-through trips later
             if from.osm_building.is_some() {
                 timer.warn(format!(
                     "Skipping trip from parcel {} to {}; both match OSM building {:?}",
-                    &rec[15], &rec[6], from.osm_building
+                    rec.opcl, rec.dpcl, from.osm_building
                 ));
             }
             continue;
         }
 
-        // deptm
-        let depart_at =
-            Time::START_OF_DAY + Duration::minutes(rec[4].trim_end_matches(".0").parse::<usize>()?);
+        let depart_at = Time::START_OF_DAY + Duration::minutes(rec.deptm as usize);
 
-        // mode
-        let mode = skip_fail!(get_mode(&rec[13]));
+        let mode = skip_fail!(get_mode(&rec.mode));
 
-        // opurp and dpurp
-        let purpose = (get_purpose(&rec[16]), get_purpose(&rec[7]));
+        let purpose = (get_purpose(&rec.opurp), get_purpose(&rec.dpurp));
 
-        // travtime
-        let trip_time = Duration::f64_minutes(rec[25].parse::<f64>()?);
-        // travdist
-        let trip_dist = Distance::miles(rec[24].parse::<f64>()?);
+        let trip_time = Duration::f64_minutes(rec.travtime);
+        let trip_dist = Distance::miles(rec.travdist);
 
-        // (hhno, pno)
-        let person = (
-            rec[11].trim_end_matches(".0").parse::<usize>()?,
-            rec[19].trim_end_matches(".0").parse::<usize>()?,
-        );
-        // (tour, half, tseg)
-        let seq = (
-            rec[21].trim_end_matches(".0").parse::<usize>()?,
-            &rec[10] == "2.0",
-            rec[27].trim_end_matches(".0").parse::<usize>()?,
-        );
+        let person = (rec.hhno as usize, rec.pno as usize);
+        let seq = (rec.tour as usize, rec.half == 2.0, rec.tseg as usize);
 
         trips.push(Trip {
             from,
@@ -139,7 +122,7 @@ pub fn import_trips(
 fn import_parcels(
     path: &str,
     timer: &mut Timer,
-) -> Result<(HashMap<String, Endpoint>, BTreeMap<i64, Parcel>), failure::Error> {
+) -> Result<(HashMap<usize, Endpoint>, BTreeMap<i64, Parcel>), failure::Error> {
     let map = Map::new(abstutil::path_map("huge_seattle"), false, timer);
 
     // TODO I really just want to do polygon containment with a quadtree. FindClosest only does
@@ -157,18 +140,17 @@ fn import_parcels(
     for rec in csv::ReaderBuilder::new()
         .delimiter(b' ')
         .from_reader(reader)
-        .records()
+        .deserialize()
     {
-        let rec = rec?;
-        // parcelid, hh_p, emptot_p, parkdy_p, parkhr_p
+        let rec: RawParcel = rec?;
         // Note parkdy_p and parkhr_p might overlap, so this could be double-counting. >_<
         parcel_metadata.push((
-            rec[15].to_string(),
-            rec[12].parse::<usize>()?,
-            rec[11].parse::<usize>()?,
-            rec[16].parse::<usize>()? + rec[17].parse::<usize>()?,
+            rec.parcelid,
+            rec.hh_p,
+            rec.emptot_p,
+            rec.parkdy_p + rec.parkhr_p,
         ));
-        coords.write_fmt(format_args!("{} {}\n", &rec[25], &rec[26]))?;
+        coords.write_fmt(format_args!("{} {}\n", rec.xcoord_p, rec.ycoord_p))?;
     }
     done(timer);
     coords.flush()?;
@@ -267,3 +249,33 @@ fn get_mode(code: &str) -> Option<Mode> {
         _ => None,
     }
 }
+
+// See https://github.com/psrc/soundcast/wiki/Outputs#trip-file-_triptsv
+#[derive(Debug, Deserialize)]
+struct RawTrip {
+    opcl: f64,
+    dpcl: f64,
+    deptm: f64,
+    mode: String,
+    opurp: String,
+    dpurp: String,
+    travtime: f64,
+    travdist: f64,
+    hhno: f64,
+    pno: f64,
+    tour: f64,
+    half: f64,
+    tseg: f64,
+}
+
+// See https://github.com/psrc/soundcast/wiki/Outputs#buffered-parcel-file-buffered_parcelsdat
+#[derive(Debug, Deserialize)]
+struct RawParcel {
+    parcelid: usize,
+    hh_p: usize,
+    emptot_p: usize,
+    parkdy_p: usize,
+    parkhr_p: usize,
+    xcoord_p: f64,
+    ycoord_p: f64,
+}

popdat/src/trips.rs

@@ -1,7 +1,9 @@
 use crate::psrc::{Endpoint, Mode, Parcel, Purpose};
 use crate::PopDat;
+use abstutil::prettyprint_usize;
 use abstutil::Timer;
 use geom::{Distance, Duration, LonLat, Polygon, Pt2D, Time};
+use itertools::Itertools;
 use map_model::{BuildingID, IntersectionID, Map, PathConstraints, Position};
 use sim::{DrivingGoal, Scenario, SidewalkSpot, SpawnTrip, TripSpec};
 use std::collections::{BTreeMap, HashMap};
@@ -286,14 +288,36 @@ pub fn clip_trips(map: &Map, timer: &mut Timer) -> (Vec<Trip>, HashMap<BuildingI
 
 pub fn trips_to_scenario(map: &Map, timer: &mut Timer) -> Scenario {
     let (trips, _) = clip_trips(map, timer);
+    let mut individ_trips: Vec<SpawnTrip> = Vec::new();
     // TODO Don't clone trips for parallelize
-    let individ_trips = timer
+    let mut num_ppl = 0;
+    for (person, list) in timer
         .parallelize("turn PSRC trips into SpawnTrips", trips.clone(), |trip| {
             trip.to_spawn_trip(map)
+                .map(|spawn| (spawn, trip.person, trip.seq))
         })
         .into_iter()
         .flatten()
-        .collect();
+        .group_by(|(_, person, _)| *person)
+        .into_iter()
+    {
+        // TODO Try doing the grouping earlier, before we filter out cases where to_spawn_trip
+        // fails
+        num_ppl += 1;
+        let mut seqs = Vec::new();
+        for (spawn, _, seq) in list {
+            seqs.push(seq);
+            individ_trips.push(spawn);
+        }
+        if seqs.len() > 1 {
+            println!("{:?} takes {} trips: {:?}", person, seqs.len(), seqs);
+        }
+    }
+    timer.note(format!(
+        "{} trips over {} people",
+        prettyprint_usize(individ_trips.len()),
+        prettyprint_usize(num_ppl)
+    ));
 
     // How many parked cars do we need to spawn near each building?
     // TODO This assumes trips are instantaneous. At runtime, somebody might try to use a parked