From ee35ff79b9b0d81ec220a3af3b1761390e2780dc Mon Sep 17 00:00:00 2001 From: Dustin Carlino Date: Sun, 17 May 2020 14:19:45 -0700 Subject: [PATCH] dont needlessly load files in importer --- docs/new_city.md | 4 ++ geom/src/gps.rs | 1 - importer/src/main.rs | 81 +++++++++++++++++++------------- importer/src/seattle.rs | 19 +++++--- importer/src/soundcast/mod.rs | 2 +- importer/src/soundcast/popdat.rs | 24 +++++----- importer/src/soundcast/trips.rs | 75 ++++++++++++++++------------- importer/src/utils.rs | 12 +++-- 8 files changed, 129 insertions(+), 89 deletions(-) diff --git a/docs/new_city.md b/docs/new_city.md index e7246ba6d0..c2c7b9840b 100644 --- a/docs/new_city.md +++ b/docs/new_city.md @@ -10,6 +10,10 @@ If you have a `.osm` file, you can just run `./import.sh --oneshot=/absolute/path/to/map.osm`. This tool will generate a new file in `data/system/maps` that you can then load in the game. +If you're using a binary release, you have to be sure to run the tool from the +`importer/` directory, so that `../data/` exists: +`cd importer; ./importer --oneshot=/absolute/path/to/file.osm` + If you have an Osmosis polygon filter (see below), you can also pass `--oneshot_clip=/absolute/path/to/clip.poly` to improve the result. You should first make sure your .osm has been clipped: diff --git a/geom/src/gps.rs b/geom/src/gps.rs index 40717b6ede..3b3cd7e2b9 100644 --- a/geom/src/gps.rs +++ b/geom/src/gps.rs @@ -76,7 +76,6 @@ impl LonLat { .map_err(|err| Error::new(ErrorKind::Other, err))?, )); } - pts.pop(); Ok(pts) } } diff --git a/importer/src/main.rs b/importer/src/main.rs index 4f5beea3c7..ad20e01709 100644 --- a/importer/src/main.rs +++ b/importer/src/main.rs @@ -4,7 +4,7 @@ mod seattle; mod soundcast; mod utils; -use std::thread; +// TODO Might be cleaner to express as a dependency graph? struct Job { city: String, @@ -13,7 +13,6 @@ struct Job { scenario: bool, scenario_everyone: bool, - seq: bool, skip_ch: bool, only_map: Option, @@ -34,8 +33,6 @@ fn main() { scenario: args.enabled("--scenario"), // Produce a variation of the weekday scenario including off-map trips. scenario_everyone: args.enabled("--scenario_everyone"), - // Don't use multiple threads for conversion. Useful when needing to see errors. - seq: args.enabled("--seq"), // Skip the most expensive step of --map, building contraction hierarchies. The resulting // map won't be usable for simulation; as soon as you try to pathfind, it'll crash. skip_ch: args.enabled("--skip_ch"), @@ -75,7 +72,30 @@ fn main() { abstutil::list_all_objects(format!("../data/input/{}/polygons", job.city)) }; - let mut handles = vec![]; + let mut timer = abstutil::Timer::new("import map data"); + + let (maybe_popdat, maybe_huge_map) = if job.scenario || job.scenario_everyone { + assert_eq!(job.city, "seattle"); + + #[cfg(feature = "scenarios")] + { + let (popdat, huge_map) = seattle::ensure_popdat_exists(&mut timer); + (Some(popdat), Some(huge_map)) + } + + #[cfg(not(feature = "scenarios"))] + { + panic!( + "Can't do --scenario or --scenario_everyone without the scenarios feature \ + compiled in" + ); + // Nonsense to make the type-checker work + (Some(true), Some(true)) + } + } else { + (None, None) + }; + for name in names { if job.osm_to_raw { match job.city.as_ref() { @@ -85,42 +105,39 @@ fn main() { } } - if job.raw_to_map { - // TODO Bug: if regenerating map and scenario at the same time, this doesn't work. - if job.scenario || job.seq { - utils::raw_to_map(&name, !job.skip_ch); - } else { - let name = name.clone(); - let build_ch = !job.skip_ch; - handles.push(thread::spawn(move || { - utils::raw_to_map(&name, build_ch); - })); - } - } + let maybe_map = if job.raw_to_map { + Some(utils::raw_to_map(&name, !job.skip_ch, &mut timer)) + } else if job.scenario || job.scenario_everyone { + Some(map_model::Map::new(abstutil::path_map(&name), &mut timer)) + } else { + None + }; #[cfg(feature = "scenarios")] if job.scenario { - assert_eq!(job.city, "seattle"); - seattle::ensure_popdat_exists(); - - let mut timer = abstutil::Timer::new(format!("Scenario for {}", name)); - let map = map_model::Map::new(abstutil::path_map(&name), &mut timer); - soundcast::make_weekday_scenario(&map, &mut timer).save(); + timer.start(format!("scenario for {}", name)); + soundcast::make_weekday_scenario( + maybe_map.as_ref().unwrap(), + maybe_popdat.as_ref().unwrap(), + maybe_huge_map.as_ref().unwrap(), + &mut timer, + ) + .save(); + timer.stop(format!("scenario for {}", name)); } #[cfg(feature = "scenarios")] if job.scenario_everyone { - assert_eq!(job.city, "seattle"); - seattle::ensure_popdat_exists(); - - let mut timer = abstutil::Timer::new(format!("Scenario for {}", name)); - let map = map_model::Map::new(abstutil::path_map(&name), &mut timer); - soundcast::make_weekday_scenario_with_everyone(&map, &mut timer).save(); + timer.start(format!("scenario_everyone for {}", name)); + soundcast::make_weekday_scenario_with_everyone( + maybe_map.as_ref().unwrap(), + maybe_popdat.as_ref().unwrap(), + &mut timer, + ) + .save(); + timer.stop(format!("scenario_everyone for {}", name)); } } - for handle in handles { - handle.join().unwrap(); - } } fn oneshot(osm_path: String, clip: Option) { diff --git a/importer/src/seattle.rs b/importer/src/seattle.rs index 63501568be..6f70c34666 100644 --- a/importer/src/seattle.rs +++ b/importer/src/seattle.rs @@ -71,18 +71,25 @@ pub fn osm_to_raw(name: &str) { // Download and pre-process data needed to generate Seattle scenarios. #[cfg(feature = "scenarios")] -pub fn ensure_popdat_exists() { +pub fn ensure_popdat_exists( + timer: &mut abstutil::Timer, +) -> (crate::soundcast::PopDat, map_model::Map) { if abstutil::file_exists(abstutil::path_popdat()) { println!("- {} exists, not regenerating it", abstutil::path_popdat()); - return; + return ( + abstutil::read_binary(abstutil::path_popdat(), timer), + map_model::Map::new(abstutil::path_map("huge_seattle"), timer), + ); } if !abstutil::file_exists(abstutil::path_raw_map("huge_seattle")) { osm_to_raw("huge_seattle"); } - if !abstutil::file_exists(abstutil::path_map("huge_seattle")) { - crate::utils::raw_to_map("huge_seattle", true); - } + let huge_map = if abstutil::file_exists(abstutil::path_map("huge_seattle")) { + map_model::Map::new(abstutil::path_map("huge_seattle"), timer) + } else { + crate::utils::raw_to_map("huge_seattle", true, timer) + }; - crate::soundcast::import_data(); + (crate::soundcast::import_data(&huge_map), huge_map) } diff --git a/importer/src/soundcast/mod.rs b/importer/src/soundcast/mod.rs index 8dcf0dccf8..135c554a6d 100644 --- a/importer/src/soundcast/mod.rs +++ b/importer/src/soundcast/mod.rs @@ -1,5 +1,5 @@ mod popdat; mod trips; -pub use self::popdat::import_data; +pub use self::popdat::{import_data, PopDat}; pub use self::trips::{make_weekday_scenario, make_weekday_scenario_with_everyone}; diff --git a/importer/src/soundcast/popdat.rs b/importer/src/soundcast/popdat.rs index 1374be61f4..beb354f99b 100644 --- a/importer/src/soundcast/popdat.rs +++ b/importer/src/soundcast/popdat.rs @@ -12,15 +12,16 @@ pub struct PopDat { } // Extract trip demand data from PSRC's Soundcast outputs. -pub fn import_data() { +pub fn import_data(huge_map: &Map) -> PopDat { let mut timer = abstutil::Timer::new("creating popdat"); - let trips = import_trips(&mut timer); + let trips = import_trips(huge_map, &mut timer); let popdat = PopDat { trips }; abstutil::write_binary(abstutil::path_popdat(), &popdat); + popdat } -fn import_trips(timer: &mut Timer) -> Vec { - let (parcels, mut keyed_shapes) = import_parcels(timer); +fn import_trips(huge_map: &Map, timer: &mut Timer) -> Vec { + let (parcels, mut keyed_shapes) = import_parcels(huge_map, timer); let mut trips = Vec::new(); let (reader, done) = FileWithProgress::new("../data/input/seattle/trips_2014.csv").unwrap(); @@ -109,13 +110,14 @@ fn import_trips(timer: &mut Timer) -> Vec { // TODO Do we also need the zone ID, or is parcel ID globally unique? // Keyed by parcel ID -fn import_parcels(timer: &mut Timer) -> (HashMap, HashMap) { - let map = Map::new(abstutil::path_map("huge_seattle"), timer); - +fn import_parcels( + huge_map: &Map, + timer: &mut Timer, +) -> (HashMap, HashMap) { // TODO I really just want to do polygon containment with a quadtree. FindClosest only does // line-string stuff right now, which'll be weird for the last->first pt line and stuff. - let mut closest_bldg: FindClosest = FindClosest::new(map.get_bounds()); - for b in map.all_buildings() { + let mut closest_bldg: FindClosest = FindClosest::new(huge_map.get_bounds()); + for b in huge_map.all_buildings() { closest_bldg.add(b.osm_way_id, b.polygon.points()); } @@ -160,8 +162,8 @@ fn import_parcels(timer: &mut Timer) -> (HashMap, HashMap, ), constraints: PathConstraints, - maybe_huge_map: Option<&(Map, HashMap)>, + maybe_huge_map: Option<&(&Map, HashMap)>, ) -> Option<(TripEndpt, TripEndpt)> { let from_bldg = from .osm_building @@ -223,13 +223,10 @@ impl TripEndpt { } } -fn clip_trips(map: &Map, timer: &mut Timer) -> Vec { - let popdat: PopDat = abstutil::read_binary(abstutil::path_popdat(), timer); +fn clip_trips(map: &Map, popdat: &PopDat, huge_map: &Map, timer: &mut Timer) -> Vec { let maybe_huge_map = if map.get_name() == "huge_seattle" { None } else { - let huge_map = Map::new(abstutil::path_map("huge_seattle"), timer); - let mut huge_osm_id_to_bldg = HashMap::new(); for b in huge_map.all_buildings() { huge_osm_id_to_bldg.insert(b.osm_way_id, b.id); @@ -287,28 +284,33 @@ fn clip_trips(map: &Map, timer: &mut Timer) -> Vec { .collect(); let total_trips = popdat.trips.len(); - let maybe_results: Vec> = timer.parallelize("clip trips", popdat.trips, |orig| { - let (from, to) = TripEndpt::new( - &orig.from, - &orig.to, - map, - &osm_id_to_bldg, - match orig.mode { - TripMode::Walk | TripMode::Transit => { - (&incoming_borders_walking, &outgoing_borders_walking) - } - TripMode::Drive => (&incoming_borders_driving, &outgoing_borders_driving), - TripMode::Bike => (&incoming_borders_biking, &outgoing_borders_biking), - }, - match orig.mode { - TripMode::Walk | TripMode::Transit => PathConstraints::Pedestrian, - TripMode::Drive => PathConstraints::Car, - TripMode::Bike => PathConstraints::Bike, - }, - maybe_huge_map.as_ref(), - )?; - Some(Trip { from, to, orig }) - }); + let maybe_results: Vec> = + timer.parallelize("clip trips", popdat.trips.iter().collect(), |orig| { + let (from, to) = TripEndpt::new( + &orig.from, + &orig.to, + map, + &osm_id_to_bldg, + match orig.mode { + TripMode::Walk | TripMode::Transit => { + (&incoming_borders_walking, &outgoing_borders_walking) + } + TripMode::Drive => (&incoming_borders_driving, &outgoing_borders_driving), + TripMode::Bike => (&incoming_borders_biking, &outgoing_borders_biking), + }, + match orig.mode { + TripMode::Walk | TripMode::Transit => PathConstraints::Pedestrian, + TripMode::Drive => PathConstraints::Car, + TripMode::Bike => PathConstraints::Bike, + }, + maybe_huge_map.as_ref(), + )?; + Some(Trip { + from, + to, + orig: orig.clone(), + }) + }); let trips: Vec = maybe_results.into_iter().flatten().collect(); timer.note(format!( @@ -320,8 +322,13 @@ fn clip_trips(map: &Map, timer: &mut Timer) -> Vec { trips } -pub fn make_weekday_scenario(map: &Map, timer: &mut Timer) -> Scenario { - let trips = clip_trips(map, timer); +pub fn make_weekday_scenario( + map: &Map, + popdat: &PopDat, + huge_map: &Map, + timer: &mut Timer, +) -> Scenario { + let trips = clip_trips(map, popdat, huge_map, timer); let orig_trips = trips.len(); let mut individ_trips: Vec> = Vec::new(); @@ -381,15 +388,17 @@ pub fn make_weekday_scenario(map: &Map, timer: &mut Timer) -> Scenario { .remove_weird_schedules(map) } -pub fn make_weekday_scenario_with_everyone(map: &Map, timer: &mut Timer) -> Scenario { - let popdat: PopDat = abstutil::read_binary(abstutil::path_popdat(), timer); - +pub fn make_weekday_scenario_with_everyone( + map: &Map, + popdat: &PopDat, + timer: &mut Timer, +) -> Scenario { let mut individ_trips: Vec> = Vec::new(); // person -> (trip seq, index into individ_trips) let mut trips_per_person: MultiMap = MultiMap::new(); timer.start_iter("turn Soundcast trips into SpawnTrips", popdat.trips.len()); - for orig_trip in popdat.trips { + for orig_trip in &popdat.trips { timer.next(); let trip = SpawnTrip::Remote { from: OffMapLocation { diff --git a/importer/src/utils.rs b/importer/src/utils.rs index c5893c0e77..26086aceb5 100644 --- a/importer/src/utils.rs +++ b/importer/src/utils.rs @@ -1,3 +1,4 @@ +use abstutil::Timer; use std::path::Path; use std::process::Command; @@ -101,12 +102,13 @@ fn run(cmd: &mut Command) { } // Converts a RawMap to a Map. -pub fn raw_to_map(name: &str, build_ch: bool) { - let mut timer = abstutil::Timer::new(format!("Raw->Map for {}", name)); - let raw: map_model::raw::RawMap = - abstutil::read_binary(abstutil::path_raw_map(name), &mut timer); - let map = map_model::Map::create_from_raw(raw, build_ch, &mut timer); +pub fn raw_to_map(name: &str, build_ch: bool, timer: &mut Timer) -> map_model::Map { + timer.start(format!("Raw->Map for {}", name)); + let raw: map_model::raw::RawMap = abstutil::read_binary(abstutil::path_raw_map(name), timer); + let map = map_model::Map::create_from_raw(raw, build_ch, timer); timer.start("save map"); map.save(); timer.stop("save map"); + timer.stop(format!("Raw->Map for {}", name)); + map }