a-b-street/abstreet
1
1
Fork 0
mirror of https://github.com/a-b-street/abstreet.git synced 2024-12-27 00:12:55 +03:00
Code Issues Projects Releases Wiki Activity

dont needlessly load files in importer

Browse Source
This commit is contained in:
Dustin Carlino 2020-05-17 14:19:45 -07:00
parent f8e89f2de8
commit ee35ff79b9
8 changed files with 129 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/new_city.md
View File

@ -10,6 +10,10 @@ If you have a `.osm` file, you can just run
`./import.sh --oneshot=/absolute/path/to/map.osm`. This tool will generate a new
file in `data/system/maps` that you can then load in the game.
If you're using a binary release, you have to be sure to run the tool from the
`importer/` directory, so that `../data/` exists:
`cd importer; ./importer --oneshot=/absolute/path/to/file.osm`
If you have an Osmosis polygon filter (see below), you can also pass
`--oneshot_clip=/absolute/path/to/clip.poly` to improve the result. You should
first make sure your .osm has been clipped:

1
geom/src/gps.rs
View File

@ -76,7 +76,6 @@ impl LonLat {
.map_err(|err| Error::new(ErrorKind::Other, err))?,
));
}
pts.pop();
Ok(pts)
}
}

81
importer/src/main.rs
View File

@ -4,7 +4,7 @@ mod seattle;
mod soundcast;
mod utils;
use std::thread;
// TODO Might be cleaner to express as a dependency graph?
struct Job {
city: String,
@ -13,7 +13,6 @@ struct Job {
scenario: bool,
scenario_everyone: bool,
seq: bool,
skip_ch: bool,
only_map: Option<String>,
@ -34,8 +33,6 @@ fn main() {
scenario: args.enabled("--scenario"),
// Produce a variation of the weekday scenario including off-map trips.
scenario_everyone: args.enabled("--scenario_everyone"),
// Don't use multiple threads for conversion. Useful when needing to see errors.
seq: args.enabled("--seq"),
// Skip the most expensive step of --map, building contraction hierarchies. The resulting
// map won't be usable for simulation; as soon as you try to pathfind, it'll crash.
skip_ch: args.enabled("--skip_ch"),
@ -75,7 +72,30 @@ fn main() {
abstutil::list_all_objects(format!("../data/input/{}/polygons", job.city))
};
let mut handles = vec![];
let mut timer = abstutil::Timer::new("import map data");
let (maybe_popdat, maybe_huge_map) = if job.scenario || job.scenario_everyone {
assert_eq!(job.city, "seattle");
#[cfg(feature = "scenarios")]
{
let (popdat, huge_map) = seattle::ensure_popdat_exists(&mut timer);
(Some(popdat), Some(huge_map))
}
#[cfg(not(feature = "scenarios"))]
{
panic!(
"Can't do --scenario or --scenario_everyone without the scenarios feature \
compiled in"
);
// Nonsense to make the type-checker work
(Some(true), Some(true))
}
} else {
(None, None)
};
for name in names {
if job.osm_to_raw {
match job.city.as_ref() {
@ -85,42 +105,39 @@ fn main() {
}
}
if job.raw_to_map {
// TODO Bug: if regenerating map and scenario at the same time, this doesn't work.
if job.scenario || job.seq {
utils::raw_to_map(&name, !job.skip_ch);
} else {
let name = name.clone();
let build_ch = !job.skip_ch;
handles.push(thread::spawn(move || {
utils::raw_to_map(&name, build_ch);
}));
}
}
let maybe_map = if job.raw_to_map {
Some(utils::raw_to_map(&name, !job.skip_ch, &mut timer))
} else if job.scenario || job.scenario_everyone {
Some(map_model::Map::new(abstutil::path_map(&name), &mut timer))
} else {
None
};
#[cfg(feature = "scenarios")]
if job.scenario {
assert_eq!(job.city, "seattle");
seattle::ensure_popdat_exists();
let mut timer = abstutil::Timer::new(format!("Scenario for {}", name));
let map = map_model::Map::new(abstutil::path_map(&name), &mut timer);
soundcast::make_weekday_scenario(&map, &mut timer).save();
timer.start(format!("scenario for {}", name));
soundcast::make_weekday_scenario(
maybe_map.as_ref().unwrap(),
maybe_popdat.as_ref().unwrap(),
maybe_huge_map.as_ref().unwrap(),
&mut timer,
)
.save();
timer.stop(format!("scenario for {}", name));
}
#[cfg(feature = "scenarios")]
if job.scenario_everyone {
assert_eq!(job.city, "seattle");
seattle::ensure_popdat_exists();
let mut timer = abstutil::Timer::new(format!("Scenario for {}", name));
let map = map_model::Map::new(abstutil::path_map(&name), &mut timer);
soundcast::make_weekday_scenario_with_everyone(&map, &mut timer).save();
timer.start(format!("scenario_everyone for {}", name));
soundcast::make_weekday_scenario_with_everyone(
maybe_map.as_ref().unwrap(),
maybe_popdat.as_ref().unwrap(),
&mut timer,
)
.save();
timer.stop(format!("scenario_everyone for {}", name));
}
}
for handle in handles {
handle.join().unwrap();
}
}
fn oneshot(osm_path: String, clip: Option<String>) {

19
importer/src/seattle.rs
View File

@ -71,18 +71,25 @@ pub fn osm_to_raw(name: &str) {
// Download and pre-process data needed to generate Seattle scenarios.
#[cfg(feature = "scenarios")]
pub fn ensure_popdat_exists() {
pub fn ensure_popdat_exists(
timer: &mut abstutil::Timer,
) -> (crate::soundcast::PopDat, map_model::Map) {
if abstutil::file_exists(abstutil::path_popdat()) {
println!("- {} exists, not regenerating it", abstutil::path_popdat());
return;
return (
abstutil::read_binary(abstutil::path_popdat(), timer),
map_model::Map::new(abstutil::path_map("huge_seattle"), timer),
);
}
if !abstutil::file_exists(abstutil::path_raw_map("huge_seattle")) {
osm_to_raw("huge_seattle");
}
if !abstutil::file_exists(abstutil::path_map("huge_seattle")) {
crate::utils::raw_to_map("huge_seattle", true);
}
let huge_map = if abstutil::file_exists(abstutil::path_map("huge_seattle")) {
map_model::Map::new(abstutil::path_map("huge_seattle"), timer)
} else {
crate::utils::raw_to_map("huge_seattle", true, timer)
};
crate::soundcast::import_data();
(crate::soundcast::import_data(&huge_map), huge_map)
}

2
importer/src/soundcast/mod.rs
View File

@ -1,5 +1,5 @@
mod popdat;
mod trips;
pub use self::popdat::import_data;
pub use self::popdat::{import_data, PopDat};
pub use self::trips::{make_weekday_scenario, make_weekday_scenario_with_everyone};

24
importer/src/soundcast/popdat.rs
View File

@ -12,15 +12,16 @@ pub struct PopDat {
}
// Extract trip demand data from PSRC's Soundcast outputs.
pub fn import_data() {
pub fn import_data(huge_map: &Map) -> PopDat {
let mut timer = abstutil::Timer::new("creating popdat");
let trips = import_trips(&mut timer);
let trips = import_trips(huge_map, &mut timer);
let popdat = PopDat { trips };
abstutil::write_binary(abstutil::path_popdat(), &popdat);
popdat
}
fn import_trips(timer: &mut Timer) -> Vec<OrigTrip> {
let (parcels, mut keyed_shapes) = import_parcels(timer);
fn import_trips(huge_map: &Map, timer: &mut Timer) -> Vec<OrigTrip> {
let (parcels, mut keyed_shapes) = import_parcels(huge_map, timer);
let mut trips = Vec::new();
let (reader, done) = FileWithProgress::new("../data/input/seattle/trips_2014.csv").unwrap();
@ -109,13 +110,14 @@ fn import_trips(timer: &mut Timer) -> Vec<OrigTrip> {
// TODO Do we also need the zone ID, or is parcel ID globally unique?
// Keyed by parcel ID
fn import_parcels(timer: &mut Timer) -> (HashMap<usize, Endpoint>, HashMap<usize, ExtraShape>) {
let map = Map::new(abstutil::path_map("huge_seattle"), timer);
fn import_parcels(
huge_map: &Map,
timer: &mut Timer,
) -> (HashMap<usize, Endpoint>, HashMap<usize, ExtraShape>) {
// TODO I really just want to do polygon containment with a quadtree. FindClosest only does
// line-string stuff right now, which'll be weird for the last->first pt line and stuff.
let mut closest_bldg: FindClosest<i64> = FindClosest::new(map.get_bounds());
for b in map.all_buildings() {
let mut closest_bldg: FindClosest<i64> = FindClosest::new(huge_map.get_bounds());
for b in huge_map.all_buildings() {
closest_bldg.add(b.osm_way_id, b.polygon.points());
}
@ -160,8 +162,8 @@ fn import_parcels(timer: &mut Timer) -> (HashMap<usize, Endpoint>, HashMap<usize
timer.stop(format!("transform {} points", parcel_metadata.len()));
let bounds = map.get_gps_bounds();
let boundary = map.get_boundary_polygon();
let bounds = huge_map.get_gps_bounds();
let boundary = huge_map.get_boundary_polygon();
let mut result = HashMap::new();
let mut shapes = HashMap::new();
timer.start_iter("finalize parcel output", parcel_metadata.len());

75
importer/src/soundcast/trips.rs
View File

@ -80,7 +80,7 @@ impl TripEndpt {
&Vec<(IntersectionID, LonLat)>,
),
constraints: PathConstraints,
maybe_huge_map: Option<&(Map, HashMap<i64, BuildingID>)>,
maybe_huge_map: Option<&(&Map, HashMap<i64, BuildingID>)>,
) -> Option<(TripEndpt, TripEndpt)> {
let from_bldg = from
.osm_building
@ -223,13 +223,10 @@ impl TripEndpt {
}
}
fn clip_trips(map: &Map, timer: &mut Timer) -> Vec<Trip> {
let popdat: PopDat = abstutil::read_binary(abstutil::path_popdat(), timer);
fn clip_trips(map: &Map, popdat: &PopDat, huge_map: &Map, timer: &mut Timer) -> Vec<Trip> {
let maybe_huge_map = if map.get_name() == "huge_seattle" {
None
} else {
let huge_map = Map::new(abstutil::path_map("huge_seattle"), timer);
let mut huge_osm_id_to_bldg = HashMap::new();
for b in huge_map.all_buildings() {
huge_osm_id_to_bldg.insert(b.osm_way_id, b.id);
@ -287,28 +284,33 @@ fn clip_trips(map: &Map, timer: &mut Timer) -> Vec<Trip> {
.collect();
let total_trips = popdat.trips.len();
let maybe_results: Vec<Option<Trip>> = timer.parallelize("clip trips", popdat.trips, |orig| {
let (from, to) = TripEndpt::new(
&orig.from,
&orig.to,
map,
&osm_id_to_bldg,
match orig.mode {
TripMode::Walk | TripMode::Transit => {
(&incoming_borders_walking, &outgoing_borders_walking)
}
TripMode::Drive => (&incoming_borders_driving, &outgoing_borders_driving),
TripMode::Bike => (&incoming_borders_biking, &outgoing_borders_biking),
},
match orig.mode {
TripMode::Walk | TripMode::Transit => PathConstraints::Pedestrian,
TripMode::Drive => PathConstraints::Car,
TripMode::Bike => PathConstraints::Bike,
},
maybe_huge_map.as_ref(),
)?;
Some(Trip { from, to, orig })
});
let maybe_results: Vec<Option<Trip>> =
timer.parallelize("clip trips", popdat.trips.iter().collect(), |orig| {
let (from, to) = TripEndpt::new(
&orig.from,
&orig.to,
map,
&osm_id_to_bldg,
match orig.mode {
TripMode::Walk | TripMode::Transit => {
(&incoming_borders_walking, &outgoing_borders_walking)
}
TripMode::Drive => (&incoming_borders_driving, &outgoing_borders_driving),
TripMode::Bike => (&incoming_borders_biking, &outgoing_borders_biking),
},
match orig.mode {
TripMode::Walk | TripMode::Transit => PathConstraints::Pedestrian,
TripMode::Drive => PathConstraints::Car,
TripMode::Bike => PathConstraints::Bike,
},
maybe_huge_map.as_ref(),
)?;
Some(Trip {
from,
to,
orig: orig.clone(),
})
});
let trips: Vec<Trip> = maybe_results.into_iter().flatten().collect();
timer.note(format!(
@ -320,8 +322,13 @@ fn clip_trips(map: &Map, timer: &mut Timer) -> Vec<Trip> {
trips
}
pub fn make_weekday_scenario(map: &Map, timer: &mut Timer) -> Scenario {
let trips = clip_trips(map, timer);
pub fn make_weekday_scenario(
map: &Map,
popdat: &PopDat,
huge_map: &Map,
timer: &mut Timer,
) -> Scenario {
let trips = clip_trips(map, popdat, huge_map, timer);
let orig_trips = trips.len();
let mut individ_trips: Vec<Option<IndividTrip>> = Vec::new();
@ -381,15 +388,17 @@ pub fn make_weekday_scenario(map: &Map, timer: &mut Timer) -> Scenario {
.remove_weird_schedules(map)
}
pub fn make_weekday_scenario_with_everyone(map: &Map, timer: &mut Timer) -> Scenario {
let popdat: PopDat = abstutil::read_binary(abstutil::path_popdat(), timer);
pub fn make_weekday_scenario_with_everyone(
map: &Map,
popdat: &PopDat,
timer: &mut Timer,
) -> Scenario {
let mut individ_trips: Vec<Option<IndividTrip>> = Vec::new();
// person -> (trip seq, index into individ_trips)
let mut trips_per_person: MultiMap<OrigPersonID, ((usize, bool, usize), usize)> =
MultiMap::new();
timer.start_iter("turn Soundcast trips into SpawnTrips", popdat.trips.len());
for orig_trip in popdat.trips {
for orig_trip in &popdat.trips {
timer.next();
let trip = SpawnTrip::Remote {
from: OffMapLocation {

12
importer/src/utils.rs
View File

@ -1,3 +1,4 @@
use abstutil::Timer;
use std::path::Path;
use std::process::Command;
@ -101,12 +102,13 @@ fn run(cmd: &mut Command) {
}
// Converts a RawMap to a Map.
pub fn raw_to_map(name: &str, build_ch: bool) {
let mut timer = abstutil::Timer::new(format!("Raw->Map for {}", name));
let raw: map_model::raw::RawMap =
abstutil::read_binary(abstutil::path_raw_map(name), &mut timer);
let map = map_model::Map::create_from_raw(raw, build_ch, &mut timer);
pub fn raw_to_map(name: &str, build_ch: bool, timer: &mut Timer) -> map_model::Map {
timer.start(format!("Raw->Map for {}", name));
let raw: map_model::raw::RawMap = abstutil::read_binary(abstutil::path_raw_map(name), timer);
let map = map_model::Map::create_from_raw(raw, build_ch, timer);
timer.start("save map");
map.save();
timer.stop("save map");
timer.stop(format!("Raw->Map for {}", name));
map
}