diff --git a/data/MANIFEST.json b/data/MANIFEST.json index 4979259442..3b503c2979 100644 --- a/data/MANIFEST.json +++ b/data/MANIFEST.json @@ -90,7 +90,7 @@ "uncompressed_size_bytes": 130969890, "compressed_size_bytes": 12269621 }, - "data/input/leeds/osm/west-yorkshire.osm.pbf": { + "data/input/leeds/osm/west-yorkshire-latest.osm.pbf": { "checksum": "0de99323ac548f889860f597704b2910", "uncompressed_size_bytes": 33826429, "compressed_size_bytes": 33785174 diff --git a/importer/config/berlin/cfg.json b/importer/config/berlin/cfg.json new file mode 100644 index 0000000000..d20f0ccfaf --- /dev/null +++ b/importer/config/berlin/cfg.json @@ -0,0 +1,15 @@ +{ + "osm_url": "http://download.geofabrik.de/europe/germany/berlin-latest.osm.pbf", + "map_config": { + "driving_side": "Right", + "bikes_can_use_bus_lanes": true, + "inferred_sidewalks": true + }, + "onstreet_parking": "JustOSM", + "public_offstreet_parking": "None", + "private_offstreet_parking": { + "FixedPerBldg": 3 + }, + "elevation": null, + "include_railroads": true +} diff --git a/importer/config/leeds/cfg.json b/importer/config/leeds/cfg.json new file mode 100644 index 0000000000..bb69bf646a --- /dev/null +++ b/importer/config/leeds/cfg.json @@ -0,0 +1,15 @@ +{ + "osm_url": "https://download.geofabrik.de/europe/great-britain/england/west-yorkshire-latest.osm.pbf", + "map_config": { + "driving_side": "Left", + "bikes_can_use_bus_lanes": false, + "inferred_sidewalks": true + }, + "onstreet_parking": "JustOSM", + "public_offstreet_parking": "None", + "private_offstreet_parking": { + "FixedPerBldg": 3 + }, + "elevation": null, + "include_railroads": true +} diff --git a/importer/config/london/cfg.json b/importer/config/london/cfg.json new file mode 100644 index 0000000000..1e84cff780 --- /dev/null +++ b/importer/config/london/cfg.json @@ -0,0 +1,15 @@ +{ + "osm_url": "http://download.geofabrik.de/europe/great-britain/england/greater-london-latest.osm.pbf", + "map_config": { + "driving_side": "Left", + "bikes_can_use_bus_lanes": true, + "inferred_sidewalks": true + }, + "onstreet_parking": "JustOSM", + "public_offstreet_parking": "None", + "private_offstreet_parking": { + "FixedPerBldg": 10 + }, + "elevation": null, + "include_railroads": true +} diff --git a/importer/src/berlin.rs b/importer/src/berlin.rs index 84cd267d95..b36cf6ffff 100644 --- a/importer/src/berlin.rs +++ b/importer/src/berlin.rs @@ -4,29 +4,21 @@ use rand::{Rng, SeedableRng}; use rand_xorshift::XorShiftRng; use serde::Deserialize; -use abstutil::{prettyprint_usize, MapName, Timer}; +use abstutil::{prettyprint_usize, Timer}; use geom::{Polygon, Ring}; use kml::ExtraShapes; +use map_model::raw::RawMap; use map_model::BuildingType; use crate::configuration::ImporterConfiguration; -use crate::utils::{download, download_kml, osmconvert}; +use crate::utils::{download, download_kml}; -fn input(config: &ImporterConfiguration, timer: &mut Timer) { - download( - config, - "input/berlin/osm/berlin-latest.osm.pbf", - "http://download.geofabrik.de/europe/germany/berlin-latest.osm.pbf", - ); - - let bounds = geom::GPSBounds::from( - geom::LonLat::read_osmosis_polygon("importer/config/berlin/center.poly").unwrap(), - ); +pub fn import_extra_data(map: &RawMap, config: &ImporterConfiguration, timer: &mut Timer) { // From https://data.technologiestiftung-berlin.de/dataset/lor_planungsgraeume/en download_kml( "input/berlin/planning_areas.bin", "https://tsb-opendata.s3.eu-central-1.amazonaws.com/lor_planungsgraeume/lor_planungsraeume.kml", - &bounds, + &map.gps_bounds, // Keep partly out-of-bounds polygons false, timer @@ -48,38 +40,6 @@ fn input(config: &ImporterConfiguration, timer: &mut Timer) { ); } -pub fn osm_to_raw(name: &str, timer: &mut Timer, config: &ImporterConfiguration) { - input(config, timer); - osmconvert( - "input/berlin/osm/berlin-latest.osm.pbf", - format!("importer/config/berlin/{}.poly", name), - format!("input/berlin/osm/{}.osm", name), - config, - ); - - let map = convert_osm::convert( - convert_osm::Options { - osm_input: abstutil::path(format!("input/berlin/osm/{}.osm", name)), - name: MapName::new("berlin", name), - - clip: Some(format!("importer/config/berlin/{}.poly", name)), - map_config: map_model::MapConfig { - driving_side: map_model::DrivingSide::Right, - bikes_can_use_bus_lanes: true, - inferred_sidewalks: true, - }, - - onstreet_parking: convert_osm::OnstreetParking::JustOSM, - public_offstreet_parking: convert_osm::PublicOffstreetParking::None, - private_offstreet_parking: convert_osm::PrivateOffstreetParking::FixedPerBldg(3), - elevation: None, - include_railroads: true, - }, - timer, - ); - map.save(); -} - // Modify the filtered KML of planning areas with the number of residents from a different dataset. fn correlate_population(kml_path: &str, csv_path: &str, timer: &mut Timer) { let mut shapes = abstutil::read_binary::(kml_path.to_string(), timer); diff --git a/importer/src/generic.rs b/importer/src/generic.rs index 923f920ff3..2a96bf814b 100644 --- a/importer/src/generic.rs +++ b/importer/src/generic.rs @@ -1,6 +1,7 @@ use serde::{Deserialize, Serialize}; use abstutil::MapName; +use map_model::raw::RawMap; use crate::configuration::ImporterConfiguration; use crate::utils::{download, osmconvert}; @@ -33,7 +34,7 @@ impl GenericCityImporter { name: MapName, timer: &mut abstutil::Timer, config: &ImporterConfiguration, - ) { + ) -> RawMap { let local_osm_file = format!( "input/{}/osm/{}", name.city, @@ -69,5 +70,6 @@ impl GenericCityImporter { timer, ); map.save(); + map } } diff --git a/importer/src/leeds.rs b/importer/src/leeds.rs index 077cc44d5e..17fe0c4424 100644 --- a/importer/src/leeds.rs +++ b/importer/src/leeds.rs @@ -1,51 +1,14 @@ -use abstutil::MapName; +use abstutil::Timer; +use map_model::raw::RawMap; use crate::configuration::ImporterConfiguration; -use crate::utils::{download, osmconvert}; - -fn input(config: &ImporterConfiguration) { - download( - config, - "input/leeds/osm/west-yorkshire.osm.pbf", - "https://download.geofabrik.de/europe/great-britain/england/west-yorkshire-latest.osm.pbf", - ); +use crate::utils::download; +pub fn import_extra_data(map: &RawMap, config: &ImporterConfiguration, timer: &mut Timer) { download( config, "input/leeds/Road Safety Data - Accidents 2019.csv", "http://data.dft.gov.uk.s3.amazonaws.com/road-accidents-safety-data/DfTRoadSafety_Accidents_2019.zip"); -} - -pub fn osm_to_raw(name: &str, timer: &mut abstutil::Timer, config: &ImporterConfiguration) { - input(config); - osmconvert( - "input/leeds/osm/west-yorkshire.osm.pbf", - format!("importer/config/leeds/{}.poly", name), - format!("input/leeds/osm/{}.osm", name), - config, - ); - - let map = convert_osm::convert( - convert_osm::Options { - osm_input: abstutil::path(format!("input/leeds/osm/{}.osm", name)), - name: MapName::new("leeds", name), - - clip: Some(format!("importer/config/leeds/{}.poly", name)), - map_config: map_model::MapConfig { - driving_side: map_model::DrivingSide::Left, - bikes_can_use_bus_lanes: false, - inferred_sidewalks: true, - }, - - onstreet_parking: convert_osm::OnstreetParking::JustOSM, - public_offstreet_parking: convert_osm::PublicOffstreetParking::None, - private_offstreet_parking: convert_osm::PrivateOffstreetParking::FixedPerBldg(3), - elevation: None, - include_railroads: true, - }, - timer, - ); - map.save(); // Always do this, it's idempotent and fast let shapes = kml::ExtraShapes::load_csv( diff --git a/importer/src/london.rs b/importer/src/london.rs index 5149e49b4d..f09c52f4f4 100644 --- a/importer/src/london.rs +++ b/importer/src/london.rs @@ -1,51 +1,14 @@ -use abstutil::MapName; +use abstutil::Timer; +use map_model::raw::RawMap; use crate::configuration::ImporterConfiguration; -use crate::utils::{download, osmconvert}; - -fn input(config: &ImporterConfiguration) { - download( - config, - "input/london/osm/greater-london-latest.osm.pbf", - "http://download.geofabrik.de/europe/great-britain/england/greater-london-latest.osm.pbf", - ); +use crate::utils::download; +pub fn import_extra_data(map: &RawMap, config: &ImporterConfiguration, timer: &mut Timer) { download( config, "input/london/Road Safety Data - Accidents 2019.csv", "http://data.dft.gov.uk.s3.amazonaws.com/road-accidents-safety-data/DfTRoadSafety_Accidents_2019.zip"); -} - -pub fn osm_to_raw(name: &str, timer: &mut abstutil::Timer, config: &ImporterConfiguration) { - input(config); - osmconvert( - "input/london/osm/greater-london-latest.osm.pbf", - format!("importer/config/london/{}.poly", name), - format!("input/london/osm/{}.osm", name), - config, - ); - - let map = convert_osm::convert( - convert_osm::Options { - osm_input: abstutil::path(format!("input/london/osm/{}.osm", name)), - name: MapName::new("london", name), - - clip: Some(format!("importer/config/london/{}.poly", name)), - map_config: map_model::MapConfig { - driving_side: map_model::DrivingSide::Left, - bikes_can_use_bus_lanes: true, - inferred_sidewalks: true, - }, - - onstreet_parking: convert_osm::OnstreetParking::JustOSM, - public_offstreet_parking: convert_osm::PublicOffstreetParking::None, - private_offstreet_parking: convert_osm::PrivateOffstreetParking::FixedPerBldg(10), - elevation: None, - include_railroads: true, - }, - timer, - ); - map.save(); // Always do this, it's idempotent and fast let shapes = kml::ExtraShapes::load_csv( diff --git a/importer/src/main.rs b/importer/src/main.rs index 97a8117228..422c46d127 100644 --- a/importer/src/main.rs +++ b/importer/src/main.rs @@ -137,24 +137,29 @@ fn main() { for name in names { if job.osm_to_raw { - match job.city.as_ref() { - "berlin" => berlin::osm_to_raw(&name, &mut timer, &config), - "leeds" => leeds::osm_to_raw(&name, &mut timer, &config), - "london" => london::osm_to_raw(&name, &mut timer, &config), - "seattle" => seattle::osm_to_raw(&name, &mut timer, &config), - x => { - match abstutil::maybe_read_json::( - format!("importer/config/{}/cfg.json", x), - &mut timer, - ) { - Ok(city_cfg) => { - city_cfg.osm_to_raw(MapName::new(x, &name), &mut timer, &config); - } - Err(err) => { - panic!("Can't import city {}: {}", x, err); - } - } + // Still special-cased + if job.city == "seattle" { + seattle::osm_to_raw(&name, &mut timer, &config); + continue; + } + + let raw = match abstutil::maybe_read_json::( + format!("importer/config/{}/cfg.json", job.city), + &mut timer, + ) { + Ok(city_cfg) => { + city_cfg.osm_to_raw(MapName::new(&job.city, &name), &mut timer, &config) } + Err(err) => { + panic!("Can't import city {}: {}", job.city, err); + } + }; + + match job.city.as_ref() { + "berlin" => berlin::import_extra_data(&raw, &config, &mut timer), + "leeds" => leeds::import_extra_data(&raw, &config, &mut timer), + "london" => london::import_extra_data(&raw, &config, &mut timer), + _ => {} } } let name = MapName::new(&job.city, &name); diff --git a/updater/src/main.rs b/updater/src/main.rs index 641c256344..f7c5d73944 100644 --- a/updater/src/main.rs +++ b/updater/src/main.rs @@ -126,8 +126,10 @@ fn upload(version: String) { std::io::copy(&mut input, &mut encoder).unwrap(); encoder.finish().unwrap(); } - entry.compressed_size_bytes = std::fs::metadata(&remote_path).unwrap().len() as usize; } + // Always do this -- even if nothing changed, compressed_size_bytes isn't filled out by + // generate_manifest. + entry.compressed_size_bytes = std::fs::metadata(&remote_path).unwrap().len() as usize; } abstutil::write_json(format!("{}/MANIFEST.json", remote_base), &local);