From 14f692fc39e31c8009cf6cd91347544d3b15a841 Mon Sep 17 00:00:00 2001 From: Dustin Carlino Date: Tue, 22 Dec 2020 13:38:29 -0800 Subject: [PATCH] Turn more code in the importer into config. Ideally we just have code for all the custom extra import data. #326 Verified there are no changes when importing the affected cities (except for renaming the original OSM input file for Leeds to match the geofabrik source) Woops, and fix updater uploading with the compressed_size_bytes change. --- data/MANIFEST.json | 2 +- importer/config/berlin/cfg.json | 15 ++++++++++ importer/config/leeds/cfg.json | 15 ++++++++++ importer/config/london/cfg.json | 15 ++++++++++ importer/src/berlin.rs | 50 ++++----------------------------- importer/src/generic.rs | 4 ++- importer/src/leeds.rs | 45 +++-------------------------- importer/src/london.rs | 45 +++-------------------------- importer/src/main.rs | 39 ++++++++++++++----------- updater/src/main.rs | 4 ++- 10 files changed, 87 insertions(+), 147 deletions(-) create mode 100644 importer/config/berlin/cfg.json create mode 100644 importer/config/leeds/cfg.json create mode 100644 importer/config/london/cfg.json diff --git a/data/MANIFEST.json b/data/MANIFEST.json index 4979259442..3b503c2979 100644 --- a/data/MANIFEST.json +++ b/data/MANIFEST.json @@ -90,7 +90,7 @@ "uncompressed_size_bytes": 130969890, "compressed_size_bytes": 12269621 }, - "data/input/leeds/osm/west-yorkshire.osm.pbf": { + "data/input/leeds/osm/west-yorkshire-latest.osm.pbf": { "checksum": "0de99323ac548f889860f597704b2910", "uncompressed_size_bytes": 33826429, "compressed_size_bytes": 33785174 diff --git a/importer/config/berlin/cfg.json b/importer/config/berlin/cfg.json new file mode 100644 index 0000000000..d20f0ccfaf --- /dev/null +++ b/importer/config/berlin/cfg.json @@ -0,0 +1,15 @@ +{ + "osm_url": "http://download.geofabrik.de/europe/germany/berlin-latest.osm.pbf", + "map_config": { + "driving_side": "Right", + "bikes_can_use_bus_lanes": true, + "inferred_sidewalks": true + }, + "onstreet_parking": "JustOSM", + "public_offstreet_parking": "None", + "private_offstreet_parking": { + "FixedPerBldg": 3 + }, + "elevation": null, + "include_railroads": true +} diff --git a/importer/config/leeds/cfg.json b/importer/config/leeds/cfg.json new file mode 100644 index 0000000000..bb69bf646a --- /dev/null +++ b/importer/config/leeds/cfg.json @@ -0,0 +1,15 @@ +{ + "osm_url": "https://download.geofabrik.de/europe/great-britain/england/west-yorkshire-latest.osm.pbf", + "map_config": { + "driving_side": "Left", + "bikes_can_use_bus_lanes": false, + "inferred_sidewalks": true + }, + "onstreet_parking": "JustOSM", + "public_offstreet_parking": "None", + "private_offstreet_parking": { + "FixedPerBldg": 3 + }, + "elevation": null, + "include_railroads": true +} diff --git a/importer/config/london/cfg.json b/importer/config/london/cfg.json new file mode 100644 index 0000000000..1e84cff780 --- /dev/null +++ b/importer/config/london/cfg.json @@ -0,0 +1,15 @@ +{ + "osm_url": "http://download.geofabrik.de/europe/great-britain/england/greater-london-latest.osm.pbf", + "map_config": { + "driving_side": "Left", + "bikes_can_use_bus_lanes": true, + "inferred_sidewalks": true + }, + "onstreet_parking": "JustOSM", + "public_offstreet_parking": "None", + "private_offstreet_parking": { + "FixedPerBldg": 10 + }, + "elevation": null, + "include_railroads": true +} diff --git a/importer/src/berlin.rs b/importer/src/berlin.rs index 84cd267d95..b36cf6ffff 100644 --- a/importer/src/berlin.rs +++ b/importer/src/berlin.rs @@ -4,29 +4,21 @@ use rand::{Rng, SeedableRng}; use rand_xorshift::XorShiftRng; use serde::Deserialize; -use abstutil::{prettyprint_usize, MapName, Timer}; +use abstutil::{prettyprint_usize, Timer}; use geom::{Polygon, Ring}; use kml::ExtraShapes; +use map_model::raw::RawMap; use map_model::BuildingType; use crate::configuration::ImporterConfiguration; -use crate::utils::{download, download_kml, osmconvert}; +use crate::utils::{download, download_kml}; -fn input(config: &ImporterConfiguration, timer: &mut Timer) { - download( - config, - "input/berlin/osm/berlin-latest.osm.pbf", - "http://download.geofabrik.de/europe/germany/berlin-latest.osm.pbf", - ); - - let bounds = geom::GPSBounds::from( - geom::LonLat::read_osmosis_polygon("importer/config/berlin/center.poly").unwrap(), - ); +pub fn import_extra_data(map: &RawMap, config: &ImporterConfiguration, timer: &mut Timer) { // From https://data.technologiestiftung-berlin.de/dataset/lor_planungsgraeume/en download_kml( "input/berlin/planning_areas.bin", "https://tsb-opendata.s3.eu-central-1.amazonaws.com/lor_planungsgraeume/lor_planungsraeume.kml", - &bounds, + &map.gps_bounds, // Keep partly out-of-bounds polygons false, timer @@ -48,38 +40,6 @@ fn input(config: &ImporterConfiguration, timer: &mut Timer) { ); } -pub fn osm_to_raw(name: &str, timer: &mut Timer, config: &ImporterConfiguration) { - input(config, timer); - osmconvert( - "input/berlin/osm/berlin-latest.osm.pbf", - format!("importer/config/berlin/{}.poly", name), - format!("input/berlin/osm/{}.osm", name), - config, - ); - - let map = convert_osm::convert( - convert_osm::Options { - osm_input: abstutil::path(format!("input/berlin/osm/{}.osm", name)), - name: MapName::new("berlin", name), - - clip: Some(format!("importer/config/berlin/{}.poly", name)), - map_config: map_model::MapConfig { - driving_side: map_model::DrivingSide::Right, - bikes_can_use_bus_lanes: true, - inferred_sidewalks: true, - }, - - onstreet_parking: convert_osm::OnstreetParking::JustOSM, - public_offstreet_parking: convert_osm::PublicOffstreetParking::None, - private_offstreet_parking: convert_osm::PrivateOffstreetParking::FixedPerBldg(3), - elevation: None, - include_railroads: true, - }, - timer, - ); - map.save(); -} - // Modify the filtered KML of planning areas with the number of residents from a different dataset. fn correlate_population(kml_path: &str, csv_path: &str, timer: &mut Timer) { let mut shapes = abstutil::read_binary::(kml_path.to_string(), timer); diff --git a/importer/src/generic.rs b/importer/src/generic.rs index 923f920ff3..2a96bf814b 100644 --- a/importer/src/generic.rs +++ b/importer/src/generic.rs @@ -1,6 +1,7 @@ use serde::{Deserialize, Serialize}; use abstutil::MapName; +use map_model::raw::RawMap; use crate::configuration::ImporterConfiguration; use crate::utils::{download, osmconvert}; @@ -33,7 +34,7 @@ impl GenericCityImporter { name: MapName, timer: &mut abstutil::Timer, config: &ImporterConfiguration, - ) { + ) -> RawMap { let local_osm_file = format!( "input/{}/osm/{}", name.city, @@ -69,5 +70,6 @@ impl GenericCityImporter { timer, ); map.save(); + map } } diff --git a/importer/src/leeds.rs b/importer/src/leeds.rs index 077cc44d5e..17fe0c4424 100644 --- a/importer/src/leeds.rs +++ b/importer/src/leeds.rs @@ -1,51 +1,14 @@ -use abstutil::MapName; +use abstutil::Timer; +use map_model::raw::RawMap; use crate::configuration::ImporterConfiguration; -use crate::utils::{download, osmconvert}; - -fn input(config: &ImporterConfiguration) { - download( - config, - "input/leeds/osm/west-yorkshire.osm.pbf", - "https://download.geofabrik.de/europe/great-britain/england/west-yorkshire-latest.osm.pbf", - ); +use crate::utils::download; +pub fn import_extra_data(map: &RawMap, config: &ImporterConfiguration, timer: &mut Timer) { download( config, "input/leeds/Road Safety Data - Accidents 2019.csv", "http://data.dft.gov.uk.s3.amazonaws.com/road-accidents-safety-data/DfTRoadSafety_Accidents_2019.zip"); -} - -pub fn osm_to_raw(name: &str, timer: &mut abstutil::Timer, config: &ImporterConfiguration) { - input(config); - osmconvert( - "input/leeds/osm/west-yorkshire.osm.pbf", - format!("importer/config/leeds/{}.poly", name), - format!("input/leeds/osm/{}.osm", name), - config, - ); - - let map = convert_osm::convert( - convert_osm::Options { - osm_input: abstutil::path(format!("input/leeds/osm/{}.osm", name)), - name: MapName::new("leeds", name), - - clip: Some(format!("importer/config/leeds/{}.poly", name)), - map_config: map_model::MapConfig { - driving_side: map_model::DrivingSide::Left, - bikes_can_use_bus_lanes: false, - inferred_sidewalks: true, - }, - - onstreet_parking: convert_osm::OnstreetParking::JustOSM, - public_offstreet_parking: convert_osm::PublicOffstreetParking::None, - private_offstreet_parking: convert_osm::PrivateOffstreetParking::FixedPerBldg(3), - elevation: None, - include_railroads: true, - }, - timer, - ); - map.save(); // Always do this, it's idempotent and fast let shapes = kml::ExtraShapes::load_csv( diff --git a/importer/src/london.rs b/importer/src/london.rs index 5149e49b4d..f09c52f4f4 100644 --- a/importer/src/london.rs +++ b/importer/src/london.rs @@ -1,51 +1,14 @@ -use abstutil::MapName; +use abstutil::Timer; +use map_model::raw::RawMap; use crate::configuration::ImporterConfiguration; -use crate::utils::{download, osmconvert}; - -fn input(config: &ImporterConfiguration) { - download( - config, - "input/london/osm/greater-london-latest.osm.pbf", - "http://download.geofabrik.de/europe/great-britain/england/greater-london-latest.osm.pbf", - ); +use crate::utils::download; +pub fn import_extra_data(map: &RawMap, config: &ImporterConfiguration, timer: &mut Timer) { download( config, "input/london/Road Safety Data - Accidents 2019.csv", "http://data.dft.gov.uk.s3.amazonaws.com/road-accidents-safety-data/DfTRoadSafety_Accidents_2019.zip"); -} - -pub fn osm_to_raw(name: &str, timer: &mut abstutil::Timer, config: &ImporterConfiguration) { - input(config); - osmconvert( - "input/london/osm/greater-london-latest.osm.pbf", - format!("importer/config/london/{}.poly", name), - format!("input/london/osm/{}.osm", name), - config, - ); - - let map = convert_osm::convert( - convert_osm::Options { - osm_input: abstutil::path(format!("input/london/osm/{}.osm", name)), - name: MapName::new("london", name), - - clip: Some(format!("importer/config/london/{}.poly", name)), - map_config: map_model::MapConfig { - driving_side: map_model::DrivingSide::Left, - bikes_can_use_bus_lanes: true, - inferred_sidewalks: true, - }, - - onstreet_parking: convert_osm::OnstreetParking::JustOSM, - public_offstreet_parking: convert_osm::PublicOffstreetParking::None, - private_offstreet_parking: convert_osm::PrivateOffstreetParking::FixedPerBldg(10), - elevation: None, - include_railroads: true, - }, - timer, - ); - map.save(); // Always do this, it's idempotent and fast let shapes = kml::ExtraShapes::load_csv( diff --git a/importer/src/main.rs b/importer/src/main.rs index 97a8117228..422c46d127 100644 --- a/importer/src/main.rs +++ b/importer/src/main.rs @@ -137,24 +137,29 @@ fn main() { for name in names { if job.osm_to_raw { - match job.city.as_ref() { - "berlin" => berlin::osm_to_raw(&name, &mut timer, &config), - "leeds" => leeds::osm_to_raw(&name, &mut timer, &config), - "london" => london::osm_to_raw(&name, &mut timer, &config), - "seattle" => seattle::osm_to_raw(&name, &mut timer, &config), - x => { - match abstutil::maybe_read_json::( - format!("importer/config/{}/cfg.json", x), - &mut timer, - ) { - Ok(city_cfg) => { - city_cfg.osm_to_raw(MapName::new(x, &name), &mut timer, &config); - } - Err(err) => { - panic!("Can't import city {}: {}", x, err); - } - } + // Still special-cased + if job.city == "seattle" { + seattle::osm_to_raw(&name, &mut timer, &config); + continue; + } + + let raw = match abstutil::maybe_read_json::( + format!("importer/config/{}/cfg.json", job.city), + &mut timer, + ) { + Ok(city_cfg) => { + city_cfg.osm_to_raw(MapName::new(&job.city, &name), &mut timer, &config) } + Err(err) => { + panic!("Can't import city {}: {}", job.city, err); + } + }; + + match job.city.as_ref() { + "berlin" => berlin::import_extra_data(&raw, &config, &mut timer), + "leeds" => leeds::import_extra_data(&raw, &config, &mut timer), + "london" => london::import_extra_data(&raw, &config, &mut timer), + _ => {} } } let name = MapName::new(&job.city, &name); diff --git a/updater/src/main.rs b/updater/src/main.rs index 641c256344..f7c5d73944 100644 --- a/updater/src/main.rs +++ b/updater/src/main.rs @@ -126,8 +126,10 @@ fn upload(version: String) { std::io::copy(&mut input, &mut encoder).unwrap(); encoder.finish().unwrap(); } - entry.compressed_size_bytes = std::fs::metadata(&remote_path).unwrap().len() as usize; } + // Always do this -- even if nothing changed, compressed_size_bytes isn't filled out by + // generate_manifest. + entry.compressed_size_bytes = std::fs::metadata(&remote_path).unwrap().len() as usize; } abstutil::write_json(format!("{}/MANIFEST.json", remote_base), &local);