Turn more code in the importer into config. Ideally we just have code

for all the custom extra import data. #326

Verified there are no changes when importing the affected cities (except
for renaming the original OSM input file for Leeds to match the
geofabrik source)

Woops, and fix updater uploading with the compressed_size_bytes change.

data/MANIFEST.json

@@ -90,7 +90,7 @@
       "uncompressed_size_bytes": 130969890,
       "compressed_size_bytes": 12269621
     },
-    "data/input/leeds/osm/west-yorkshire.osm.pbf": {
+    "data/input/leeds/osm/west-yorkshire-latest.osm.pbf": {
       "checksum": "0de99323ac548f889860f597704b2910",
       "uncompressed_size_bytes": 33826429,
       "compressed_size_bytes": 33785174

importer/config/berlin/cfg.json

@@ -0,0 +1,15 @@
+{
+  "osm_url": "http://download.geofabrik.de/europe/germany/berlin-latest.osm.pbf",
+  "map_config": {
+    "driving_side": "Right",
+    "bikes_can_use_bus_lanes": true,
+    "inferred_sidewalks": true
+  },
+  "onstreet_parking": "JustOSM",
+  "public_offstreet_parking": "None",
+  "private_offstreet_parking": {
+    "FixedPerBldg": 3
+  },
+  "elevation": null,
+  "include_railroads": true
+}

importer/config/leeds/cfg.json

@@ -0,0 +1,15 @@
+{
+  "osm_url": "https://download.geofabrik.de/europe/great-britain/england/west-yorkshire-latest.osm.pbf",
+  "map_config": {
+    "driving_side": "Left",
+    "bikes_can_use_bus_lanes": false,
+    "inferred_sidewalks": true
+  },
+  "onstreet_parking": "JustOSM",
+  "public_offstreet_parking": "None",
+  "private_offstreet_parking": {
+    "FixedPerBldg": 3
+  },
+  "elevation": null,
+  "include_railroads": true
+}

importer/config/london/cfg.json

@@ -0,0 +1,15 @@
+{
+  "osm_url": "http://download.geofabrik.de/europe/great-britain/england/greater-london-latest.osm.pbf",
+  "map_config": {
+    "driving_side": "Left",
+    "bikes_can_use_bus_lanes": true,
+    "inferred_sidewalks": true
+  },
+  "onstreet_parking": "JustOSM",
+  "public_offstreet_parking": "None",
+  "private_offstreet_parking": {
+    "FixedPerBldg": 10
+  },
+  "elevation": null,
+  "include_railroads": true
+}

importer/src/berlin.rs

@@ -4,29 +4,21 @@ use rand::{Rng, SeedableRng};
 use rand_xorshift::XorShiftRng;
 use serde::Deserialize;
 
-use abstutil::{prettyprint_usize, MapName, Timer};
+use abstutil::{prettyprint_usize, Timer};
 use geom::{Polygon, Ring};
 use kml::ExtraShapes;
+use map_model::raw::RawMap;
 use map_model::BuildingType;
 
 use crate::configuration::ImporterConfiguration;
-use crate::utils::{download, download_kml, osmconvert};
+use crate::utils::{download, download_kml};
 
-fn input(config: &ImporterConfiguration, timer: &mut Timer) {
-    download(
-        config,
-        "input/berlin/osm/berlin-latest.osm.pbf",
-        "http://download.geofabrik.de/europe/germany/berlin-latest.osm.pbf",
-    );
-
-    let bounds = geom::GPSBounds::from(
-        geom::LonLat::read_osmosis_polygon("importer/config/berlin/center.poly").unwrap(),
-    );
+pub fn import_extra_data(map: &RawMap, config: &ImporterConfiguration, timer: &mut Timer) {
     // From https://data.technologiestiftung-berlin.de/dataset/lor_planungsgraeume/en
     download_kml(
         "input/berlin/planning_areas.bin",
         "https://tsb-opendata.s3.eu-central-1.amazonaws.com/lor_planungsgraeume/lor_planungsraeume.kml",
-        &bounds,
+        &map.gps_bounds,
         // Keep partly out-of-bounds polygons
         false,
         timer
@@ -48,38 +40,6 @@ fn input(config: &ImporterConfiguration, timer: &mut Timer) {
     );
 }
 
-pub fn osm_to_raw(name: &str, timer: &mut Timer, config: &ImporterConfiguration) {
-    input(config, timer);
-    osmconvert(
-        "input/berlin/osm/berlin-latest.osm.pbf",
-        format!("importer/config/berlin/{}.poly", name),
-        format!("input/berlin/osm/{}.osm", name),
-        config,
-    );
-
-    let map = convert_osm::convert(
-        convert_osm::Options {
-            osm_input: abstutil::path(format!("input/berlin/osm/{}.osm", name)),
-            name: MapName::new("berlin", name),
-
-            clip: Some(format!("importer/config/berlin/{}.poly", name)),
-            map_config: map_model::MapConfig {
-                driving_side: map_model::DrivingSide::Right,
-                bikes_can_use_bus_lanes: true,
-                inferred_sidewalks: true,
-            },
-
-            onstreet_parking: convert_osm::OnstreetParking::JustOSM,
-            public_offstreet_parking: convert_osm::PublicOffstreetParking::None,
-            private_offstreet_parking: convert_osm::PrivateOffstreetParking::FixedPerBldg(3),
-            elevation: None,
-            include_railroads: true,
-        },
-        timer,
-    );
-    map.save();
-}
-
 // Modify the filtered KML of planning areas with the number of residents from a different dataset.
 fn correlate_population(kml_path: &str, csv_path: &str, timer: &mut Timer) {
     let mut shapes = abstutil::read_binary::<ExtraShapes>(kml_path.to_string(), timer);

importer/src/generic.rs

@@ -1,6 +1,7 @@
 use serde::{Deserialize, Serialize};
 
 use abstutil::MapName;
+use map_model::raw::RawMap;
 
 use crate::configuration::ImporterConfiguration;
 use crate::utils::{download, osmconvert};
@@ -33,7 +34,7 @@ impl GenericCityImporter {
         name: MapName,
         timer: &mut abstutil::Timer,
         config: &ImporterConfiguration,
-    ) {
+    ) -> RawMap {
         let local_osm_file = format!(
             "input/{}/osm/{}",
             name.city,
@@ -69,5 +70,6 @@ impl GenericCityImporter {
             timer,
         );
         map.save();
+        map
     }
 }

importer/src/leeds.rs

@@ -1,51 +1,14 @@
-use abstutil::MapName;
+use abstutil::Timer;
+use map_model::raw::RawMap;
 
 use crate::configuration::ImporterConfiguration;
-use crate::utils::{download, osmconvert};
-
-fn input(config: &ImporterConfiguration) {
-    download(
-        config,
-        "input/leeds/osm/west-yorkshire.osm.pbf",
-        "https://download.geofabrik.de/europe/great-britain/england/west-yorkshire-latest.osm.pbf",
-    );
+use crate::utils::download;
 
+pub fn import_extra_data(map: &RawMap, config: &ImporterConfiguration, timer: &mut Timer) {
     download(
         config,
         "input/leeds/Road Safety Data - Accidents 2019.csv",
         "http://data.dft.gov.uk.s3.amazonaws.com/road-accidents-safety-data/DfTRoadSafety_Accidents_2019.zip");
-}
-
-pub fn osm_to_raw(name: &str, timer: &mut abstutil::Timer, config: &ImporterConfiguration) {
-    input(config);
-    osmconvert(
-        "input/leeds/osm/west-yorkshire.osm.pbf",
-        format!("importer/config/leeds/{}.poly", name),
-        format!("input/leeds/osm/{}.osm", name),
-        config,
-    );
-
-    let map = convert_osm::convert(
-        convert_osm::Options {
-            osm_input: abstutil::path(format!("input/leeds/osm/{}.osm", name)),
-            name: MapName::new("leeds", name),
-
-            clip: Some(format!("importer/config/leeds/{}.poly", name)),
-            map_config: map_model::MapConfig {
-                driving_side: map_model::DrivingSide::Left,
-                bikes_can_use_bus_lanes: false,
-                inferred_sidewalks: true,
-            },
-
-            onstreet_parking: convert_osm::OnstreetParking::JustOSM,
-            public_offstreet_parking: convert_osm::PublicOffstreetParking::None,
-            private_offstreet_parking: convert_osm::PrivateOffstreetParking::FixedPerBldg(3),
-            elevation: None,
-            include_railroads: true,
-        },
-        timer,
-    );
-    map.save();
 
     // Always do this, it's idempotent and fast
     let shapes = kml::ExtraShapes::load_csv(

importer/src/london.rs

@@ -1,51 +1,14 @@
-use abstutil::MapName;
+use abstutil::Timer;
+use map_model::raw::RawMap;
 
 use crate::configuration::ImporterConfiguration;
-use crate::utils::{download, osmconvert};
-
-fn input(config: &ImporterConfiguration) {
-    download(
-        config,
-        "input/london/osm/greater-london-latest.osm.pbf",
-        "http://download.geofabrik.de/europe/great-britain/england/greater-london-latest.osm.pbf",
-    );
+use crate::utils::download;
 
+pub fn import_extra_data(map: &RawMap, config: &ImporterConfiguration, timer: &mut Timer) {
     download(
         config,
         "input/london/Road Safety Data - Accidents 2019.csv",
         "http://data.dft.gov.uk.s3.amazonaws.com/road-accidents-safety-data/DfTRoadSafety_Accidents_2019.zip");
-}
-
-pub fn osm_to_raw(name: &str, timer: &mut abstutil::Timer, config: &ImporterConfiguration) {
-    input(config);
-    osmconvert(
-        "input/london/osm/greater-london-latest.osm.pbf",
-        format!("importer/config/london/{}.poly", name),
-        format!("input/london/osm/{}.osm", name),
-        config,
-    );
-
-    let map = convert_osm::convert(
-        convert_osm::Options {
-            osm_input: abstutil::path(format!("input/london/osm/{}.osm", name)),
-            name: MapName::new("london", name),
-
-            clip: Some(format!("importer/config/london/{}.poly", name)),
-            map_config: map_model::MapConfig {
-                driving_side: map_model::DrivingSide::Left,
-                bikes_can_use_bus_lanes: true,
-                inferred_sidewalks: true,
-            },
-
-            onstreet_parking: convert_osm::OnstreetParking::JustOSM,
-            public_offstreet_parking: convert_osm::PublicOffstreetParking::None,
-            private_offstreet_parking: convert_osm::PrivateOffstreetParking::FixedPerBldg(10),
-            elevation: None,
-            include_railroads: true,
-        },
-        timer,
-    );
-    map.save();
 
     // Always do this, it's idempotent and fast
     let shapes = kml::ExtraShapes::load_csv(

importer/src/main.rs

@@ -137,24 +137,29 @@ fn main() {
 
     for name in names {
         if job.osm_to_raw {
-            match job.city.as_ref() {
-                "berlin" => berlin::osm_to_raw(&name, &mut timer, &config),
-                "leeds" => leeds::osm_to_raw(&name, &mut timer, &config),
-                "london" => london::osm_to_raw(&name, &mut timer, &config),
-                "seattle" => seattle::osm_to_raw(&name, &mut timer, &config),
-                x => {
-                    match abstutil::maybe_read_json::<generic::GenericCityImporter>(
-                        format!("importer/config/{}/cfg.json", x),
-                        &mut timer,
-                    ) {
-                        Ok(city_cfg) => {
-                            city_cfg.osm_to_raw(MapName::new(x, &name), &mut timer, &config);
-                        }
-                        Err(err) => {
-                            panic!("Can't import city {}: {}", x, err);
-                        }
-                    }
+            // Still special-cased
+            if job.city == "seattle" {
+                seattle::osm_to_raw(&name, &mut timer, &config);
+                continue;
+            }
+
+            let raw = match abstutil::maybe_read_json::<generic::GenericCityImporter>(
+                format!("importer/config/{}/cfg.json", job.city),
+                &mut timer,
+            ) {
+                Ok(city_cfg) => {
+                    city_cfg.osm_to_raw(MapName::new(&job.city, &name), &mut timer, &config)
                 }
+                Err(err) => {
+                    panic!("Can't import city {}: {}", job.city, err);
+                }
+            };
+
+            match job.city.as_ref() {
+                "berlin" => berlin::import_extra_data(&raw, &config, &mut timer),
+                "leeds" => leeds::import_extra_data(&raw, &config, &mut timer),
+                "london" => london::import_extra_data(&raw, &config, &mut timer),
+                _ => {}
             }
         }
         let name = MapName::new(&job.city, &name);

updater/src/main.rs

@@ -126,8 +126,10 @@ fn upload(version: String) {
                 std::io::copy(&mut input, &mut encoder).unwrap();
                 encoder.finish().unwrap();
             }
-            entry.compressed_size_bytes = std::fs::metadata(&remote_path).unwrap().len() as usize;
         }
+        // Always do this -- even if nothing changed, compressed_size_bytes isn't filled out by
+        // generate_manifest.
+        entry.compressed_size_bytes = std::fs::metadata(&remote_path).unwrap().len() as usize;
     }
 
     abstutil::write_json(format!("{}/MANIFEST.json", remote_base), &local);