moving popdat+precompute into the new unified importer

Cargo.toml

@@ -13,7 +13,6 @@ members = [
   "map_editor",
   "map_model",
   "popdat",
-  "precompute",
   "sim",
 ]
 

data/MANIFEST.txt

@@ -106,21 +106,21 @@ dba9eedaf3ae1f8bc8be3e08be179f14  data/input/raw_maps/23rd.bin
 c4caf484acb064e55301d88328646295  data/input/screenshots/montlake/02x05_i124.png
 77ea71d6940759fca2bdf59d16b7d443  data/input/screenshots/montlake/01x06_i26.png
 1717bfb53aa375bd3359de6e7db860ff  data/input/screenshots/montlake/01x01_i19.png
-7f5712e29165a3abe849a475657c72e5  data/input/screenshots/montlake/MANIFEST
+835ec530218e685a60f9226f94f25fa1  data/input/screenshots/montlake/MANIFEST
 0ece3a6bc65266c96c10ecb11f6738f9  data/input/screenshots/montlake/03x06_i27.png
 1fd8670f993197431622eb0dab20936f  data/input/screenshots/montlake/02x03_i1.png
 612d5c710d77ae2fcd639fc8c1716dac  data/input/screenshots/montlake/02x04_i25.png
 33b3c92b72edd8fd757e2df73eb87a52  data/input/screenshots/montlake/02x02_i289.png
-6317c0f6981814a9c08fb88feba138aa  data/input/screenshots/montlake/03x02_i8.png
+d499d4c37f4319d943fa1e3dd2ccf102  data/input/screenshots/montlake/03x02_i8.png
 920eaf9c480d89e68234b399fecce33a  data/input/screenshots/montlake/01x04_i31.png
 dc10dd940c4f3b41086f7b40983ba25f  data/input/screenshots/montlake/combine.sh
-f01be3cc20163205f07bc22e75cdb427  data/input/screenshots/montlake/03x01_i0.png
+fcde70018c3659d5a4e0c32c94e2c52c  data/input/screenshots/montlake/03x01_i0.png
 f85b649d9aa151d8f5957d64bfe968c8  data/input/screenshots/montlake/01x03_i4.png
 8d74e49cab21536e37c0fc52ba07cefa  data/input/screenshots/montlake/01x02_i20.png
 b1929464274d34c3bce5040821f7869d  data/input/screenshots/montlake/03x04_i112.png
 5449cc2d3ac688dfd34781092ff8e62e  data/input/screenshots/montlake/03x03_i59.png
 587a29517ab20ebe69c03cc2d0eb6b9e  data/input/screenshots/montlake/02x01_i24.png
-0cfba587f63ae66e59dc6961e4aab07b  data/input/screenshots/montlake/03x05_i2.png
+57c9105b0ec76b86ffc1e4600ab25c85  data/input/screenshots/montlake/03x05_i2.png
 778efa9f883533862c504689fb6b9a1b  data/input/screenshots/montlake/02x06_i85.png
 6c894ae6c793e96056493524254ea28d  data/input/screenshots/montlake/01x05_i40.png
 129b460f56f5eb41cab3bfd70fb5fde9  data/input/sidewalks.bin

docs/dev.md

@@ -57,8 +57,7 @@ You can skip this section if you're just touching code in `game`, `ezgui`, and
 You have two options: you can seed some of the intermediate data by running
 `./data/grab_all_seed_data.sh` (downloads ~1GB, expands to ~5GB), or you can
 build everything totally from scratch by running
-`cd importer && cargo run --release && cd .. && ./precompute.sh --release`. This
-takes a while.
+`./import.sh --osm --map --scenario`. This takes a while.
 
 You'll need some extra dependencies:
 
@@ -66,16 +65,14 @@ You'll need some extra dependencies:
 - `libgdal-dev`: See https://gdal.org/ if your OS package manager doesn't have
   this
 
-Some tips:
+You can rerun specific stages of the importer:
 
 - If you're modifying the initial OSM data -> RawMap conversion in
-  `convert_osm`, then you do need to rerun
-  `cd importer; cargo run --release; cd ..` and `precompute.sh` to regenerate
-  the map.
+  `convert_osm`, you need `./import.sh --osm --map`.
 - If you're modifying `map_model` but not the OSM -> RawMap conversion, then you
-  can just do `precompute.sh`.
-- Both of those scripts can just regenerate a single map, which is much faster:
-  `cd importer && cargo run --release -- caphill && cd .. && ./precompute.sh caphill`
+  just need `./import.sh --map`.
+- By default, all maps are regenerated. You can also specify a single map:
+  `./import.sh --map downtown`.
 
 ## Understanding stuff
 
@@ -96,12 +93,11 @@ Constructing the map:
 - `kml`: extract shapes from KML shapefiles
 - `map_model`: the final representation of the map, also conversion from the
   intermediate map format into the final format
-- `precompute`: small tool to run the second stage of map conversion and write
-  final output
 - `popdat`: importing daily trips from PSRC's Soundcast model, specific to
   Seattle
 - `map_editor`: GUI for modifying geometry of maps and creating maps from
   scratch
+- `importer`: tool to run the entire import pipeline
 
 Traffic simulation:
 

import.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+cd importer;
+RUST_BACKTRACE=1 cargo run --release -- $@;
+cd ..;

importer/Cargo.toml

@@ -7,5 +7,12 @@ edition = "2018"
 [dependencies]
 abstutil = { path = "../abstutil" }
 convert_osm = { path = "../convert_osm" }
+csv = "1.0.1"
+failure = "0.1.2"
 geom = { path = "../geom" }
+gdal = "0.6.0"
 kml = { path = "../kml" }
+map_model = { path = "../map_model" }
+popdat = { path = "../popdat" }
+serde = "1.0.98"
+serde_derive = "1.0.98"

importer/src/main.rs

@@ -1,15 +1,61 @@
+mod psrc;
+mod seattle;
+mod utils;
+
+struct Job {
+    osm_to_raw: bool,
+    raw_to_map: bool,
+    scenario: bool,
+
+    use_fixes: bool,
+    only_map: Option<String>,
+}
+
 fn main() {
     let mut args = abstutil::CmdArgs::new();
-    let only_map = args.optional_free();
-    args.done();
+    let job = Job {
+        // Download all raw input files, then convert OSM to the intermediate RawMap.
+        osm_to_raw: args.enabled("--raw"),
+        // Convert the RawMap to the final Map format.
+        raw_to_map: args.enabled("--map"),
+        // Download trip demand data, then produce the typical weekday scenario.
+        scenario: args.enabled("--scenario"),
 
-    if let Some(n) = only_map {
-        println!("- Just producing RawMap for {}", n);
-        importer::seattle_map(n);
+        // By default, use geometry fixes from map_editor.
+        use_fixes: !args.enabled("--nofixes"),
+        // Only process one map. If not specified, process all maps defined by clipping polygons in
+        // data/input/polygons/.
+        only_map: args.optional_free(),
+    };
+    args.done();
+    if !job.osm_to_raw && !job.raw_to_map && !job.scenario {
+        println!("Nothing to do! Pass some combination of --raw, --map, --scenario");
+        std::process::exit(1);
+    }
+
+    let names = if let Some(n) = job.only_map {
+        println!("- Just working on {}", n);
+        vec![n]
     } else {
-        println!("- Producing all RawMaps for Seattle");
-        for name in abstutil::list_all_objects("../data/input/polygons".to_string()) {
-            importer::seattle_map(name);
+        println!("- Working on all Seattle maps");
+        abstutil::list_all_objects("../data/input/polygons".to_string())
+    };
+
+    for name in names {
+        if job.osm_to_raw {
+            seattle::osm_to_raw(&name);
+        }
+
+        if job.raw_to_map {
+            utils::raw_to_map(&name, job.use_fixes);
+        }
+
+        if job.scenario {
+            seattle::ensure_popdat_exists(job.use_fixes);
+
+            let mut timer = abstutil::Timer::new(format!("Scenario for {}", name));
+            let map = map_model::Map::new(abstutil::path_map(&name), job.use_fixes, &mut timer);
+            popdat::trips_to_scenario(&map, &mut timer).save();
         }
     }
 }

importer/src/psrc.rs

@@ -7,7 +7,8 @@ use std::collections::{BTreeMap, HashMap, HashSet};
 use std::fs::File;
 use std::io::Write;
 
-fn main() {
+// Extract trip demand data from PSRC's Soundcast outputs.
+pub fn import_psrc_data() {
     let mut timer = abstutil::Timer::new("creating popdat");
     let (trips, parcels) = import_trips(
         "../data/input/parcels_urbansim.txt",

importer/src/seattle.rs

@@ -1,7 +1,6 @@
-mod utils;
-
-use utils::{download, osmconvert, rm};
+use crate::utils::{download, osmconvert, rm};
 
+// Grab all raw input files for Seattle.
 fn seattle_input() {
     download(
         "../data/input/google_transit/",
@@ -37,7 +36,8 @@ fn seattle_input() {
     download("../data/input/offstreet_parking.bin", "http://data-seattlecitygis.opendata.arcgis.com/datasets/8e52dfde6d5d45948f7a90654c8d50cd_0.kml");
 }
 
-pub fn seattle_map(name: String) {
+// Raw input -> RawMap for Seattle
+pub fn osm_to_raw(name: &str) {
     seattle_input();
     osmconvert(
         "../data/input/osm/Seattle.osm",
@@ -68,3 +68,20 @@ pub fn seattle_map(name: String) {
     println!("- Saving {}", output);
     abstutil::write_binary(output, &map);
 }
+
+// Download and pre-process data needed to generate Seattle scenarios.
+pub fn ensure_popdat_exists(use_fixes: bool) {
+    if abstutil::file_exists(abstutil::path_popdat()) {
+        println!("- {} exists, not regenerating it", abstutil::path_popdat());
+        return;
+    }
+
+    if !abstutil::file_exists(abstutil::path_raw_map("huge_seattle")) {
+        osm_to_raw("huge_seattle");
+    }
+    if !abstutil::file_exists(abstutil::path_map("huge_seattle")) {
+        crate::utils::raw_to_map("huge_seattle", use_fixes);
+    }
+
+    crate::psrc::import_psrc_data();
+}

importer/src/utils.rs

@@ -1,6 +1,9 @@
 use std::path::Path;
 use std::process::Command;
 
+// If the output file doesn't already exist, downloads the URL into that location. Automatically
+// uncompresses .zip and .gz files. .kml files are automatically clipped to a hardcoded boundary of
+// Seattle.
 pub fn download(output: &str, url: &str) {
     if Path::new(output).exists() {
         println!("- {} already exists", output);
@@ -45,6 +48,8 @@ pub fn download(output: &str, url: &str) {
     }
 }
 
+// Uses osmconvert to clip the input .osm (or .pbf) against a polygon and produce some output.
+// Skips if the output exists.
 pub fn osmconvert(input: &str, clipping_polygon: String, output: String) {
     if Path::new(&output).exists() {
         println!("- {} already exists", output);
@@ -59,12 +64,14 @@ pub fn osmconvert(input: &str, clipping_polygon: String, output: String) {
         .arg(format!("-o={}", output)));
 }
 
+// Removes files. Be careful!
 pub fn rm<I: Into<String>>(path: I) {
     let path = path.into();
     println!("- Removing {}", path);
     run(Command::new("rm").arg("-rfv").arg(path));
 }
 
+// Runs a command, asserts success. STDOUT and STDERR aren't touched.
 fn run(cmd: &mut Command) {
     println!("- Running {:?}", cmd);
     match cmd.status() {
@@ -78,3 +85,12 @@ fn run(cmd: &mut Command) {
         }
     }
 }
+
+// Converts a RawMap to a Map.
+pub fn raw_to_map(name: &str, use_fixes: bool) {
+    let mut timer = abstutil::Timer::new(format!("Raw->Map for {}", name));
+    let map = map_model::Map::new(abstutil::path_raw_map(name), use_fixes, &mut timer);
+    timer.start("save map");
+    map.save();
+    timer.stop("save map");
+}

popdat/Cargo.toml

@@ -4,22 +4,10 @@ version = "0.1.0"
 authors = ["Dustin Carlino <dabreegster@gmail.com>"]
 edition = "2018"
 
-[features]
-default = []
-import = ["gdal"]
-
 [dependencies]
 abstutil = { path = "../abstutil" }
-csv = "1.0.1"
-failure = "0.1.2"
 geom = { path = "../geom" }
-# Only needed for the offline precompute step, not at runtime. Skip on WASM.
-gdal = { version = "0.6.0", optional = true }
 map_model = { path = "../map_model" }
 serde = "1.0.98"
 serde_derive = "1.0.98"
 sim = { path = "../sim" }
-
-[[bin]]
-name = "popdat"
-required-features = ["import"]

precompute.sh

@@ -1,57 +0,0 @@
-#!/bin/bash
-
-set -e
-
-mkdir -p data/system/maps/
-
-# Need this first
-if [ ! -f data/input/popdat.bin ]; then
-	# We probably don't have this map yet.
-	if [ ! -f data/system/maps/huge_seattle.bin ]; then
-		cd precompute;
-		RUST_BACKTRACE=1 cargo run --release ../data/input/raw_maps/huge_seattle.bin --disable_psrc_scenarios;
-		cd ..;
-	fi
-
-	cd popdat;
-	RUST_BACKTRACE=1 cargo run --release --features import;
-	cd ..;
-fi
-
-release_mode=""
-psrc_scenarios=""
-no_fixes=""
-just_psrc_scenarios=false
-for arg in "$@"; do
-	if [ "$arg" == "--release" ]; then
-		release_mode="--release";
-	elif [ "$arg" == "--disable_psrc_scenarios" ]; then
-		psrc_scenarios="--disable_psrc_scenarios";
-	elif [ "$arg" == "--just_psrc_scenarios" ]; then
-		just_psrc_scenarios=true;
-	elif [ "$arg" == "--nofixes" ]; then
-		no_fixes="--nofixes";
-	else
-		# Just recompute a single map.
-		cd precompute;
-		if $just_psrc_scenarios; then
-			RUST_BACKTRACE=1 cargo run $release_mode ../data/system/maps/$arg.bin $psrc_scenarios $no_fixes;
-		else
-			RUST_BACKTRACE=1 cargo run $release_mode ../data/input/raw_maps/$arg.bin $psrc_scenarios $no_fixes;
-		fi
-		cd ..;
-		exit;
-	fi
-done
-
-for map_path in `ls data/input/raw_maps/`; do
-	map=`basename $map_path .bin`;
-	echo "Precomputing $map";
-	cd precompute;
-	if $just_psrc_scenarios; then
-		RUST_BACKTRACE=1 cargo run $release_mode ../data/system/maps/$map.bin $psrc_scenarios $no_fixes;
-	else
-		RUST_BACKTRACE=1 cargo run $release_mode ../data/input/raw_maps/$map.bin $psrc_scenarios $no_fixes;
-	fi
-	cd ..;
-done

precompute/Cargo.toml

@@ -1,11 +0,0 @@
-[package]
-name = "precompute"
-version = "0.1.0"
-authors = ["Dustin Carlino <dabreegster@gmail.com>"]
-edition = "2018"
-
-[dependencies]
-abstutil = { path = "../abstutil" }
-geom = { path = "../geom" }
-map_model = { path = "../map_model" }
-popdat = { path = "../popdat" }

precompute/src/main.rs

@@ -1,22 +0,0 @@
-use abstutil::{CmdArgs, Timer};
-use map_model::Map;
-use popdat::trips_to_scenario;
-
-fn main() {
-    let mut args = CmdArgs::new();
-    let load = args.required_free();
-    let disable_psrc_scenarios = args.enabled("--disable_psrc_scenarios");
-    let use_fixes = !args.enabled("--nofixes");
-    args.done();
-
-    let mut timer = Timer::new(format!("precompute {}", load));
-
-    let map = Map::new(load, use_fixes, &mut timer);
-    timer.start("save map");
-    map.save();
-    timer.stop("save map");
-
-    if !disable_psrc_scenarios {
-        trips_to_scenario(&map, &mut timer).save();
-    }
-}