1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
use std::path::Path;
use std::process::Command;

use abstio::{CityName, MapName};
use abstutil::{must_run_cmd, Timer};
use map_model::raw::RawMap;
use map_model::RawToMapOptions;

use crate::configuration::ImporterConfiguration;

/// If the output file doesn't already exist, downloads the URL into that location. Automatically
/// uncompresses .zip and .gz files. Assumes a proper path is passed in (including data/).
pub async fn download(config: &ImporterConfiguration, output: String, url: &str) {
    if Path::new(&output).exists() {
        println!("- {} already exists", output);
        return;
    }
    // Create the directory
    std::fs::create_dir_all(Path::new(&output).parent().unwrap())
        .expect("Creating parent dir failed");

    let tmp = "tmp_output";
    println!("- Missing {}, so downloading {}", output, url);
    abstio::download_to_file(url, None, tmp).await.unwrap();

    if url.ends_with(".zip") {
        let unzip_to = if output.ends_with('/') {
            output
        } else {
            // TODO In this case, there's no guarantee there's only one unzipped file and the name
            // matches!
            Path::new(&output).parent().unwrap().display().to_string()
        };
        println!("- Unzipping into {}", unzip_to);
        must_run_cmd(Command::new(&config.unzip).arg(tmp).arg("-d").arg(unzip_to));
        std::fs::remove_file(tmp).unwrap();
    } else if url.ends_with(".gz") {
        println!("- Gunzipping");
        std::fs::rename(tmp, format!("{}.gz", output)).unwrap();

        let mut gunzip_cmd = Command::new(&config.gunzip);
        for arg in config.gunzip_args.split_ascii_whitespace() {
            gunzip_cmd.arg(arg);
        }
        must_run_cmd(gunzip_cmd.arg(format!("{}.gz", output)));
    } else {
        std::fs::rename(tmp, output).unwrap();
    }
}

/// If the output file doesn't already exist, downloads the URL into that location. Clips .kml
/// files and converts to a .bin.
pub async fn download_kml(
    output: String,
    url: &str,
    bounds: &geom::GPSBounds,
    require_all_pts_in_bounds: bool,
    timer: &mut Timer<'_>,
) {
    assert!(url.ends_with(".kml"));
    if Path::new(&output).exists() {
        println!("- {} already exists", output);
        return;
    }
    // Create the directory
    std::fs::create_dir_all(Path::new(&output).parent().unwrap())
        .expect("Creating parent dir failed");

    let tmp = "tmp_output";
    if Path::new(&output.replace(".bin", ".kml")).exists() {
        std::fs::copy(output.replace(".bin", ".kml"), tmp).unwrap();
    } else {
        println!("- Missing {}, so downloading {}", output, url);
        abstio::download_to_file(url, None, tmp).await.unwrap();
    }

    println!("- Extracting KML data");

    let shapes = kml::load(tmp.to_string(), bounds, require_all_pts_in_bounds, timer).unwrap();
    abstio::write_binary(output.clone(), &shapes);
    // Keep the intermediate file; otherwise we inadvertently grab new upstream data when
    // changing some binary formats
    std::fs::rename(tmp, output.replace(".bin", ".kml")).unwrap();
}

/// Uses osmconvert to clip the input .osm (or .pbf) against a polygon and produce some output.
/// Skips if the output exists.
fn osmconvert(
    input: String,
    clipping_polygon: String,
    output: String,
    config: &ImporterConfiguration,
) {
    let clipping_polygon = clipping_polygon;

    if Path::new(&output).exists() {
        println!("- {} already exists", output);
        return;
    }
    // Create the output directory if needed
    std::fs::create_dir_all(Path::new(&output).parent().unwrap())
        .expect("Creating parent dir failed");

    println!("- Clipping {} to {}", input, clipping_polygon);

    must_run_cmd(
        Command::new(&config.osmconvert)
            .arg(input)
            .arg(format!("-B={}", clipping_polygon))
            .arg("--complete-ways")
            .arg(format!("-o={}", output)),
    );
}

/// Creates a RawMap from OSM and other input data.
pub async fn osm_to_raw(
    name: MapName,
    timer: &mut abstutil::Timer<'_>,
    config: &ImporterConfiguration,
) -> RawMap {
    if name.city == CityName::seattle() {
        crate::seattle::input(config, timer).await;
    }
    let opts = crate::map_config::config_for_map(&name);
    if let Some(ref url) = opts.gtfs_url {
        download(config, name.city.input_path("gtfs/"), url).await;
    }

    let boundary_polygon = format!(
        "importer/config/{}/{}/{}.poly",
        name.city.country, name.city.city, name.map
    );
    let osm_url = crate::pick_geofabrik(boundary_polygon.clone())
        .await
        .unwrap();

    let local_osm_file = name.city.input_path(format!(
        "osm/{}",
        std::path::Path::new(&osm_url)
            .file_name()
            .unwrap()
            .to_os_string()
            .into_string()
            .unwrap()
    ));
    download(config, local_osm_file.clone(), &osm_url).await;

    osmconvert(
        local_osm_file,
        boundary_polygon.clone(),
        name.city.input_path(format!("osm/{}.osm", name.map)),
        config,
    );

    let map = convert_osm::convert(
        name.city.input_path(format!("osm/{}.osm", name.map)),
        name.clone(),
        Some(boundary_polygon),
        opts,
        timer,
    );
    map.save();
    map
}

/// Converts a RawMap to a Map.
pub fn raw_to_map(name: &MapName, opts: RawToMapOptions, timer: &mut Timer) -> map_model::Map {
    timer.start(format!("Raw->Map for {}", name.describe()));
    let raw: RawMap = abstio::read_binary(abstio::path_raw_map(name), timer);
    let map = map_model::Map::create_from_raw(raw, opts, timer);
    timer.start("save map");
    map.save();
    timer.stop("save map");
    timer.stop(format!("Raw->Map for {}", name.describe()));

    // TODO Just sticking this here for now
    if name.map == "huge_seattle" || name == &MapName::new("gb", "leeds", "huge") {
        timer.start("generating city manifest");
        abstio::write_binary(
            abstio::path(format!(
                "system/{}/{}/city.bin",
                map.get_city_name().country,
                map.get_city_name().city
            )),
            &map_model::City::from_huge_map(&map),
        );
        timer.stop("generating city manifest");
    }

    map
}