1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156

use std::path::Path;
use std::process::Command;

use abstio::MapName;
use abstutil::{must_run_cmd, Timer};

use crate::configuration::ImporterConfiguration;

/// If the output file doesn't already exist, downloads the URL into that location. Automatically
/// uncompresses .zip and .gz files. Assumes a proper path is passed in (including data/).
pub fn download(config: &ImporterConfiguration, output: String, url: &str) {
    if Path::new(&output).exists() {
        println!("- {} already exists", output);
        return;
    }
    // Create the directory
    std::fs::create_dir_all(Path::new(&output).parent().unwrap())
        .expect("Creating parent dir failed");

    let tmp = "tmp_output";
    println!("- Missing {}, so downloading {}", output, url);
    must_run_cmd(
        Command::new("curl")
            .arg("--fail")
            .arg("-L")
            .arg("-o")
            .arg(tmp)
            .arg(url),
    );

    // Argh the Dropbox URL is .zip?dl=0
    if url.contains(".zip") {
        let unzip_to = if output.ends_with("/") {
            output
        } else {
            Path::new(&output).parent().unwrap().display().to_string()
        };
        println!("- Unzipping into {}", unzip_to);
        must_run_cmd(Command::new(&config.unzip).arg(tmp).arg("-d").arg(unzip_to));
        std::fs::remove_file(tmp).unwrap();
    } else if url.ends_with(".gz") {
        println!("- Gunzipping");
        std::fs::rename(tmp, format!("{}.gz", output)).unwrap();

        let mut gunzip_cmd = Command::new(&config.gunzip);
        for arg in config.gunzip_args.split_ascii_whitespace() {
            gunzip_cmd.arg(arg);
        }
        must_run_cmd(gunzip_cmd.arg(format!("{}.gz", output)));
    } else {
        std::fs::rename(tmp, output).unwrap();
    }
}

/// If the output file doesn't already exist, downloads the URL into that location. Clips .kml
/// files and converts to a .bin.
pub fn download_kml(
    output: String,
    url: &str,
    bounds: &geom::GPSBounds,
    require_all_pts_in_bounds: bool,
    timer: &mut Timer,
) {
    assert!(url.ends_with(".kml"));
    if Path::new(&output).exists() {
        println!("- {} already exists", output);
        return;
    }
    // Create the directory
    std::fs::create_dir_all(Path::new(&output).parent().unwrap())
        .expect("Creating parent dir failed");

    let tmp = "tmp_output";
    if Path::new(&output.replace(".bin", ".kml")).exists() {
        std::fs::copy(output.replace(".bin", ".kml"), tmp).unwrap();
    } else {
        println!("- Missing {}, so downloading {}", output, url);
        must_run_cmd(
            Command::new("curl")
                .arg("--fail")
                .arg("-L")
                .arg("-o")
                .arg(tmp)
                .arg(url),
        );
    }

    println!("- Extracting KML data");

    let shapes = kml::load(tmp.to_string(), bounds, require_all_pts_in_bounds, timer).unwrap();
    abstio::write_binary(output.clone(), &shapes);
    // Keep the intermediate file; otherwise we inadvertently grab new upstream data when
    // changing some binary formats
    std::fs::rename(tmp, output.replace(".bin", ".kml")).unwrap();
}

/// Uses osmconvert to clip the input .osm (or .pbf) against a polygon and produce some output.
/// Skips if the output exists.
pub fn osmconvert(
    input: String,
    clipping_polygon: String,
    output: String,
    config: &ImporterConfiguration,
) {
    let clipping_polygon = clipping_polygon;

    if Path::new(&output).exists() {
        println!("- {} already exists", output);
        return;
    }
    // Create the output directory if needed
    std::fs::create_dir_all(Path::new(&output).parent().unwrap())
        .expect("Creating parent dir failed");

    println!("- Clipping {} to {}", input, clipping_polygon);

    must_run_cmd(
        Command::new(&config.osmconvert)
            .arg(input)
            .arg(format!("-B={}", clipping_polygon))
            .arg("--complete-ways")
            .arg(format!("-o={}", output)),
    );
}

/// Converts a RawMap to a Map.
pub fn raw_to_map(
    name: &MapName,
    build_ch: bool,
    keep_bldg_tags: bool,
    timer: &mut Timer,
) -> map_model::Map {
    timer.start(format!("Raw->Map for {}", name.describe()));
    let raw: map_model::raw::RawMap = abstio::read_binary(abstio::path_raw_map(name), timer);
    let map = map_model::Map::create_from_raw(raw, build_ch, keep_bldg_tags, timer);
    timer.start("save map");
    map.save();
    timer.stop("save map");
    timer.stop(format!("Raw->Map for {}", name.describe()));

    // TODO Just sticking this here for now
    if name.map == "huge_seattle" || name == &MapName::new("gb", "leeds", "huge") {
        timer.start("generating city manifest");
        abstio::write_binary(
            abstio::path(format!(
                "system/{}/{}/city.bin",
                map.get_city_name().country,
                map.get_city_name().city
            )),
            &map_model::City::from_huge_map(&map),
        );
        timer.stop("generating city manifest");
    }

    map
}