switch to bincode. almost half file size, reading small_seattle went

from 40s to 7s in debug mode!!!
This commit is contained in:
Dustin Carlino 2019-02-10 16:03:00 -08:00
parent ec01207369
commit 606340e2db
3 changed files with 6 additions and 5 deletions

View File

@ -5,13 +5,13 @@ authors = ["Dustin Carlino <dabreegster@gmail.com>"]
edition = "2018"
[dependencies]
bincode = "1.1.1"
lazy_static = "1.1.0"
log = "0.4.5"
multimap = "0.4.0"
rand = { version = "0.6.5", features = ["serde1"] }
rand_xorshift = "0.1.1"
serde = "1.0.87"
serde_cbor = "0.9.0"
serde_derive = "1.0"
serde_json = "1.0"
yansi = "0.5.0"

View File

@ -1,8 +1,8 @@
use crate::{elapsed_seconds, Timer, PROGRESS_FREQUENCY_SECONDS};
use bincode;
use multimap;
use serde::de::DeserializeOwned;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde_cbor;
use serde_json;
use std;
use std::collections::{BTreeMap, BTreeSet};
@ -42,14 +42,14 @@ pub fn write_binary<T: Serialize>(path: &str, obj: &T) -> Result<(), Error> {
std::fs::create_dir_all(std::path::Path::new(path).parent().unwrap())
.expect("Creating parent dir failed");
let mut file = BufWriter::new(File::create(path)?);
serde_cbor::to_writer(&mut file, obj).map_err(|err| Error::new(ErrorKind::Other, err))
let file = BufWriter::new(File::create(path)?);
bincode::serialize_into(file, obj).map_err(|err| Error::new(ErrorKind::Other, err))
}
pub fn read_binary<T: DeserializeOwned>(path: &str, timer: &mut Timer) -> Result<T, Error> {
let (reader, done) = FileWithProgress::new(path)?;
let obj: T =
serde_cbor::from_reader(reader).map_err(|err| Error::new(ErrorKind::Other, err))?;
bincode::deserialize_from(reader).map_err(|err| Error::new(ErrorKind::Other, err))?;
done(timer);
Ok(obj)
}

View File

@ -44,6 +44,7 @@
- https://gis-kingcounty.opendata.arcgis.com/datasets/traffic-signs--sign-point/
- population
- http://seattlecitygis.maps.arcgis.com/apps/MapSeries/index.html?appid=3eb44a4fdf9a4fff9e1c105cd5e7fe27
- https://data-seattlecitygis.opendata.arcgis.com/datasets/residential-building-permits-issued-and-final has NET_UNITS
## Low-priority geometry issues