From 606340e2dbf269eb53dfc67dc8d2cc62a4dcb78d Mon Sep 17 00:00:00 2001 From: Dustin Carlino Date: Sun, 10 Feb 2019 16:03:00 -0800 Subject: [PATCH] switch to bincode. almost half file size, reading small_seattle went from 40s to 7s in debug mode!!! --- abstutil/Cargo.toml | 2 +- abstutil/src/io.rs | 8 ++++---- docs/TODO_quality.md | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/abstutil/Cargo.toml b/abstutil/Cargo.toml index 8fe4fc737c..3d6facfb0c 100644 --- a/abstutil/Cargo.toml +++ b/abstutil/Cargo.toml @@ -5,13 +5,13 @@ authors = ["Dustin Carlino "] edition = "2018" [dependencies] +bincode = "1.1.1" lazy_static = "1.1.0" log = "0.4.5" multimap = "0.4.0" rand = { version = "0.6.5", features = ["serde1"] } rand_xorshift = "0.1.1" serde = "1.0.87" -serde_cbor = "0.9.0" serde_derive = "1.0" serde_json = "1.0" yansi = "0.5.0" diff --git a/abstutil/src/io.rs b/abstutil/src/io.rs index 6241bf229b..ca927f9012 100644 --- a/abstutil/src/io.rs +++ b/abstutil/src/io.rs @@ -1,8 +1,8 @@ use crate::{elapsed_seconds, Timer, PROGRESS_FREQUENCY_SECONDS}; +use bincode; use multimap; use serde::de::DeserializeOwned; use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use serde_cbor; use serde_json; use std; use std::collections::{BTreeMap, BTreeSet}; @@ -42,14 +42,14 @@ pub fn write_binary(path: &str, obj: &T) -> Result<(), Error> { std::fs::create_dir_all(std::path::Path::new(path).parent().unwrap()) .expect("Creating parent dir failed"); - let mut file = BufWriter::new(File::create(path)?); - serde_cbor::to_writer(&mut file, obj).map_err(|err| Error::new(ErrorKind::Other, err)) + let file = BufWriter::new(File::create(path)?); + bincode::serialize_into(file, obj).map_err(|err| Error::new(ErrorKind::Other, err)) } pub fn read_binary(path: &str, timer: &mut Timer) -> Result { let (reader, done) = FileWithProgress::new(path)?; let obj: T = - serde_cbor::from_reader(reader).map_err(|err| Error::new(ErrorKind::Other, err))?; + bincode::deserialize_from(reader).map_err(|err| Error::new(ErrorKind::Other, err))?; done(timer); Ok(obj) } diff --git a/docs/TODO_quality.md b/docs/TODO_quality.md index 1b90771129..6f96107ada 100644 --- a/docs/TODO_quality.md +++ b/docs/TODO_quality.md @@ -44,6 +44,7 @@ - https://gis-kingcounty.opendata.arcgis.com/datasets/traffic-signs--sign-point/ - population - http://seattlecitygis.maps.arcgis.com/apps/MapSeries/index.html?appid=3eb44a4fdf9a4fff9e1c105cd5e7fe27 + - https://data-seattlecitygis.opendata.arcgis.com/datasets/residential-building-permits-issued-and-final has NET_UNITS ## Low-priority geometry issues