Support other formats. (#11)

Broad refactor of how things work, moving `json.rs` to `format.rs`.

Adds support for TOML, along with CLI stuff for setting formats (which are automatically inferred from filenames).

Adds support for binary data using base64 encodings.

Cargo.lock

@@ -35,6 +35,12 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
 
+[[package]]
+name = "base64"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
+
 [[package]]
 name = "bitflags"
 version = "1.2.1"
@@ -84,10 +90,12 @@ dependencies = [
 name = "ffs"
 version = "0.1.0"
 dependencies = [
+ "base64",
  "clap",
  "fuser",
  "libc",
  "serde_json",
+ "toml",
  "tracing",
  "tracing-subscriber",
 ]
@@ -333,6 +341,15 @@ dependencies = [
  "once_cell",
 ]
 
+[[package]]
+name = "toml"
+version = "0.5.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "tracing"
 version = "0.1.26"

Cargo.toml

@@ -7,9 +7,11 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-fuser = "0.8"
-serde_json = "1.0"
+base64 = "0.13.0"
 clap = "2.0"
+fuser = "0.8"
 libc = "0.2.51"
+serde_json = "1.0"
+toml = "0.5"
 tracing = "0.1"
-tracing-subscriber = "0.2.18"
+tracing-subscriber = "0.2.18"

json/empty.json

@@ -0,0 +1 @@
+{}

json/single.json

@@ -0,0 +1 @@
+{"onlyone":"highlander"}

src/config.rs

@@ -1,7 +1,11 @@
 use std::path::PathBuf;
 
+use super::format::Format;
+
 #[derive(Debug)]
 pub struct Config {
+    pub input_format: Format,
+    pub output_format: Format,
     pub timestamp: std::time::SystemTime,
     pub uid: u32,
     pub gid: u32,
@@ -9,6 +13,8 @@ pub struct Config {
     pub dirmode: u16,
     pub add_newlines: bool,
     pub pad_element_names: bool,
+    pub base64: base64::Config,
+    pub try_decode_base64: bool,
     pub read_only: bool,
     pub output: Output,
 }
@@ -37,12 +43,13 @@ impl Config {
             .replace("=", "equal")
             .replace(" ", "space")
     }
-    
 }
 
 impl Default for Config {
     fn default() -> Self {
         Config {
+            input_format: Format::Json,
+            output_format: Format::Json,
             timestamp: std::time::SystemTime::now(),
             uid: 501,
             gid: 501,
@@ -50,8 +57,10 @@ impl Default for Config {
             dirmode: 0o755,
             add_newlines: false,
             pad_element_names: true,
+            base64: base64::STANDARD,
+            try_decode_base64: false,
             read_only: false,
             output: Output::Stdout,
         }
     }
-}
+}

src/format.rs

@@ -0,0 +1,485 @@
+use std::collections::HashMap;
+use std::fs::File;
+use std::str::FromStr;
+
+use tracing::{debug, error, info, instrument};
+
+use fuser::FileType;
+
+use super::config::{Config, Output};
+use super::fs::{DirEntry, DirType, Entry, Inode, FS};
+
+use ::toml as serde_toml;
+
+#[derive(Copy, Clone, Debug)]
+pub enum Format {
+    Json,
+    Toml,
+}
+
+pub const POSSIBLE_FORMATS: &[&str] = &["json", "toml"];
+
+impl std::fmt::Display for Format {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
+        write!(
+            f,
+            "{}",
+            match self {
+                Format::Json => "json",
+                Format::Toml => "toml",
+            }
+        )
+    }
+}
+
+#[derive(Debug)]
+pub enum ParseFormatError {
+    NoSuchFormat(String),
+    NoFormatProvided,
+}
+
+impl FromStr for Format {
+    type Err = ParseFormatError;
+
+    fn from_str(s: &str) -> Result<Self, ParseFormatError> {
+        let s = s.trim().to_lowercase();
+
+        if s == "json" {
+            Ok(Format::Json)
+        } else if s == "toml" {
+            Ok(Format::Toml)
+        } else {
+            Err(ParseFormatError::NoSuchFormat(s))
+        }
+    }
+}
+
+impl Format {
+    /// Generates a filesystem `fs`, reading from `reader` according to a
+    /// particular `Config`.
+    ///
+    /// NB there is no check that `self == fs.config.input_format`!
+    #[instrument(level = "info", skip(reader, config))]
+    pub fn load(&self, reader: Box<dyn std::io::Read>, config: Config) -> FS {
+        let mut inodes: Vec<Option<Inode>> = Vec::new();
+
+        match self {
+            Format::Json => {
+                info!("reading json value");
+                let v: serde_json::Value = serde_json::from_reader(reader).expect("JSON");
+                info!("building inodes");
+                fs_from_value(v, &config, &mut inodes);
+                info!("done");
+            }
+            Format::Toml => {
+                info!("reading toml value");
+                let v = toml::from_reader(reader).expect("TOML");
+                info!("building inodes");
+                fs_from_value(v, &config, &mut inodes);
+                info!("done");
+            }
+        };
+
+        FS::new(inodes, config)
+    }
+
+    /// Given a filesystem `fs`, it outputs a file in the appropriate format,
+    /// following `fs.config`.
+    ///
+    /// NB there is no check that `self == fs.config.output_format`!
+    #[instrument(level = "info", skip(fs))]
+    pub fn save(&self, fs: &FS) {
+        let writer: Box<dyn std::io::Write> = match &fs.config.output {
+            Output::Stdout => {
+                debug!("outputting on STDOUT");
+                Box::new(std::io::stdout())
+            }
+            Output::File(path) => {
+                debug!("output {}", path.display());
+                Box::new(File::create(path).unwrap())
+            }
+            Output::Quiet => {
+                debug!("no output path, skipping");
+                return;
+            }
+        };
+
+        match self {
+            Format::Json => {
+                info!("generating json value");
+                let v: serde_json::Value = value_from_fs(fs, fuser::FUSE_ROOT_ID);
+                info!("writing");
+                debug!("outputting {}", v);
+                serde_json::to_writer(writer, &v).unwrap();
+                info!("done")
+            }
+            Format::Toml => {
+                info!("generating toml value");
+                let v: serde_toml::Value = value_from_fs(fs, fuser::FUSE_ROOT_ID);
+                info!("writing");
+                debug!("outputting {}", v);
+                toml::to_writer(writer, &v).unwrap();
+                info!("done");
+            }
+        }
+    }
+}
+
+enum Node<V> {
+    String(String),
+    Bytes(Vec<u8>),
+
+    /// TODO 2021-06-18 can we make these Iter, to avoid any intermediate allocation/structures?
+    List(Vec<V>),
+    /// We use a `Vec` rather than a `Map` or `HashMap` to ensure we preserve
+    /// whatever order.
+    Map(Vec<(String, V)>),
+}
+
+/// Values that can be converted to a `Node`, which can be in turn processed by
+/// the worklist algorithm
+trait Nodelike
+where
+    Self: Sized,
+{
+    /// Number of "nodes" in the given value. This should correspond to the
+    /// number of inodes needed to accommodate the value.
+    fn size(&self) -> usize;
+
+    /// Predicts filetypes (directory vs. regular file) for values.
+    ///
+    /// Since FUSE filesystems need to have directories at the root, it's
+    /// important that only compound values be converted to fileysstems, i.e.,
+    /// values which yield `FileType::Directory`.
+    fn kind(&self) -> FileType;
+
+    /// Characterizes the outermost value. Drives the worklist algorithm.
+    fn node(self, config: &Config) -> Node<Self>;
+
+    fn from_bytes<T>(v: T, config: &Config) -> Self
+    where
+        T: AsRef<[u8]>;
+    fn from_string(v: String, config: &Config) -> Self;
+    fn from_list_dir(files: Vec<Self>, config: &Config) -> Self;
+    fn from_named_dir(files: HashMap<String, Self>, config: &Config) -> Self;
+}
+
+/// Given a `Nodelike` value `v`, initializes the vector `inodes` of (nullable)
+/// `Inodes` according to a given `config`.
+///
+/// The current implementation is eager: it preallocates enough inodes and then
+/// fills them in using a depth-first traversal.
+///
+/// Invariant: the index in the vector is the inode number. Inode 0 is invalid,
+/// and is left empty.
+fn fs_from_value<V>(v: V, config: &Config, inodes: &mut Vec<Option<Inode>>)
+where
+    V: Nodelike + std::fmt::Display,
+{
+    // reserve space for everyone else
+    // won't work with streaming or lazy generation, but avoids having to resize the vector midway through
+    inodes.resize_with(v.size() + 1, || None);
+    info!("allocated {} inodes", inodes.len());
+
+    if v.kind() != FileType::Directory {
+        error!("The root of the filesystem must be a directory, but '{}' only generates a single file.", v);
+        std::process::exit(1);
+    }
+
+    let mut next_id = fuser::FUSE_ROOT_ID;
+    // parent inum, inum, value
+    let mut worklist: Vec<(u64, u64, V)> = vec![(next_id, next_id, v)];
+
+    next_id += 1;
+
+    while !worklist.is_empty() {
+        let (parent, inum, v) = worklist.pop().unwrap();
+
+        let entry = match v.node(config) {
+            Node::Bytes(b) => Entry::File(b),
+            Node::String(s) => Entry::File(s.into_bytes()),
+            Node::List(vs) => {
+                let mut children = HashMap::new();
+                children.reserve(vs.len());
+
+                let num_elts = vs.len() as f64;
+                let width = num_elts.log10().ceil() as usize;
+
+                for (i, child) in vs.into_iter().enumerate() {
+                    // TODO 2021-06-08 ability to add prefixes
+                    let name = if config.pad_element_names {
+                        format!("{:0width$}", i, width = width)
+                    } else {
+                        format!("{}", i)
+                    };
+
+                    children.insert(
+                        name,
+                        DirEntry {
+                            inum: next_id,
+                            kind: child.kind(),
+                        },
+                    );
+                    worklist.push((inum, next_id, child));
+                    next_id += 1;
+                }
+
+                Entry::Directory(DirType::List, children)
+            }
+            Node::Map(fvs) => {
+                let mut children = HashMap::new();
+                children.reserve(fvs.len());
+
+                for (field, child) in fvs.into_iter() {
+                    let original = field.clone();
+                    let mut nfield = config.normalize_name(field);
+
+                    while children.contains_key(&nfield) {
+                        nfield.push('_');
+                    }
+
+                    if original != nfield {
+                        info!(
+                            "renamed {} to {} (inode {} with parent {})",
+                            original, nfield, next_id, parent
+                        );
+                    }
+
+                    children.insert(
+                        nfield,
+                        DirEntry {
+                            inum: next_id,
+                            kind: child.kind(),
+                        },
+                    );
+
+                    worklist.push((inum, next_id, child));
+                    next_id += 1;
+                }
+
+                Entry::Directory(DirType::Named, children)
+            }
+        };
+
+        inodes[inum as usize] = Some(Inode {
+            parent,
+            inum,
+            entry,
+        });
+    }
+    assert_eq!(inodes.len() as u64, next_id);
+}
+
+/// Walks `fs` starting at the inode with number `inum`, producing an
+/// appropriate value.
+fn value_from_fs<V>(fs: &FS, inum: u64) -> V
+where
+    V: Nodelike,
+{
+    match &fs.get(inum).unwrap().entry {
+        Entry::File(contents) => match String::from_utf8(contents.clone()) {
+            Ok(mut contents) => {
+                if fs.config.add_newlines && contents.ends_with('\n') {
+                    contents.truncate(contents.len() - 1);
+                }
+                V::from_string(contents, &fs.config)
+            }
+            Err(_) => V::from_bytes(contents, &fs.config),
+        },
+        Entry::Directory(DirType::List, files) => {
+            let mut entries = Vec::with_capacity(files.len());
+
+            let mut files = files.iter().collect::<Vec<_>>();
+            files.sort_unstable_by(|(name1, _), (name2, _)| name1.cmp(name2));
+            for (_name, DirEntry { inum, .. }) in files.iter() {
+                let v = value_from_fs(fs, *inum);
+                entries.push(v);
+            }
+
+            V::from_list_dir(entries, &fs.config)
+        }
+        Entry::Directory(DirType::Named, files) => {
+            let mut entries = HashMap::with_capacity(files.len());
+
+            for (name, DirEntry { inum, .. }) in files.iter() {
+                let v = value_from_fs(fs, *inum);
+                entries.insert(name.into(), v);
+            }
+
+            V::from_named_dir(entries, &fs.config)
+        }
+    }
+}
+
+mod json {
+    use super::*;
+    use serde_json::Value;
+
+    impl Nodelike for Value {
+        /// `Value::Object` and `Value::Array` map to directories; everything else is a
+        /// regular file.
+        fn kind(&self) -> FileType {
+            match self {
+                Value::Object(_) | Value::Array(_) => FileType::Directory,
+                _ => FileType::RegularFile,
+            }
+        }
+
+        fn size(&self) -> usize {
+            match self {
+                Value::Null | Value::Bool(_) | Value::Number(_) | Value::String(_) => 1,
+                Value::Array(vs) => vs.iter().map(|v| v.size()).sum::<usize>() + 1,
+                Value::Object(fvs) => fvs.iter().map(|(_, v)| v.size()).sum::<usize>() + 1,
+            }
+        }
+
+        fn node(self, config: &Config) -> Node<Self> {
+            let nl = if config.add_newlines { "\n" } else { "" };
+
+            match self {
+                Value::Null => Node::Bytes("".into()), // always empty
+                Value::Bool(b) => Node::Bytes(format!("{}{}", b, nl).into_bytes()),
+                Value::Number(n) => Node::Bytes(format!("{}{}", n, nl).into_bytes()),
+                Value::String(s) => {
+                    if config.try_decode_base64 {
+                        if let Ok(bytes) = base64::decode_config(&s, config.base64) {
+                            return Node::Bytes(bytes);
+                        }
+                    }
+
+                    Node::String(if s.ends_with('\n') { s } else { s + nl })
+                }
+                Value::Array(vs) => Node::List(vs),
+                Value::Object(fvs) => Node::Map(fvs.into_iter().collect()),
+            }
+        }
+
+        fn from_string(contents: String, _config: &Config) -> Self {
+            if contents.is_empty() {
+                Value::Null
+            } else if contents == "true" {
+                Value::Bool(true)
+            } else if contents == "false" {
+                Value::Bool(false)
+            } else if let Ok(n) = serde_json::Number::from_str(&contents) {
+                Value::Number(n)
+            } else {
+                Value::String(contents)
+            }
+        }
+
+        fn from_bytes<T>(contents: T, config: &Config) -> Self
+        where
+            T: AsRef<[u8]>,
+        {
+            Value::String(base64::encode_config(contents, config.base64))
+        }
+
+        fn from_list_dir(files: Vec<Self>, _config: &Config) -> Self {
+            Value::Array(files)
+        }
+
+        fn from_named_dir(files: HashMap<String, Self>, _config: &Config) -> Self {
+            Value::Object(files.into_iter().collect())
+        }
+    }
+}
+
+mod toml {
+    use super::*;
+
+    use serde_toml::Value;
+    #[derive(Debug)]
+    pub enum Error<E> {
+        Io(std::io::Error),
+        Toml(E),
+    }
+
+    pub fn from_reader(
+        mut reader: Box<dyn std::io::Read>,
+    ) -> Result<Value, Error<serde_toml::de::Error>> {
+        let mut text = String::new();
+        let _len = reader.read_to_string(&mut text).map_err(Error::Io)?;
+        serde_toml::from_str(&text).map_err(Error::Toml)
+    }
+
+    pub fn to_writer(
+        mut writer: Box<dyn std::io::Write>,
+        v: &Value,
+    ) -> Result<(), Error<serde_toml::ser::Error>> {
+        let text = serde_toml::to_string(v).map_err(Error::Toml)?;
+        writer.write_all(text.as_bytes()).map_err(Error::Io)
+    }
+
+    impl Nodelike for Value {
+        fn kind(&self) -> FileType {
+            match self {
+                Value::Table(_) | Value::Array(_) => FileType::Directory,
+                _ => FileType::RegularFile,
+            }
+        }
+
+        fn size(&self) -> usize {
+            match self {
+                Value::Boolean(_)
+                | Value::Datetime(_)
+                | Value::Float(_)
+                | Value::Integer(_)
+                | Value::String(_) => 1,
+                Value::Array(vs) => vs.iter().map(|v| v.size()).sum::<usize>() + 1,
+                Value::Table(fvs) => fvs.iter().map(|(_, v)| v.size()).sum::<usize>() + 1,
+            }
+        }
+
+        fn node(self, config: &Config) -> Node<Self> {
+            let nl = if config.add_newlines { "\n" } else { "" };
+
+            match self {
+                Value::Boolean(b) => Node::Bytes(format!("{}{}", b, nl).into_bytes()),
+                Value::Datetime(s) => Node::String(s.to_string()),
+                Value::Float(n) => Node::Bytes(format!("{}{}", n, nl).into_bytes()),
+                Value::Integer(n) => Node::Bytes(format!("{}{}", n, nl).into_bytes()),
+                Value::String(s) => {
+                    if config.try_decode_base64 {
+                        if let Ok(bytes) = base64::decode_config(&s, config.base64) {
+                            return Node::Bytes(bytes);
+                        }
+                    }
+
+                    Node::String(if s.ends_with('\n') { s } else { s + nl })
+                }
+                Value::Array(vs) => Node::List(vs),
+                Value::Table(fvs) => Node::Map(fvs.into_iter().collect()),
+            }
+        }
+
+        fn from_string(contents: String, _config: &Config) -> Self {
+            if contents == "true" {
+                Value::Boolean(true)
+            } else if contents == "false" {
+                Value::Boolean(false)
+            } else if let Ok(n) = i64::from_str(&contents) {
+                Value::Integer(n)
+            } else if let Ok(n) = f64::from_str(&contents) {
+                Value::Float(n)
+            } else {
+                Value::String(contents)
+            }
+        }
+
+        fn from_bytes<T>(contents: T, config: &Config) -> Self
+        where
+            T: AsRef<[u8]>,
+        {
+            Value::String(base64::encode_config(contents, config.base64))
+        }
+
+        fn from_list_dir(files: Vec<Self>, _config: &Config) -> Self {
+            Value::Array(files)
+        }
+
+        fn from_named_dir(files: HashMap<String, Self>, _config: &Config) -> Self {
+            Value::Table(files.into_iter().collect())
+        }
+    }
+}

src/fs.rs

@@ -17,8 +17,6 @@ use tracing::{debug, info, instrument, warn};
 
 use super::config::{Config, Output};
 
-use super::json;
-
 /// A filesystem `FS` is just a vector of nullable inodes, where the index is
 /// the inode number.
 ///
@@ -181,15 +179,12 @@ impl FS {
     ///
     ///   - if `self.config.output == Output::Stdout` and `last_sync == false`,
     ///     nothing will happen (to prevent redundant writes to STDOUT)
-    ///
-    /// TODO 2021-06-16 need some reference to the output format to do the right
-    /// thing
     #[instrument(level = "debug", skip(self), fields(synced = self.dirty.get(), dirty = self.dirty.get()))]
     pub fn sync(&self, last_sync: bool) {
         info!("called");
         debug!("{:?}", self.inodes);
 
-        if !self.synced.get() && !self.dirty.get() {
+        if self.synced.get() && !self.dirty.get() {
             info!("skipping sync; already synced and not dirty");
             return;
         }
@@ -202,7 +197,7 @@ impl FS {
             _ => (),
         };
 
-        json::save_fs(self);
+        self.config.output_format.save(self);
         self.dirty.set(false);
         self.synced.set(true);
     }

src/json.rs

@@ -1,227 +0,0 @@
-use std::collections::HashMap;
-use std::fs::File;
-use std::str::FromStr;
-
-use serde_json::Value;
-
-use tracing::{debug, info, instrument};
-
-use fuser::FileType;
-
-use super::config::{Config, Output};
-use super::fs::{DirEntry, DirType, Entry, Inode, FS};
-
-/// Parses JSON into a value; just a shim for `serde_json::from_reader`.
-#[instrument(level = "info", skip(reader))]
-pub fn parse(reader: Box<dyn std::io::BufRead>) -> Value {
-    serde_json::from_reader(reader).expect("JSON")
-}
-
-/// Predicts filetypes from JSON values.
-///
-/// `Value::Object` and `Value::Array` map to directories; everything else is a
-/// regular file.
-fn kind(v: &Value) -> FileType {
-    match v {
-        Value::Object(_) | Value::Array(_) => FileType::Directory,
-        _ => FileType::RegularFile,
-    }
-}
-
-/// Calculates the size of a JSON value, i.e., the number of AST nodes used to
-/// represent it. Used for pre-allocating space for inodes in `fs()` below.
-fn size(v: &Value) -> usize {
-    match v {
-        Value::Null | Value::Bool(_) | Value::Number(_) | Value::String(_) => 1,
-        Value::Array(vs) => vs.iter().map(|v| size(v)).sum::<usize>() + 1,
-        Value::Object(fvs) => fvs.iter().map(|(_, v)| size(v)).sum::<usize>() + 1,
-    }
-}
-
-/// Generates `fs::FS` from a `serde_json::Value` in a particular `Config`.
-///
-/// The current implementation is eager: it preallocates enough inodes and then
-/// fills them in using a depth-first traversal.
-///
-/// Invariant: the index in the vector is the inode number. Inode 0 is invalid,
-/// and is left empty.
-#[instrument(level = "info", skip(v, config))]
-pub fn load_fs(config: Config, v: Value) -> FS {
-    let mut inodes: Vec<Option<Inode>> = Vec::new();
-
-    // reserve space for everyone else
-    // won't work with streaming or lazy generation, but avoids having to resize the vector midway through
-    inodes.resize_with(size(&v) + 1, || None);
-    info!("allocated {} inodes", inodes.len());
-
-    let mut next_id = fuser::FUSE_ROOT_ID;
-    // parent inum, inum, value
-    let mut worklist: Vec<(u64, u64, Value)> = Vec::new();
-
-    if !(v.is_array() || v.is_object()) {
-        panic!(
-            "Unable to build a filesystem out of the primitive value '{}'",
-            v
-        );
-    }
-    worklist.push((next_id, next_id, v));
-    next_id += 1;
-
-    while !worklist.is_empty() {
-        let (parent, inum, v) = worklist.pop().unwrap();
-
-        let nl = if config.add_newlines { "\n" } else { "" };
-        let entry = match v {
-            Value::Null => Entry::File("".into()),
-            Value::Bool(b) => Entry::File(format!("{}{}", b, nl).into_bytes()),
-            Value::Number(n) => Entry::File(format!("{}{}", n, nl).into_bytes()),
-            Value::String(s) => {
-                let contents = if s.ends_with('\n') { s } else { s + nl };
-                Entry::File(contents.into_bytes())
-            }
-            Value::Array(vs) => {
-                let mut children = HashMap::new();
-                children.reserve(vs.len());
-
-                let num_elts = vs.len() as f64;
-                let width = num_elts.log10().ceil() as usize;
-
-                for (i, child) in vs.into_iter().enumerate() {
-                    // TODO 2021-06-08 ability to add prefixes
-                    let name = if config.pad_element_names {
-                        format!("{:0width$}", i, width = width)
-                    } else {
-                        format!("{}", i)
-                    };
-
-                    children.insert(
-                        name,
-                        DirEntry {
-                            inum: next_id,
-                            kind: kind(&child),
-                        },
-                    );
-                    worklist.push((inum, next_id, child));
-                    next_id += 1;
-                }
-
-                Entry::Directory(DirType::List, children)
-            }
-            Value::Object(fvs) => {
-                let mut children = HashMap::new();
-                children.reserve(fvs.len());
-
-                for (field, child) in fvs.into_iter() {
-                    let original = field.clone();
-                    let mut nfield = config.normalize_name(field);
-
-                    while children.contains_key(&nfield) {
-                        nfield.push('_');
-                    }
-
-                    if original != nfield {
-                        info!(
-                            "renamed {} to {} (inode {} with parent {})",
-                            original, nfield, next_id, parent
-                        );
-                    }
-
-                    children.insert(
-                        nfield,
-                        DirEntry {
-                            inum: next_id,
-                            kind: kind(&child),
-                        },
-                    );
-
-                    worklist.push((inum, next_id, child));
-                    next_id += 1;
-                }
-
-                Entry::Directory(DirType::Named, children)
-            }
-        };
-
-        inodes[inum as usize] = Some(Inode {
-            parent,
-            inum,
-            entry,
-        });
-    }
-    assert_eq!(inodes.len() as u64, next_id);
-
-    FS::new(inodes, config)
-}
-
-#[instrument(level = "info", skip(fs))]
-pub fn save_fs(fs: &FS) {
-    let writer: Box<dyn std::io::Write> = match &fs.config.output {
-        Output::Stdout => {
-            debug!("outputting on STDOUT");
-            Box::new(std::io::stdout())
-        }
-        Output::File(path) => {
-            debug!("output {}", path.display());
-            Box::new(File::create(path).unwrap())
-        }
-        Output::Quiet => {
-            debug!("no output path, skipping");
-            return;
-        }
-    };
-
-    let v = value_from_fs(fs, fuser::FUSE_ROOT_ID);
-    debug!("outputting {}", v);
-    serde_json::to_writer(writer, &v).unwrap();
-}
-
-pub fn value_from_fs(fs: &FS, inum: u64) -> Value {
-    match &fs.get(inum).unwrap().entry {
-        Entry::File(contents) => {
-            // TODO 2021-06-16 better newline handling
-            let contents = match String::from_utf8(contents.clone()) {
-                Ok(mut contents) => {
-                    if fs.config.add_newlines && contents.ends_with('\n') {
-                        contents.truncate(contents.len() - 1);
-                    }
-                    contents
-                }
-                Err(_) => unimplemented!("binary data JSON serialization"),
-            };
-
-            if contents.is_empty() {
-                Value::Null
-            } else if contents == "true" {
-                Value::Bool(true)
-            } else if contents == "false" {
-                Value::Bool(false)
-            } else if let Ok(n) = serde_json::Number::from_str(&contents) {
-                Value::Number(n)
-            } else {
-                Value::String(contents)
-            }
-        }
-        Entry::Directory(DirType::List, files) => {
-            let mut entries = Vec::with_capacity(files.len());
-
-            let mut files = files.iter().collect::<Vec<_>>();
-            files.sort_unstable_by(|(name1, _), (name2, _)| name1.cmp(name2));
-            for (_name, DirEntry { inum, .. }) in files.iter() {
-                let v = value_from_fs(fs, *inum);
-                entries.push(v);
-            }
-
-            Value::Array(entries)
-        }
-        Entry::Directory(DirType::Named, files) => {
-            let mut entries = serde_json::map::Map::new();
-
-            for (name, DirEntry { inum, .. }) in files.iter() {
-                let v = value_from_fs(fs, *inum);
-                entries.insert(name.into(), v);
-            }
-
-            Value::Object(entries)
-        }
-    }
-}

src/main.rs

@@ -3,15 +3,16 @@ use std::path::PathBuf;
 
 use clap::{App, Arg};
 
-use tracing::{error, info, warn};
+use tracing::{debug, error, info, warn};
 use tracing_subscriber::prelude::*;
 use tracing_subscriber::{filter::EnvFilter, fmt};
 
 mod config;
+mod format;
 mod fs;
-mod json;
 
 use config::{Config, Output};
+use format::Format;
 
 use fuser::MountOption;
 
@@ -102,6 +103,22 @@ fn main() {
                 .overrides_with("OUTPUT")            
                 .overrides_with("NOOUTPUT")
         )
+        .arg(
+            Arg::with_name("SOURCE_FORMAT")
+                .help("Specify the source format explicitly (by default, automatically inferred from filename extension)")
+                .long("source")
+                .short("s")
+                .takes_value(true)
+                .possible_values(format::POSSIBLE_FORMATS)
+        )
+        .arg(
+            Arg::with_name("TARGET_FORMAT")
+                .help("Specify the target format explicitly (by default, automatically inferred from filename extension)")
+                .long("target")
+                .short("t")
+                .takes_value(true)
+                .possible_values(format::POSSIBLE_FORMATS)
+        )
         .arg(
             Arg::with_name("MOUNT")
                 .help("Sets the mountpoint")
@@ -176,8 +193,6 @@ fn main() {
         };
     }
 
-    let autounmount = args.is_present("AUTOUNMOUNT");
-
     // TODO 2021-06-08 infer and create mountpoint from filename as possible
     let mount_point = Path::new(args.value_of("MOUNT").expect("mount point"));
     if !mount_point.exists() {
@@ -229,26 +244,111 @@ fn main() {
     } else {
         Output::Stdout
     };
-    let reader: Box<dyn std::io::BufRead> = if input_source == "-" {
-        Box::new(std::io::BufReader::new(std::io::stdin()))
-    } else {
-        let file = std::fs::File::open(input_source).unwrap_or_else(|e| {
-            error!("Unable to open {} for JSON input: {}", input_source, e);
-            std::process::exit(1);
-        });
-        Box::new(std::io::BufReader::new(file))
+
+    // try to autodetect the input format.
+    //
+    // first see if it's specified and parses okay.
+    //
+    // then see if we can pull it out of the extension.
+    //
+    // then give up and use json
+    config.input_format = match args
+        .value_of("SOURCE_FORMAT")
+        .ok_or(format::ParseFormatError::NoFormatProvided)
+        .and_then(|s| s.parse::<Format>())
+    {
+        Ok(source_format) => source_format,
+        Err(e) => {
+            match e {
+                format::ParseFormatError::NoSuchFormat(s) => {
+                    warn!("Unrecognized format '{}', inferring from input.", s)
+                }
+                format::ParseFormatError::NoFormatProvided => {
+                    debug!("Inferring format from input.")
+                }
+            };
+
+            match Path::new(input_source)
+                .extension() // will fail for STDIN, no worries
+                .map(|s| s.to_str().expect("utf8 filename").to_lowercase())
+            {
+                Some(s) => match s.parse::<Format>() {
+                    Ok(format) => format,
+                    Err(_) => {
+                        warn!("Unrecognized format {}, defaulting to JSON.", s);
+                        Format::Json
+                    }
+                },
+                None => Format::Json,
+            }
+        }
+    };
+
+    // try to autodetect the input format.
+    //
+    // first see if it's specified and parses okay.
+    //
+    // then see if we can pull it out of the extension (if specified)
+    //
+    // then give up and use the input format
+    config.output_format = match args
+        .value_of("TARGET_FORMAT")
+        .ok_or(format::ParseFormatError::NoFormatProvided)
+        .and_then(|s| s.parse::<Format>())
+    {
+        Ok(target_format) => target_format,
+        Err(e) => {
+            match e {
+                format::ParseFormatError::NoSuchFormat(s) => {
+                    warn!(
+                        "Unrecognized format '{}', inferring from input and output.",
+                        s
+                    )
+                }
+                format::ParseFormatError::NoFormatProvided => {
+                    debug!("Inferring output format from input.")
+                }
+            };
+
+            match args
+                .value_of("OUTPUT")
+                .and_then(|s| Path::new(s).extension())
+                .and_then(|s| s.to_str())
+            {
+                Some(s) => match s.parse::<Format>() {
+                    Ok(format) => format,
+                    Err(_) => {
+                        warn!(
+                            "Unrecognized format {}, defaulting to input format '{}'.",
+                            s, config.input_format
+                        );
+                        config.input_format
+                    }
+                },
+                None => config.input_format,
+            }
+        }
     };
 
     let mut options = vec![MountOption::FSName(input_source.into())];
-    if autounmount {
+    if args.is_present("AUTOUNMOUNT") {
         options.push(MountOption::AutoUnmount);
     }
     if config.read_only {
         options.push(MountOption::RO);
     }
 
-    let v = json::parse(reader);
-    let fs = json::load_fs(config, v);
+    let input_format = config.input_format;
+    let reader: Box<dyn std::io::Read> = if input_source == "-" {
+        Box::new(std::io::stdin())
+    } else {
+        let file = std::fs::File::open(input_source).unwrap_or_else(|e| {
+            error!("Unable to open {} for JSON input: {}", input_source, e);
+            std::process::exit(1);
+        });
+        Box::new(file)
+    };
+    let fs = input_format.load(reader, config);
 
     info!("mounting on {:?} with options {:?}", mount_point, options);
     fuser::mount2(fs, mount_point, &options).unwrap();

tests/bad_root.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+fail() {
+    echo FAILED: $1
+    if [ "$MNT" ]
+    then
+        umount "$MNT"
+        rmdir "$MNT"
+        rm "$MSG" "$OUT"
+    fi
+    exit 1
+}
+
+MNT=$(mktemp -d)
+OUT=$(mktemp)
+MSG=$(mktemp)
+
+ffs "$MNT" ../json/null.json >"$OUT" 2>"$MSG" &
+PID=$!
+sleep 1
+kill -0 $PID >/dev/null 2>&1 && fail process
+cat "$MSG" | grep -i -e  "must be a directory" >/dev/null 2>&1 || fail error
+[ -f "$OUT" ] && ! [ -s "$OUT" ] || fail output
+sleep 1
+
+rmdir "$MNT" || fail mount
+rm "$MSG" "$OUT"

tests/bad_root_stdin.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+fail() {
+    echo FAILED: $1
+    if [ "$MNT" ]
+    then
+        umount "$MNT"
+        rmdir "$MNT"
+        rm "$MSG" "$OUT"
+    fi
+    exit 1
+}
+
+MNT=$(mktemp -d)
+OUT=$(mktemp)
+MSG=$(mktemp)
+
+echo \"just a string\" | ffs "$MNT" >"$OUT" 2>"$MSG" &
+PID=$!
+sleep 1
+kill -0 $PID >/dev/null 2>&1 && fail process
+cat "$MSG" | grep -i -e "must be a directory" >/dev/null 2>&1 || fail error
+[ -f "$OUT" ] && ! [ -s "$OUT" ] || fail output
+sleep 1
+
+rmdir "$MNT" || fail mount
+rm "$MSG" "$OUT"

tests/basic_object_stdin.sh

@@ -0,0 +1,34 @@
+#!/bin/sh
+
+fail() {
+    echo FAILED: $1
+    if [ "$MNT" ]
+    then
+        cd
+        umount "$MNT"
+        rmdir "$MNT"
+    fi
+    exit 1
+}
+
+MNT=$(mktemp -d)
+
+cat ../json/object.json | ffs "$MNT" &
+PID=$!
+sleep 2
+cd "$MNT"
+case $(ls) in
+    (eyes*fingernails*human*name) ;;
+    (*) fail ls;;
+esac
+[ "$(cat name)" = "Michael Greenberg" ] || fail name
+[ "$(cat eyes)" -eq 2 ] || fail eyes
+[ "$(cat fingernails)" -eq 10 ] || fail fingernails
+[ "$(cat human)" = "true" ] || fail human
+cd - >/dev/null 2>&1
+umount "$MNT" || fail unmount
+sleep 1
+
+kill -0 $PID >/dev/null 2>&1 && fail process
+
+rmdir "$MNT" || fail mount

tests/basic_toml.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+fail() {
+    echo FAILED: $1
+    if [ "$MNT" ]
+    then
+        cd
+        umount "$MNT"
+        rmdir "$MNT"
+    fi
+    exit 1
+}
+
+MNT=$(mktemp -d)
+
+ffs "$MNT" ../toml/eg.toml &
+PID=$!
+sleep 2
+case $(ls "$MNT") in
+    (clients*database*owner*servers*title) ;;
+    (*) fail ls;;
+esac
+[ "$(cat $MNT/title)" = "TOML Example" ] || fail title
+[ "$(cat $MNT/owner/dob)" = "1979-05-27T07:32:00-08:00" ] || fail dob
+
+umount "$MNT" || fail unmount
+sleep 1
+
+kill -0 $PID >/dev/null 2>&1 && fail process
+
+rmdir "$MNT" || fail mount

tests/binary.sh

@@ -0,0 +1,65 @@
+#!/bin/sh
+
+fail() {
+    echo FAILED: $1
+    if [ "$MNT" ]
+    then
+        cd
+        umount "$MNT"
+        rmdir "$MNT"
+        rm "$TGT"
+        rm "$TGT2"
+        rm "$ICO"
+    fi
+    exit 1
+}
+
+if [ "$RUNNER_OS" = "Linux" ]; then
+    decode() {
+        base64 -d $1 >$2
+    }
+elif [ "$RUNNER_OS" = "macOS" ]; then
+    decode() {
+        base64 -D -i $1 -o $2
+    }
+else
+    fail os
+fi
+
+MNT=$(mktemp -d)
+TGT=$(mktemp)
+TGT2=$(mktemp)
+
+ffs "$MNT" ../json/object.json >"$TGT" &
+PID=$!
+sleep 2
+cp ../binary/twitter.ico "$MNT"/favicon
+umount "$MNT" || fail unmount1    
+sleep 1
+kill -0 $PID >/dev/null 2>&1 && fail process1
+
+# easiest to just test using ffs, but would be cool to get outside validation
+[ -f "$TGT" ] || fail output1
+[ -s "$TGT" ] || fail output2
+grep favicon "$TGT" >/dev/null 2>&1 || fail text
+ffs --no-output "$MNT" "$TGT" >"$TGT2" &
+PID=$!
+sleep 2
+
+ICO=$(mktemp)
+
+ls "$MNT" | grep favicon >/dev/null 2>&1 || fail field
+decode "$MNT"/favicon "$ICO"
+diff ../binary/twitter.ico "$ICO" || fail diff
+
+umount "$MNT" || fail unmount2
+sleep 1
+kill -0 $PID >/dev/null 2>&1 && fail process2
+
+[ -f "$TGT2" ] || fail tgt2
+[ -s "$TGT2" ] && fail tgt2_nonempty
+
+rmdir "$MNT" || fail mount
+rm "$TGT"
+rm "$TGT2"
+rm "$ICO"

tests/json_to_toml.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+
+fail() {
+    echo FAILED: $1
+    if [ "$MNT" ]
+    then
+        cd
+        umount "$MNT"
+        rmdir "$MNT"
+        rm "$TGT"
+    fi
+    exit 1
+}
+
+MNT=$(mktemp -d)
+TGT=$(mktemp)
+
+ffs --source json --target toml -o "$TGT" "$MNT" ../json/single.json &
+PID=$!
+sleep 2
+umount "$MNT" || fail unmount1    
+sleep 1
+kill -0 $PID >/dev/null 2>&1 && fail process1
+
+diff "$TGT" ../toml/single.toml || fail diff
+
+rmdir "$MNT" || fail mount
+rm "$TGT"
+

tests/override_infer.sh

@@ -0,0 +1,32 @@
+#!/bin/sh
+
+fail() {
+    echo FAILED: $1
+    if [ "$MNT" ]
+    then
+        cd
+        umount "$MNT"
+        rmdir "$MNT"
+        rm "$SRC" "$TGT"
+    fi
+    exit 1
+}
+
+MNT=$(mktemp -d)
+SRC=$(mktemp)
+TGT=$(mktemp)
+
+cp ../toml/single.toml "$SRC"
+
+ffs --source toml --target json -o "$TGT" "$MNT" "$SRC" &
+PID=$!
+sleep 2
+umount "$MNT" || fail unmount1    
+sleep 1
+kill -0 $PID >/dev/null 2>&1 && fail process1
+
+diff "$TGT" ../json/single.json || fail diff
+
+rmdir "$MNT" || fail mount
+rm "$SRC" "$TGT"
+

tests/toml_output.sh

@@ -0,0 +1,50 @@
+#!/bin/sh
+
+fail() {
+    echo FAILED: $1
+    if [ "$MNT" ]
+    then
+        cd
+        umount "$MNT"
+        rmdir "$MNT"
+        rm "$TOML"
+    fi
+    exit 1
+}
+
+MNT=$(mktemp -d)
+TOML=$(mktemp)
+
+mv "$TOML" "$TOML".toml
+TOML="$TOML".toml
+
+cp ../toml/eg.toml "$TOML"
+
+ffs -i "$MNT" "$TOML" &
+PID=$!
+sleep 2
+case $(ls "$MNT") in
+    (clients*database*owner*servers*title) ;;
+    (*) fail ls;;
+esac
+[ "$(cat $MNT/title)" = "TOML Example" ] || fail title
+[ "$(cat $MNT/owner/dob)" = "1979-05-27T07:32:00-08:00" ] || fail dob
+echo aleph >"$MNT/clients/hosts/2"
+echo tav >"$MNT/clients/hosts/3"
+umount "$MNT" || fail unmount1
+sleep 1
+kill -0 $PID >/dev/null 2>&1 && fail process1
+
+ffs --readonly --no-output "$MNT" "$TOML" &
+PID=$!
+sleep 2
+[ "$(cat $MNT/clients/hosts/0)" = "alpha" ] || fail hosts0
+[ "$(cat $MNT/clients/hosts/1)" = "omega" ] || fail hosts1
+[ "$(cat $MNT/clients/hosts/2)" = "aleph" ] || fail hosts2
+[ "$(cat $MNT/clients/hosts/3)" = "tav"   ] || fail hosts3
+umount "$MNT" || fail unmount2
+sleep 1
+kill -0 $PID >/dev/null 2>&1 && fail process2
+
+rmdir "$MNT" || fail mount
+rm "$TOML"

tests/toml_to_json.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+
+fail() {
+    echo FAILED: $1
+    if [ "$MNT" ]
+    then
+        cd
+        umount "$MNT"
+        rmdir "$MNT"
+        rm "$TGT"
+    fi
+    exit 1
+}
+
+MNT=$(mktemp -d)
+TGT=$(mktemp)
+
+ffs --source toml --target json -o "$TGT" "$MNT" ../toml/single.toml &
+PID=$!
+sleep 2
+umount "$MNT" || fail unmount1    
+sleep 1
+kill -0 $PID >/dev/null 2>&1 && fail process1
+
+diff "$TGT" ../json/single.json || fail diff
+
+rmdir "$MNT" || fail mount
+rm "$TGT"
+

toml/.gitignore

@@ -0,0 +1 @@
+mnt

toml/eg.toml

@@ -0,0 +1,33 @@
+# This is a TOML document.
+
+title = "TOML Example"
+
+[owner]
+name = "Tom Preston-Werner"
+dob = 1979-05-27T07:32:00-08:00 # First class dates
+
+[database]
+server = "192.168.1.1"
+ports = [ 8000, 8001, 8002 ]
+connection_max = 5000
+enabled = true
+
+[servers]
+
+  # Indentation (tabs and/or spaces) is allowed but not required
+  [servers.alpha]
+  ip = "10.0.0.1"
+  dc = "eqdc10"
+
+  [servers.beta]
+  ip = "10.0.0.2"
+  dc = "eqdc10"
+
+[clients]
+data = [ ["gamma", "delta"], [1, 2] ]
+
+# Line breaks are OK when inside arrays
+hosts = [
+  "alpha",
+  "omega"
+]

toml/single.toml

@@ -0,0 +1 @@
+onlyone = "highlander"