diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 399d699..c2484ef 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: os: -# - macos-11 + - macos-11 - ubuntu-latest runs-on: ${{ matrix.os }} @@ -32,21 +32,67 @@ jobs: - name: Checkout code uses: actions/checkout@v2 - - name: Build ffs and run unit tests + - name: Build ffs/pack/unpack and run unit tests run: | cargo build --verbose --all --release cargo test - - name: Integration tests + - name: Integration tests for ffs and pack/unpack (Linux) + if: contains(matrix.os, 'ubuntu') run: PATH="$(pwd)/target/release:$PATH" ./run_tests.sh + - name: Integration tests for pack/unpack only (macOS) + if: contains(matrix.os, 'macos') + run: PATH="$(pwd)/target/release:$PATH" ./run_tests.sh unpack + + - name: Upload macOS release build + uses: actions/upload-artifact@v2 + if: contains(matrix.os, 'macos') + with: + name: ffs.macos + path: | + target/release/ffs + target/release/pack + target/release/unpack + + - name: Upload Linux release build + uses: actions/upload-artifact@v2 + if: contains(matrix.os, 'ubuntu') + with: + name: ffs.linux + path: | + target/release/ffs + target/release/pack + target/release/unpack + + benchmarks: + needs: build + runs-on: ubuntu-latest + + steps: + - name: Install dependencies (FUSE, attr) + run: | + if [ "$RUNNER_OS" = "Linux" ]; then + sudo apt-get install fuse libfuse-dev pkg-config attr + else + echo Unsupported RUNNER_OS=$RUNNER_OS + exit 1 + fi + + - name: Checkout code + uses: actions/checkout@v2 + + - name: Download binaries + uses: actions/download-artifact@v2 + - name: Install R uses: r-lib/actions/setup-r@v2 - name: Benchmarks run: | Rscript -e "tries <- 0; while (!require('ggplot2') && tries < 3) { cat(sprintf('TRY %d\n', tries)); install.packages('ggplot2', repos = 'https://cloud.r-project.org/'); tries <- tries + 1; }" - PATH="$(pwd)/target/release:$PATH" ./run_bench.sh -n 3 + chmod +x $(pwd)/ffs.linux/ffs + PATH="$(pwd)/ffs.linux:$PATH" FFS="$(pwd)/ffs.linux/ffs" ./run_bench.sh -n 3 # grab latest directory (output of run_bench) DATADIR=bench/$(ls -ct bench/ | head -n 1) [ -d $DATADIR ] && ls $DATADIR | grep log >/dev/null || { echo "No log files found in $DATADIR. What's going on?"; tree bench; exit 1; } @@ -55,35 +101,14 @@ jobs: do mv $x data/${x##*_} done - - - name: Upload macOS release build - uses: actions/upload-artifact@v2 - if: contains(matrix.os, 'macos') - with: - name: ffs.macos - path: target/release/ffs - - name: Upload Linux release build - uses: actions/upload-artifact@v2 - if: contains(matrix.os, 'ubuntu') - with: - name: ffs.linux - path: target/release/ffs - - - name: Upload macOS benchmark data - uses: actions/upload-artifact@v2 - if: contains(matrix.os, 'macos') - with: - name: benchmarks.macos - path: data - - name: Upload Linux benchmark data uses: actions/upload-artifact@v2 if: contains(matrix.os, 'ubuntu') with: name: benchmarks.linux - path: data - + path: data + prerelease: needs: build runs-on: ubuntu-latest @@ -96,8 +121,17 @@ jobs: - name: Rename binaries run: | mkdir ffs - mv ffs.linux/ffs ffs/ffs.linux - [ -d ffs.macos ] && [ -f ffs.macos/ffs ] && ffs.macos/ffs ffs/ffs.macos || echo "macOS is disabled 😢" + mv ffs.linux/ffs ffs/ffs.linux + mv ffs.linux/pack ffs/pack.linux + mv ffs.linux/unpack ffs/unpack.linux + if [ -d ffs.macos ] + then + mv ffs.macos/ffs ffs/ffs.macos + mv ffs.macos/pack ffs/pack.macos + mv ffs.macos/unpack ffs/unpack.macos + else + echo "macOS is disabled 😢" + fi - name: Deploy 'latest' release uses: "marvinpinto/action-automatic-releases@latest" @@ -108,6 +142,8 @@ jobs: title: "Latest development build" files: | ffs/ffs.* + ffs/pack.* + ffs/unpack.* diff --git a/Cargo.lock b/Cargo.lock index e5f6ef6..0d81dc8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3,12 +3,21 @@ version = 3 [[package]] -name = "ansi_term" -version = "0.11.0" +name = "aho-corasick" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783" dependencies = [ - "winapi", + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", ] [[package]] @@ -33,21 +42,27 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "base64" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "bitflags" -version = "1.2.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bumpalo" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "byteorder" @@ -55,6 +70,12 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + [[package]] name = "cfg-if" version = "1.0.0" @@ -63,11 +84,11 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.19" +version = "0.4.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" dependencies = [ - "libc", + "iana-time-zone", "num-integer", "num-traits", "winapi", @@ -75,11 +96,11 @@ dependencies = [ [[package]] name = "clap" -version = "2.33.3" +version = "2.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" dependencies = [ - "ansi_term 0.11.0", + "ansi_term", "atty", "bitflags", "strsim", @@ -88,6 +109,66 @@ dependencies = [ "vec_map", ] +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + +[[package]] +name = "cxx" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c00419335c41018365ddf7e4d5f1c12ee3659ddcf3e01974650ba1de73d038" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb8307ad413a98fff033c8545ecf133e3257747b3bae935e7602aab8aa92d4ca" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn 2.0.4", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc52e2eb08915cb12596d29d55f0b5384f00d697a646dbd269b6ecb0fbd9d31" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "631569015d0d8d54e6c241733f944042623ab6df7bc3be7466874b05fcdb1c5f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.4", +] + [[package]] name = "ffs" version = "0.1.2" @@ -96,18 +177,20 @@ dependencies = [ "clap", "fuser", "libc", + "regex", "serde_json", "toml", "tracing", "tracing-subscriber", + "xattr", "yaml-rust", ] [[package]] name = "fuser" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aef8400a4ea1d18a8302e2952f5137a9a21ab257825ccc7d67db4a8018b89022" +checksum = "104ed58f182bc2975062cd3fab229e82b5762de420e26cf5645f661402694599" dependencies = [ "libc", "log", @@ -121,18 +204,51 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" dependencies = [ "libc", ] [[package]] -name = "itoa" -version = "0.4.7" +name = "iana-time-zone" +version = "0.1.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" +checksum = "0c17cc76786e99f8d2f055c11159e7f0091c42474dcc3189fbab96072e873e6d" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + +[[package]] +name = "itoa" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" + +[[package]] +name = "js-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +dependencies = [ + "wasm-bindgen", +] [[package]] name = "lazy_static" @@ -142,21 +258,30 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.95" +version = "0.2.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "789da6d93f1b866ffe175afc5322a4d76c038605a1c3319bb57b06967ca98a36" +checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" + +[[package]] +name = "link-cplusplus" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" +dependencies = [ + "cc", +] [[package]] name = "linked-hash-map" -version = "0.5.4" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "log" -version = "0.4.14" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ "cfg-if", ] @@ -167,20 +292,20 @@ version = "0.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f099785f7595cc4b4553a174ce30dd7589ef93391ff414dbb67f62392b9e0ce1" dependencies = [ - "regex-automata", + "regex-automata 0.1.10", ] [[package]] name = "memchr" -version = "2.4.0" +version = "2.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" +checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" [[package]] name = "num-integer" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ "autocfg", "num-traits", @@ -188,18 +313,18 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ "autocfg", ] [[package]] name = "once_cell" -version = "1.7.2" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "page_size" @@ -213,41 +338,44 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.6" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc0e1f259c92177c30a4c9d177246edd0a3568b25756a977d0632cf8fa37e905" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" [[package]] name = "pkg-config" -version = "0.3.19" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "proc-macro2" -version = "1.0.27" +version = "1.0.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038" +checksum = "1d0e1ae9e836cc3beddd63db0df682593d7e2d3d891ae8c9083d2113e1744224" dependencies = [ - "unicode-xid", + "unicode-ident", ] [[package]] name = "quote" -version = "1.0.9" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" dependencies = [ "proc-macro2", ] [[package]] name = "regex" -version = "1.5.4" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" +checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" dependencies = [ - "regex-syntax", + "aho-corasick", + "memchr", + "regex-automata 0.3.8", + "regex-syntax 0.7.5", ] [[package]] @@ -256,32 +384,55 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" dependencies = [ - "regex-syntax", + "regex-syntax 0.6.29", +] + +[[package]] +name = "regex-automata" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.7.5", ] [[package]] name = "regex-syntax" -version = "0.6.25" +version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + +[[package]] +name = "regex-syntax" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "ryu" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" + +[[package]] +name = "scratch" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" [[package]] name = "serde" -version = "1.0.126" +version = "1.0.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03" +checksum = "771d4d9c4163ee138805e12c710dd365e4f44be8be0503cb1bb9eb989425d9c9" [[package]] name = "serde_json" -version = "1.0.64" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" +checksum = "1c533a59c9d8a93a09c6ab31f0fd5e5f4dd1b8fc9434804029839884765d04ea" dependencies = [ "itoa", "ryu", @@ -290,18 +441,18 @@ dependencies = [ [[package]] name = "sharded-slab" -version = "0.1.1" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79c719719ee05df97490f80a45acfc99e5a30ce98a1e4fb67aee422745ae14e3" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" dependencies = [ "lazy_static", ] [[package]] name = "smallvec" -version = "1.6.1" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "strsim" @@ -311,25 +462,33 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "syn" -version = "1.0.72" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e8cdbefb79a9a5a65e0db8b47b723ee907b7c7f8496c76a1770b5c310bab82" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2", "quote", - "unicode-xid", + "unicode-ident", ] [[package]] -name = "synstructure" -version = "0.12.4" +name = "syn" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" +checksum = "2c622ae390c9302e214c31013517c2061ecb2699935882c60a9b37f82f8625ae" dependencies = [ "proc-macro2", "quote", - "syn", - "unicode-xid", + "unicode-ident", +] + +[[package]] +name = "termcolor" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +dependencies = [ + "winapi-util", ] [[package]] @@ -343,27 +502,28 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.1.3" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" dependencies = [ + "cfg-if", "once_cell", ] [[package]] name = "toml" -version = "0.5.8" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" dependencies = [ "serde", ] [[package]] name = "tracing" -version = "0.1.26" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", "pin-project-lite", @@ -373,29 +533,30 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.15" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c42e6fa53307c8a17e4ccd4dc81cf5ec38db9209f59b222210375b54ee40d1e2" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "tracing-core" -version = "0.1.18" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9ff14f98b1a4b289c6248a023c1c2fa1491062964e9fed67ab29c4e4da4a052" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" dependencies = [ - "lazy_static", + "once_cell", + "valuable", ] [[package]] name = "tracing-log" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6923477a48e41c1951f1999ef8bb5a3023eb723ceadafe78ffb65dc366761e3" +checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" dependencies = [ "lazy_static", "log", @@ -404,9 +565,9 @@ dependencies = [ [[package]] name = "tracing-serde" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb65ea441fbb84f9f6748fd496cf7f63ec9af5bca94dd86456978d055e8eb28b" +checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" dependencies = [ "serde", "tracing-core", @@ -414,11 +575,11 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.2.18" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa5553bf0883ba7c9cbe493b085c29926bd41b66afc31ff72cf17ff4fb60dcd5" +checksum = "0e0d2eaa99c3c2e41547cfa109e910a68ea03823cccad4a0525dcbc9b01e8c71" dependencies = [ - "ansi_term 0.12.1", + "ansi_term", "chrono", "lazy_static", "matchers", @@ -435,16 +596,16 @@ dependencies = [ ] [[package]] -name = "unicode-width" -version = "0.1.8" +name = "unicode-ident" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" [[package]] -name = "unicode-xid" -version = "0.2.2" +name = "unicode-width" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" [[package]] name = "users" @@ -456,12 +617,72 @@ dependencies = [ "log", ] +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + [[package]] name = "vec_map" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" +[[package]] +name = "wasm-bindgen" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 1.0.109", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" + [[package]] name = "winapi" version = "0.3.9" @@ -478,12 +699,96 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdacb41e6a96a052c6cb63a144f24900236121c6f63f4f8219fef5977ecb0c25" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "xattr" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea263437ca03c1522846a4ddafbca2542d0ad5ed9b784909d4b27b76f62bc34a" +dependencies = [ + "libc", +] + [[package]] name = "yaml-rust" version = "0.4.5" @@ -505,11 +810,11 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0fbc82b82efe24da867ee52e015e58178684bd9dd64c34e66bdf21da2582a9f" +checksum = "6505e6815af7de1746a08f69c69606bb45695a17149517680f3b2149713b19a3" dependencies = [ "proc-macro2", - "syn", - "synstructure", + "quote", + "syn 1.0.109", ] diff --git a/Cargo.toml b/Cargo.toml index 92c0c09..72b213b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,8 +29,10 @@ base64 = "0.13.0" clap = "2.0" fuser = "0.11" libc = "0.2.51" +regex = "1.9.5" serde_json = "1.0" toml = "0.5" tracing = "0.1" tracing-subscriber = "0.2.18" +xattr = "1.0.0" yaml-rust = "0.4.5" diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..659536b --- /dev/null +++ b/TODO.md @@ -0,0 +1,142 @@ +binary data gets treated as base64 + +# unpack + +JSON, TOML, YAML file -> file system hierarchy + +``` +# puts the foo data into the bar directory (making bar if it doesn't exist) +cat foo.json | unpack --into bar + +# puts the foo data into the foo directory (making foo if it doesn't exists) +unpack foo.json + +# in both of those, it's an error if foo or bar exist and are non-empty + +# unpack stdin (coming from baz) into quux, treating input as YAML +cat baz | unpack -s yaml --into quux +``` + +src/format.rs describes mappings from these formats into the `Nodelike` trait + +## a possible cut through the work: + +- [ ] get JSON to work by hand + + write some tests + +- [ ] get other formats work using `Nodelike` + + + wrinkle: YAML has a special notion of anchor that would be cool to treat as a sym- or hardlink + problem not actually worth thinking about + + write some more tests + +- [ ] implement options + + --debug + --exact + --no-xattr + --quiet + --time + --unpadded + --munge + --dirmode, --mode, --gid, --uid + -s, --source # rename to -t, --type ? + -m, --mount # rename to -i, --into ? + + write tests of unpack + write separate tests that compare ffs and unpack's behavior + `diff -r` might do the trick + xattr/uid/gid/mtime/etc. stuff is a bit more subtle + +## things to think about + +- [ ] read semi-structured data + - default to stdin + - but take a file (many files?!) + + output is... at a default mountpoint, or at a directory based on the filename + follow ffs lead here + +- [ ] options that matter + +- [ ] build the directory tree, write the data, set some xattrs as necessary, that's it + +- [ ] test + + follow the general lead of run_tests.sh and tests/*.sh + + how do we ensure that we don't hose the system? + + in docker? + in `chroot`? + with `pivot_root`? + +# pack + +file system hierarchy -> JSON, TOML, YAML file + +``` +# save /etc into a JSON file +pack /etc >config.json + +pack -o lib.yaml /usr/share/lib + +# -t specifying target type +pack -t toml . >bar.toml +``` + +- [ ] get it to work for just JSON + + + wrinkle: special file types (devices, FIFOs, etc.) + what does tar do? gunzip unzip and one other to see what's standard + + + wrinkle: permissions + `pack -o everything.json /` + what does tar etc. do? + + + wrinkle: hard and symlinks + + hardlinks are just files... worst case we copy + would be cool in YAML to have them be anchors + + symlinks can cause loops, can go outside of the root specified, etc. + cf. `cp`, `tar`, `find` options `cp -L` to specify following symlinks, `--nofollow` + but also: don't infinite loop + PATH_MAX + + i think there are good rust libraries for filesystem traversal + +- [ ] get it to work for `Nodelike` + +- [ ] implement options that matter + + --debug + --keep-macos-xattr + --pretty + --time + --munge + --exact + --quite + --target + --output + +# testing wrt ffs + +ffs and pack/unpack should behave as identically as possible + we should explicitly test this on fixed and maybe also random inputs + +# fuzzing + +generate random inputs and run unpack on them + +generate random filesystems and run pack on them (or run pack on random points in the FS) + +fuzz ffs itself? + +# performance + +- [ ] think about ramdisks + +- [ ] compare pack/unpack and ffs in a bunch of ways lol diff --git a/run_tests.sh b/run_tests.sh index 7c3bb6b..a455be3 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -10,6 +10,26 @@ then } PATH="$DEBUG:$PATH" fi +if ! which unpack >/dev/null 2>&1 +then + DEBUG="$(pwd)/target/debug" + [ -x "$DEBUG/unpack" ] || { + echo Couldn\'t find unpack on "$PATH" or in "$DEBUG". >&2 + echo Are you in the root directory of the repo? >&2 + exit 1 + } + PATH="$DEBUG:$PATH" +fi +if ! which pack >/dev/null 2>&1 +then + DEBUG="$(pwd)/target/debug" + [ -x "$DEBUG/pack" ] || { + echo Couldn\'t find pack on "$PATH" or in "$DEBUG". >&2 + echo Are you in the root directory of the repo? >&2 + exit 1 + } + PATH="$DEBUG:$PATH" +fi TOTAL=0 FAILED=0 @@ -17,13 +37,14 @@ ERRORS="" cd tests LOG=$(mktemp -d) +TESTS="$(find . -name "$1*.sh")" # spawn 'em all in parallel -for test in *.sh +for test in $TESTS do tname="$(basename ${test%*.sh})" printf "========== STARTING TEST: $tname\n" - (RUST_LOG="ffs=debug,fuser=debug"; export RUST_LOG; ./${test} >$LOG/$tname.out 2>$LOG/$tname.err; echo $?>$LOG/$tname.ec) & + (RUST_LOG="ffs=debug,unpack=debug,pack=debug,fuser=debug"; export RUST_LOG; ./${test} >$LOG/$tname.out 2>$LOG/$tname.err; echo $?>$LOG/$tname.ec) & : $((TOTAL += 1)) # don't slam 'em @@ -35,7 +56,7 @@ done wait -for test in *.sh +for test in $TESTS do tname="$(basename ${test%*.sh})" if [ "$(cat $LOG/$tname.ec)" -eq 0 ] diff --git a/src/main.rs b/src/bin/ffs.rs similarity index 96% rename from src/main.rs rename to src/bin/ffs.rs index f2d6d96..4cef077 100644 --- a/src/main.rs +++ b/src/bin/ffs.rs @@ -1,9 +1,8 @@ use tracing::{error, info, warn}; -mod cli; -mod config; -mod format; -mod fs; +use ffs::config; +use ffs::format; +use ffs::fs; use config::{Config, ERROR_STATUS_CLI, ERROR_STATUS_FUSE}; use format::Format; @@ -12,7 +11,7 @@ use fs::FS; use fuser::MountOption; fn main() { - let config = Config::from_args(); + let config = Config::from_ffs_args(); let mut options = vec![MountOption::FSName(format!("{}", config.input))]; if config.read_only { options.push(MountOption::RO); diff --git a/src/bin/pack.rs b/src/bin/pack.rs new file mode 100644 index 0000000..452d8a4 --- /dev/null +++ b/src/bin/pack.rs @@ -0,0 +1,397 @@ +use std::fs; + +use std::collections::BTreeMap; +use std::collections::HashMap; +use std::io::BufReader; +use std::io::Error; +use std::io::Read; +use std::path::PathBuf; +use std::str; +use std::str::FromStr; + +use tracing::{debug, error, info, warn}; + +use ffs::config::Config; +use ffs::config::Symlink; +use ffs::config::{ERROR_STATUS_CLI, ERROR_STATUS_FUSE}; +use ffs::format; +use ffs::time_ns; +use format::json::Value as JsonValue; +use format::toml::Value as TomlValue; +use format::yaml::Value as YamlValue; +use format::Format; +use format::Nodelike; +use format::Typ; + +use ::xattr; +use regex::Regex; + +pub struct SymlinkMapData { + link: PathBuf, + is_broken: bool, +} + +pub struct Pack { + // mapping of symlink to: + // PathBuf of link destination + // bool of whether symlink chain ends in a broken link + pub symlinks: HashMap, + depth: u32, + regex: Regex, +} + +impl Pack { + pub fn new() -> Self { + Self { + symlinks: HashMap::new(), + depth: 0, + regex: Regex::new("^-?[0-9]+").unwrap(), + } + } + + pub fn pack(&mut self, path: PathBuf, config: &Config) -> std::io::Result> + where + V: Nodelike + std::fmt::Display + Default, + { + // don't continue packing if max depth is reached + if config + .max_depth + .is_some_and(|max_depth| self.depth > max_depth) + { + return Ok(None); + } + + // get the type of data from xattr if it exists + let mut path_type: Vec = Vec::new(); + + if path.is_symlink() { + match &config.symlink { + Symlink::NoFollow => { + // early return because we want to ignore symlinks, + return Ok(None); + } + Symlink::Follow => { + let mut link_trail = Vec::new(); + let mut link_follower = path.clone(); + while link_follower.is_symlink() { + if link_trail.contains(&link_follower) { + error!("Symlink loop detected at {:?}.", link_follower); + std::process::exit(ERROR_STATUS_FUSE); + } + link_trail.push(link_follower.clone()); + + if path_type.is_empty() { + // get the xattr of the first symlink that has it defined. + // this has the effect of inheriting xattrs from links down the + // chain. + match xattr::get(&link_follower, "user.type") { + Ok(Some(xattr)) if config.allow_xattr => path_type = xattr, + Ok(_) | Err(_) => (), + // TODO(nad) 2023-08-07: maybe unnecessary to check for ._ as + // symlink? + // Err(_) => { + // // Cannot call xattr::get on ._ file + // warn!( + // "._ files, like {:?}, prevent xattr calls. It will be encoded in base64.", + // link_follower + // ); + // path_type = b"bytes".to_vec() + // } + }; + } + + // add the link to the mapping to reduce future read_link calls for each + // symlink on the chain. + if !self.symlinks.contains_key(&link_follower) { + let link = link_follower.read_link()?; + self.symlinks.insert( + link_follower.clone(), + SymlinkMapData { + link: if link.is_absolute() { + link + } else { + link_follower.clone().parent().unwrap().join(link) + }, + is_broken: false, + }, + ); + } + if self.symlinks[&link_follower].is_broken { + // .1 is a bool to tell if symlink is broken + // the symlink either is broken or links to a broken symlink. + // stop the traversal immediately and update mapping if possible + break; + } + link_follower = self.symlinks[&link_follower].link.clone(); + } + + if self.symlinks[link_trail.last().unwrap()].is_broken + || !link_follower.exists() + { + // the symlink is broken, so don't pack this file. + warn!( + "The symlink at the end of the chain starting from '{:?}' is broken.", + path + ); + for link in link_trail { + let symlink_map_data = &self.symlinks[&link]; + self.symlinks.insert( + link, + SymlinkMapData { + link: symlink_map_data.link.to_path_buf(), + is_broken: true, + }, + ); + } + return Ok(None); + } + + // pack reached the actual destination + let canonicalized = link_follower.canonicalize()?; + if path.starts_with(&canonicalized) { + error!( + "The symlink {:?} points to some ancestor directory: {:?}, causing an infinite loop.", + path, canonicalized + ); + std::process::exit(ERROR_STATUS_FUSE); + } + if !config.allow_symlink_escape + && !canonicalized.starts_with(config.mount.as_ref().unwrap()) + { + warn!("The symlink {:?} points to some file outside of the directory being packed. \ + Specify --allow-symlink-escape to allow pack to follow this symlink.", path); + return Ok(None); + } + } + } + } + + // if the xattr is still not set, either path is not a symlink or + // none of the symlinks on the chain have an xattr. Use the actual file's xattr + if path_type.is_empty() { + let canonicalized = path.canonicalize()?; + path_type = match xattr::get(&canonicalized, "user.type") { + Ok(Some(xattr_type)) if config.allow_xattr => xattr_type, + Ok(_) => b"auto".to_vec(), + Err(_) => { + // Cannot call xattr::get on ._ file + warn!( + "._ files, like {:?}, prevent xattr calls. It will be encoded in base64.", + path + ); + b"bytes".to_vec() + } + }; + } + + // convert detected xattr from Vec to str + let mut path_type: &str = str::from_utf8(&path_type).unwrap(); + + // resolve path type if it is 'auto' + if path.is_dir() && (path_type == "auto" || path_type != "named" && path_type != "list") { + if path_type != "auto" { + warn!( + "Unknown directory type '{}'. Possible types are 'named' or 'list'. \ + Resolving type automatically.", + path_type + ); + } + let all_files_begin_with_num = fs::read_dir(path.clone())? + .map(|res| res.map(|e| e.path())) + .map(|e| e.unwrap().file_name().unwrap().to_str().unwrap().to_owned()) + .all(|filename| { + filename.chars().nth(0).unwrap().is_digit(10) + || filename.len() > 1 + && filename.chars().nth(0).unwrap() == '-' + && filename.chars().nth(1).unwrap().is_digit(10) + }); + if all_files_begin_with_num { + path_type = "list" + } else { + path_type = "named" + }; + } + + info!("type of {:?} is {}", path, path_type); + + // return the value based on determined type + match path_type { + "named" => { + let mut children = fs::read_dir(path.clone())? + .map(|res| res.map(|e| e.path())) + .collect::, Error>>()?; + children.sort_unstable_by(|a, b| a.file_name().cmp(&b.file_name())); + + let mut entries = BTreeMap::new(); + + for child in &children { + let child_name = child.file_name().unwrap().to_str().unwrap(); + if config.ignored_file(child_name) { + warn!("skipping ignored file {:?}", child_name); + continue; + } + let name: String; + match xattr::get(&child, "user.original_name") { + Ok(Some(original_name)) if config.allow_xattr => { + let old_name = str::from_utf8(&original_name).unwrap(); + if !config.valid_name(old_name) { + // original name must have been munged, so restore original + name = old_name.to_string(); + } else { + // original name wasn't munged, keep the current name + // in case it was renamed + name = child_name.to_string(); + } + } + Ok(_) | Err(_) => { + // use current name because either --no-xattr is set, + // xattr is None, or getting xattr on file (like ._ files) errors + name = child_name.to_string(); + } + } + self.depth += 1; + let value = self.pack(child.clone(), &config)?; + self.depth -= 1; + if let Some(value) = value { + entries.insert(name, value); + } + } + + Ok(Some(V::from_named_dir(entries, &config))) + } + "list" => { + let mut numbers_filenames_paths = fs::read_dir(path.clone())? + .map(|res| res.map(|e| e.path())) + .map(|p| { + ( + p.as_ref() + .unwrap() + .file_name() + .unwrap() + .to_str() + .unwrap() + .to_owned(), + p.unwrap(), + ) + }) + .map(|(filename, p)| { + // store a triple (integer, file basename, full pathbuf) + // full pathbuf must be retained for symlink support. + ( + match self.regex.find(&filename) { + Some(m) => filename[m.range()].parse::().unwrap(), + // use max i32 to give a default functionality for directories + // that are forced into being lists, which doesn't guarantee + // that filenames start with integers. + None => i32::MAX, + }, + // filenames in a directory are guaranteed to be different, so it + // probably is the case that the PathBuf is never compared. Also, + // filename is much shorter than the entire path, so that also saves + // time. + filename, + p, + ) + }) + .collect::>(); + numbers_filenames_paths.sort(); + + info!("parsed numbers and filenames {:?}", numbers_filenames_paths); + + let mut entries = Vec::with_capacity(numbers_filenames_paths.len()); + for (_, filename, child) in numbers_filenames_paths { + if config.ignored_file(&filename) { + warn!("skipping ignored file {:?}", child); + continue; + } + self.depth += 1; + let value = self.pack(child, &config)?; + self.depth -= 1; + if let Some(value) = value { + entries.push(value); + } + } + + Ok(Some(V::from_list_dir(entries, &config))) + } + typ => { + if let Ok(t) = Typ::from_str(typ) { + let file = fs::File::open(&path).unwrap(); + let mut reader = BufReader::new(&file); + let mut contents: Vec = Vec::new(); + reader.read_to_end(&mut contents).unwrap(); + match String::from_utf8(contents.clone()) { + Ok(mut contents) if t != Typ::Bytes => { + if config.add_newlines && contents.ends_with('\n') { + contents.truncate(contents.len() - 1); + } + Ok(Some(V::from_string(t, contents, &config))) + } + Ok(_) | Err(_) => Ok(Some(V::from_bytes(contents, &config))), + } + } else { + error!( + "This error should never be called. Received undetected and unknown type '{}' for file '{}'", + typ, + path.display() + ); + std::process::exit(ERROR_STATUS_FUSE); + } + } + } + } +} + +fn main() -> std::io::Result<()> { + let config = Config::from_pack_args(); + debug!("received config: {:?}", config); + + let mount = match &config.mount { + Some(mount) => mount, + None => { + error!("Cannot pack unspecified directory."); + std::process::exit(ERROR_STATUS_CLI); + } + }; + + let folder = PathBuf::from(mount); + + let writer = match config.output_writer() { + Some(writer) => writer, + None => return Ok(()), + }; + + let mut packer: Pack = Pack::new(); + + match &config.output_format { + Format::Json => { + let v: JsonValue = time_ns!( + "saving", + packer.pack(folder, &config)?.unwrap(), + config.timing + ); + + time_ns!("writing", v.to_writer(writer, config.pretty), config.timing); + } + Format::Toml => { + let v: TomlValue = time_ns!( + "saving", + packer.pack(folder, &config)?.unwrap(), + config.timing + ); + + time_ns!("writing", v.to_writer(writer, config.pretty), config.timing); + } + Format::Yaml => { + let v: YamlValue = time_ns!( + "saving", + packer.pack(folder, &config)?.unwrap(), + config.timing + ); + + time_ns!("writing", v.to_writer(writer, config.pretty), config.timing); + } + } + + Ok(()) +} diff --git a/src/bin/unpack.rs b/src/bin/unpack.rs new file mode 100644 index 0000000..5dcd137 --- /dev/null +++ b/src/bin/unpack.rs @@ -0,0 +1,188 @@ +use fuser::FileType; +use tracing::{debug, error, info, warn}; + +use std::collections::VecDeque; +use std::fs; +use std::io::Write; +use std::path::PathBuf; + +use ffs::config::Config; +use ffs::config::{ERROR_STATUS_CLI, ERROR_STATUS_FUSE}; +use ffs::format; +use format::json::Value as JsonValue; +use format::toml::Value as TomlValue; +use format::yaml::Value as YamlValue; +use format::{Format, Nodelike, Typ}; + +use ::xattr; + +fn unpack(root: V, root_path: PathBuf, config: &Config) -> std::io::Result<()> +where + V: Nodelike + std::fmt::Display + Default, +{ + let mut queue: VecDeque<(V, PathBuf, Option)> = VecDeque::new(); + queue.push_back((root, root_path.clone(), None)); + + while let Some((v, path, original_name)) = queue.pop_front() { + match v.node(config) { + format::Node::String(t, s) => { + // make a regular file at `path` + let mut f = fs::OpenOptions::new() + .write(true) + .create_new(true) // TODO(mmg) 2023-03-06 allow truncation? + .open(&path)?; + + // write `s` into that file + f.write(s.as_bytes())?; + + // set metadata according to `t` + if config.allow_xattr { + xattr::set(&path, "user.type", format!("{}", t).as_bytes())?; + } + } + format::Node::Bytes(b) => { + // make a regular file at `path` + let mut f = fs::OpenOptions::new() + .write(true) + .create_new(true) // TODO(mmg) 2023-03-06 allow truncation? + .open(&path)?; + + // write `b` into that file + f.write_all(b.as_slice())?; + + // set metadata to bytes + if config.allow_xattr { + xattr::set(&path, "user.type", format!("{}", Typ::Bytes).as_bytes())?; + } + } + format::Node::List(vs) => { + // if not root path, make directory + if path != root_path.clone() { + fs::create_dir(&path)?; + } + if config.allow_xattr { + xattr::set(&path, "user.type", "list".as_bytes())?; + } + + // enqueue children with appropriate names + let num_elts = vs.len() as f64; + let width = num_elts.log10().ceil() as usize; + + for (i, child) in vs.into_iter().enumerate() { + // TODO(mmg) 2021-06-08 ability to add prefixes + let name = if config.pad_element_names { + format!("{:0width$}", i, width = width) + } else { + format!("{}", i) + }; + let child_path = path.join(name); + + queue.push_back((child, child_path, None)); + } + } + format::Node::Map(fvs) => { + // if not root path, make directory + if path != root_path.clone() { + fs::create_dir(&path)?; + } + if config.allow_xattr { + xattr::set(&path, "user.type", "named".as_bytes())?; + } + + // enqueue children with appropriate names + let mut child_names = std::collections::HashSet::new(); + for (field, child) in fvs.into_iter() { + let original = field.clone(); + + // munge name to be valid and unique + let name = if !config.valid_name(&original) { + match config.munge { + ffs::config::Munge::Rename => { + let mut nfield = config.normalize_name(field); + + while child_names.contains(&nfield) { + nfield.push('_'); + } + + nfield + } + ffs::config::Munge::Filter => { + // TODO(mmg) 2023-03-06 support logging + warn!("skipping '{}'", field); + continue; + } + } + } else { + field + }; + child_names.insert(name.clone()); + + let child_path = path.join(name); + queue.push_back((child, child_path, Some(original))); + } + } + } + + if let Some(original_name) = original_name { + if config.allow_xattr { + xattr::set(&path, "user.original_name", original_name.as_bytes())?; + } + } + } + + Ok(()) +} + +fn main() -> std::io::Result<()> { + let config = Config::from_unpack_args(); + debug!("received config: {:?}", config); + + let mount = match &config.mount { + Some(mount) => mount.clone(), + None => { + error!("Directory not specified"); + std::process::exit(ERROR_STATUS_CLI); + } + }; + info!("mount: {:?}", mount); + + let reader = match config.input_reader() { + Some(reader) => reader, + None => { + error!("Input not specified"); + std::process::exit(ERROR_STATUS_CLI); + } + }; + + let result = match &config.input_format { + Format::Json => { + let value = JsonValue::from_reader(reader); + if value.kind() == FileType::Directory { + unpack(value, mount.clone(), &config) + } else { + error!("The root of the unpacked form must be a directory, but '{}' only unpacks into a single file.", mount.display()); + std::process::exit(ERROR_STATUS_FUSE); + } + } + Format::Toml => { + let value = TomlValue::from_reader(reader); + if value.kind() == FileType::Directory { + unpack(value, mount.clone(), &config) + } else { + error!("The root of the unpacked form must be a directory, but '{}' only unpacks into a single file.", mount.display()); + std::process::exit(ERROR_STATUS_FUSE); + } + } + Format::Yaml => { + let value = YamlValue::from_reader(reader); + if value.kind() == FileType::Directory { + unpack(value, mount.clone(), &config) + } else { + error!("The root of the unpacked form must be a directory, but '{}' only unpacks into a single file.", mount.display()); + std::process::exit(ERROR_STATUS_FUSE); + } + } + }; + + result +} diff --git a/src/cli.rs b/src/cli.rs index 30b5ddf..afe909e 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -6,14 +6,14 @@ pub const POSSIBLE_FORMATS: &[&str] = &["json", "toml", "yaml"]; /// The possible name munging policies. pub const MUNGE_POLICIES: &[&str] = &["filter", "rename"]; -pub fn app() -> App<'static, 'static> { +pub fn ffs() -> App<'static, 'static> { App::new("ffs") .version(env!("CARGO_PKG_VERSION")) .author(env!("CARGO_PKG_AUTHORS")) .about("file fileystem") .arg( Arg::with_name("SHELL") - .help("Generate shell completions (and exits)") + .help("Generate shell completions (and exit)") .long("completions") .takes_value(true) .possible_values(&["bash", "fish", "zsh"]) @@ -120,7 +120,7 @@ pub fn app() -> App<'static, 'static> { .help("Writes the output back over the input file") .long("in-place") .short("i") - .overrides_with("OUTPUT") + .overrides_with("OUTPUT") .overrides_with("NOOUTPUT") ) .arg( @@ -169,3 +169,190 @@ pub fn app() -> App<'static, 'static> { .index(1), ) } + +pub fn unpack() -> App<'static, 'static> { + App::new("unpack") + .version(env!("CARGO_PKG_VERSION")) + .author(env!("CARGO_PKG_AUTHORS")) + .about("unpack structured data into a directory") + .arg( + Arg::with_name("SHELL") + .help("Generate shell completions (and exit)") + .long("completions") + .takes_value(true) + .possible_values(&["bash", "fish", "zsh"]) + ) + .arg( + Arg::with_name("QUIET") + .help("Quiet mode (turns off all errors and warnings, enables `--no-output`)") + .long("quiet") + .short("q") + .overrides_with("DEBUG") + ) + .arg( + Arg::with_name("TIMING") + .help("Emit timing information on stderr in an 'event,time' format; time is in nanoseconds") + .long("time") + ) + .arg( + Arg::with_name("DEBUG") + .help("Give debug output on stderr") + .long("debug") + .short("d") + ) + .arg( + Arg::with_name("EXACT") + .help("Don't add newlines to the end of values that don't already have them (or strip them when loading)") + .long("exact") + ) + .arg( + Arg::with_name("NOXATTR") + .help("Don't use extended attributes to track metadata (see `man xattr`)") + .long("no-xattr") + ) + .arg( + Arg::with_name("MUNGE") + .help("Set the name munging policy; applies to '.', '..', and files with NUL and '/' in them") + .long("munge") + .takes_value(true) + .default_value("rename") + .possible_values(MUNGE_POLICIES) + ) + .arg( + Arg::with_name("UNPADDED") + .help("Don't pad the numeric names of list elements with zeroes; will not sort properly") + .long("unpadded") + ) + .arg( + Arg::with_name("TYPE") + .help("Specify the format type explicitly (by default, automatically inferred from filename extension)") + .long("type") + .short("t") + .takes_value(true) + .possible_values(POSSIBLE_FORMATS) + ) + .arg( + Arg::with_name("INTO") + .help("Sets the directory in which to unpack the file; will be inferred when using a file, but must be specified when running on stdin") + .long("into") + .short("i") + .takes_value(true) + ) + .arg( + Arg::with_name("INPUT") + .help("Sets the input file ('-' means STDIN)") + .default_value("-") + .index(1), + ) +} + +pub fn pack() -> App<'static, 'static> { + App::new("pack") + .version(env!("CARGO_PKG_VERSION")) + .author(env!("CARGO_PKG_AUTHORS")) + .about("pack directory") + .arg( + Arg::with_name("SHELL") + .help("Generate shell completions (and exit)") + .long("completions") + .takes_value(true) + .possible_values(&["bash", "fish", "zsh"]) + ) + .arg( + Arg::with_name("QUIET") + .help("Quiet mode (turns off all errors and warnings, enables `--no-output`)") + .long("quiet") + .short("q") + .overrides_with("DEBUG") + ) + .arg( + Arg::with_name("TIMING") + .help("Emit timing information on stderr in an 'event,time' format; time is in nanoseconds") + .long("time") + ) + .arg( + Arg::with_name("DEBUG") + .help("Give debug output on stderr") + .long("debug") + .short("d") + ) + .arg( + Arg::with_name("EXACT") + .help("Don't add newlines to the end of values that don't already have them (or strip them when loading)") + .long("exact") + ) + .arg( + Arg::with_name("NOFOLLOW_SYMLINKS") + .help("Never follow symbolic links. This is the default behaviour. `pack` will ignore all symbolic links.") + .short("P") + .overrides_with("FOLLOW_SYMLINKS") + ) + .arg( + Arg::with_name("FOLLOW_SYMLINKS") + .help("Follow all symlinks. For safety, you can also specify a --max-depth value.") + .short("L") + .overrides_with("NOFOLLOW_SYMLINKS") + ) + .arg( + Arg::with_name("MAXDEPTH") + .help("Maximum depth of filesystem traversal allowed for `pack`") + .long("max-depth") + .takes_value(true) + ) + .arg( + Arg::with_name("ALLOW_SYMLINK_ESCAPE") + .help("Allows pack to follow symlinks outside of the directory being packed.") + .long("allow-symlink-escape") + ) + .arg( + Arg::with_name("NOXATTR") + .help("Don't use extended attributes to track metadata (see `man xattr`)") + .long("no-xattr") + ) + .arg( + Arg::with_name("KEEPMACOSDOT") + .help("Include ._* extended attribute/resource fork files on macOS") + .long("keep-macos-xattr") + ) + .arg( + Arg::with_name("MUNGE") + .help("Set the name munging policy; applies to '.', '..', and files with NUL and '/' in them") + .long("munge") + .takes_value(true) + .default_value("rename") + .possible_values(MUNGE_POLICIES) + ) + .arg( + Arg::with_name("OUTPUT") + .help("Sets the output file for saving changes (defaults to stdout)") + .long("output") + .short("o") + .takes_value(true) + ) + .arg( + Arg::with_name("NOOUTPUT") + .help("Disables output of filesystem (normally on stdout)") + .long("no-output") + .overrides_with("OUTPUT") + ) + .arg( + Arg::with_name("TARGET_FORMAT") + .help("Specify the target format explicitly (by default, automatically inferred from filename extension)") + .long("target") + .short("t") + .takes_value(true) + .possible_values(POSSIBLE_FORMATS) + ) + .arg( + Arg::with_name("PRETTY") + .help("Pretty-print output (may increase size)") + .long("pretty") + .overrides_with("NOOUTPUT") + .overrides_with("QUIET") + ) + .arg( + Arg::with_name("INPUT") + .help("Sets the input folder") + .index(1), + ) +} diff --git a/src/config.rs b/src/config.rs index f3339ff..b54087b 100644 --- a/src/config.rs +++ b/src/config.rs @@ -2,6 +2,8 @@ use std::fs::File; use std::path::{Path, PathBuf}; use std::str::FromStr; +// use path_absolutize::*; + use tracing::{debug, error, warn}; use tracing_subscriber::prelude::*; use tracing_subscriber::{filter::EnvFilter, fmt}; @@ -38,6 +40,9 @@ pub struct Config { pub try_decode_base64: bool, pub allow_xattr: bool, pub keep_macos_xattr_file: bool, + pub symlink: Symlink, + pub max_depth: Option, + pub allow_symlink_escape: bool, pub munge: Munge, pub read_only: bool, pub input: Input, @@ -103,10 +108,16 @@ impl FromStr for Munge { } } +#[derive(Debug)] +pub enum Symlink { + NoFollow, + Follow, +} + impl Config { /// Parses arguments from `std::env::Args`, via `cli::app().get_matches()` - pub fn from_args() -> Self { - let args = cli::app().get_matches_safe().unwrap_or_else(|e| { + pub fn from_ffs_args() -> Self { + let args = cli::ffs().get_matches_safe().unwrap_or_else(|e| { eprintln!("{}", e.message); std::process::exit(ERROR_STATUS_CLI) }); @@ -126,7 +137,7 @@ impl Config { eprintln!("Can't generate completions for '{}'.", shell); std::process::exit(ERROR_STATUS_CLI); }; - cli::app().gen_completions_to("ffs", shell, &mut std::io::stdout()); + cli::ffs().gen_completions_to("ffs", shell, &mut std::io::stdout()); std::process::exit(0); } @@ -280,9 +291,9 @@ impl Config { Ok(format) => format, Err(_) => { error!( - "Unrecognized format '{}'; use --target or a known extension to specify a format.", - output.display() - ); + "Unrecognized format '{}'; use --target or a known extension to specify a format.", + output.display() + ); std::process::exit(ERROR_STATUS_CLI); } } @@ -416,10 +427,10 @@ impl Config { // If the mountpoint can't be created, give up and tell the user about --mount. if let Err(e) = std::fs::create_dir(&mount_dir) { error!( - "Couldn't create mountpoint '{}': {}. Use `--mount MOUNT` to specify a mountpoint.", - mount_dir.display(), - e - ); + "Couldn't create mountpoint '{}': {}. Use `--mount MOUNT` to specify a mountpoint.", + mount_dir.display(), + e + ); std::process::exit(ERROR_STATUS_FUSE); } // We did it! @@ -532,6 +543,369 @@ impl Config { config } + pub fn from_unpack_args() -> Self { + let args = cli::unpack().get_matches_safe().unwrap_or_else(|e| { + eprintln!("{}", e.message); + std::process::exit(ERROR_STATUS_CLI) + }); + + let mut config = Config::default(); + // generate completions? + // + // TODO 2021-07-06 good candidate for a subcommand + if let Some(shell) = args.value_of("SHELL") { + let shell = if shell == "bash" { + clap::Shell::Bash + } else if shell == "fish" { + clap::Shell::Fish + } else if shell == "zsh" { + clap::Shell::Zsh + } else { + eprintln!("Can't generate completions for '{}'.", shell); + std::process::exit(ERROR_STATUS_CLI); + }; + cli::unpack().gen_completions_to("unpack", shell, &mut std::io::stdout()); + std::process::exit(0); + } + + // logging + if !args.is_present("QUIET") { + let filter_layer = EnvFilter::try_from_default_env() + .unwrap_or_else(|_e| { + if args.is_present("DEBUG") { + EnvFilter::new("unpack=debug") + } else { + EnvFilter::new("unpack=warn") + } + }) + .add_directive("ffs::config=warn".parse().unwrap()); + let fmt_layer = fmt::layer().with_writer(std::io::stderr); + tracing_subscriber::registry() + .with(filter_layer) + .with(fmt_layer) + .init(); + } + + // simple flags + config.timing = args.is_present("TIMING"); + config.add_newlines = !args.is_present("EXACT"); + config.pad_element_names = !args.is_present("UNPADDED"); + config.allow_xattr = !args.is_present("NOXATTR"); + + // munging policy + config.munge = match args.value_of("MUNGE") { + None => Munge::Filter, + Some(s) => match str::parse(s) { + Ok(munge) => munge, + Err(_) => { + warn!("Invalid `--munge` mode '{}', using 'rename'.", s); + Munge::Rename + } + }, + }; + + // configure input + config.input = match args.value_of("INPUT") { + Some(input_source) => { + if input_source == "-" { + Input::Stdin + } else { + let input_source = PathBuf::from(input_source); + if !input_source.exists() { + error!("Input file {} does not exist.", input_source.display()); + std::process::exit(ERROR_STATUS_FUSE); + } + Input::File(input_source) + } + } + None => Input::Stdin, + }; + + // infer and create mountpoint from filename as possible + config.mount = match args.value_of("INTO") { + Some(mount_point) => { + match std::fs::create_dir(&mount_point) { + Ok(_) => Some(PathBuf::from(mount_point)), + Err(_) => { + // if dir is empty then we can use it + let mount = PathBuf::from(mount_point); + if mount.read_dir().unwrap().next().is_none() { + // dir exists and is empty + Some(PathBuf::from(mount_point)) + } else { + // dir exists but is not empty + error!( + "Directory `{}` already exists and is not empty.", + mount_point + ); + std::process::exit(ERROR_STATUS_FUSE); + } + } + } + } + None => { + match &config.input { + Input::Stdin => { + error!("You must specify a mount point when reading from stdin."); + std::process::exit(ERROR_STATUS_CLI); + } + Input::Empty => { + error!("--new is not an option for `unpack`, so the input should never be Empty and this error should never be seen."); + std::process::exit(ERROR_STATUS_CLI); + } + Input::File(file) => { + // If the input is from a file foo.EXT, then try to make a directory foo. + let stem = file.file_stem().unwrap_or_else(|| { + error!("Couldn't infer the directory to unpack into from input '{}'. Use `--into DIRECTORY` to specify a directory.", file.display()); + std::process::exit(ERROR_STATUS_FUSE); + }); + let mount_dir = PathBuf::from(stem); + debug!("inferred mount_dir {}", mount_dir.display()); + + // If that file already exists, give up and tell the user about --mount. + if mount_dir.exists() { + error!("Inferred directory '{mount}' for input file '{file}', but '{mount}' already exists. Use `--into DIRECTORY` to specify a directory.", + mount = mount_dir.display(), file = file.display()); + std::process::exit(ERROR_STATUS_FUSE); + } + // If the mountpoint can't be created, give up and tell the user about --mount. + if let Err(e) = std::fs::create_dir(&mount_dir) { + error!( + "Couldn't create directory '{}': {}. Use `--into DIRECTORY` to specify a directory.", + mount_dir.display(), + e + ); + std::process::exit(ERROR_STATUS_FUSE); + } + + // We did it! + Some(mount_dir) + } + } + } + }; + assert!(config.mount.is_some()); + + // try to autodetect the input format. + // + // first see if it's specified and parses okay. + // + // then see if we can pull it out of the extension. + // + // then give up and use json + config.input_format = match args + .value_of("TYPE") + .ok_or(format::ParseFormatError::NoFormatProvided) + .and_then(|s| s.parse::()) + { + Ok(source_format) => source_format, + Err(e) => { + match e { + format::ParseFormatError::NoSuchFormat(s) => { + warn!("Unrecognized format '{}', inferring from input.", s) + } + format::ParseFormatError::NoFormatProvided => { + debug!("Inferring format from input.") + } + }; + match &config.input { + Input::Stdin => Format::Json, + Input::Empty => Format::Json, + Input::File(input_source) => match input_source + .extension() + .and_then(|s| s.to_str()) + .ok_or(format::ParseFormatError::NoFormatProvided) + .and_then(|s| s.parse::()) + { + Ok(format) => format, + Err(_) => { + warn!( + "Unrecognized format {}, defaulting to JSON.", + input_source.display() + ); + Format::Json + } + }, + } + } + }; + + config + } + + pub fn from_pack_args() -> Self { + let args = cli::pack().get_matches_safe().unwrap_or_else(|e| { + eprintln!("{}", e.message); + std::process::exit(ERROR_STATUS_CLI) + }); + + let mut config = Config::default(); + // generate completions? + // + // TODO 2021-07-06 good candidate for a subcommand + if let Some(shell) = args.value_of("SHELL") { + let shell = if shell == "bash" { + clap::Shell::Bash + } else if shell == "fish" { + clap::Shell::Fish + } else if shell == "zsh" { + clap::Shell::Zsh + } else { + eprintln!("Can't generate completions for '{}'.", shell); + std::process::exit(ERROR_STATUS_CLI); + }; + cli::pack().gen_completions_to("pack", shell, &mut std::io::stdout()); + std::process::exit(0); + } + + // logging + if !args.is_present("QUIET") { + let filter_layer = EnvFilter::try_from_default_env() + .unwrap_or_else(|_e| { + if args.is_present("DEBUG") { + EnvFilter::new("pack=debug") + } else { + EnvFilter::new("pack=warn") + } + }) + .add_directive("ffs::config=warn".parse().unwrap()); + let fmt_layer = fmt::layer().with_writer(std::io::stderr); + tracing_subscriber::registry() + .with(filter_layer) + .with(fmt_layer) + .init(); + } + + // simple flags + config.timing = args.is_present("TIMING"); + config.add_newlines = !args.is_present("EXACT"); + config.read_only = args.is_present("READONLY"); + config.allow_xattr = !args.is_present("NOXATTR"); + config.allow_symlink_escape = args.is_present("ALLOW_SYMLINK_ESCAPE"); + config.keep_macos_xattr_file = args.is_present("KEEPMACOSDOT"); + config.pretty = args.is_present("PRETTY"); + + config.symlink = if args.is_present("FOLLOW_SYMLINKS") { + Symlink::Follow + } else { + Symlink::NoFollow + }; + + config.max_depth = match args.value_of("MAXDEPTH") { + Some(s) => match str::parse(s) { + Ok(depth) => Some(depth), + Err(_) => { + error!( + "Invalid `--max-depth` '{}', must be a non-negative integer.", + s + ); + std::process::exit(ERROR_STATUS_CLI); + } + }, + None => None, + }; + + // munging policy + config.munge = match args.value_of("MUNGE") { + None => Munge::Filter, + Some(s) => match str::parse(s) { + Ok(munge) => munge, + Err(_) => { + warn!("Invalid `--munge` mode '{}', using 'rename'.", s); + Munge::Rename + } + }, + }; + + // configure input + config.input = match args.value_of("INPUT") { + Some(input_source) => { + let input_source = PathBuf::from(input_source); + if !input_source.exists() { + error!("Input file {} does not exist.", input_source.display()); + std::process::exit(ERROR_STATUS_FUSE); + } + Input::File(input_source) + } + None => { + error!("The directory to pack must be specified."); + std::process::exit(ERROR_STATUS_CLI); + } + }; + + // set the mount from the input directory + config.mount = match &config.input { + Input::File(file) => Some(file.clone().canonicalize().unwrap()), + _ => { + error!("Input must be a file or directory."); + std::process::exit(ERROR_STATUS_CLI); + } + }; + + // configure output + config.output = if let Some(output) = args.value_of("OUTPUT") { + Output::File(PathBuf::from(output)) + } else if args.is_present("NOOUTPUT") || args.is_present("QUIET") { + Output::Quiet + } else { + Output::Stdout + }; + + // try to autodetect the output format. + // + // first see if it's specified and parses okay. + // + // then see if we can pull it out of the extension (if specified) + // + // then give up and use the input format + config.output_format = match args + .value_of("TARGET_FORMAT") + .ok_or(format::ParseFormatError::NoFormatProvided) + .and_then(|s| s.parse::()) + { + Ok(target_format) => target_format, + Err(e) => { + match e { + format::ParseFormatError::NoSuchFormat(s) => { + warn!( + "Unrecognized format '{}', inferring from input and output.", + s + ) + } + format::ParseFormatError::NoFormatProvided => { + debug!("Inferring output format from input.") + } + }; + match args + .value_of("OUTPUT") + .and_then(|s| Path::new(s).extension()) + .and_then(|s| s.to_str()) + { + Some(s) => match s.parse::() { + Ok(format) => format, + Err(_) => { + warn!( + "Unrecognized format {}, defaulting to input format '{}'.", + s, config.input_format + ); + config.input_format + } + }, + None => config.input_format, + } + } + }; + + if config.pretty && !config.output_format.can_be_pretty() { + warn!( + "There is no pretty printing routine for {}.", + config.output_format + ) + } + + config + } + pub fn valid_name(&self, s: &str) -> bool { s != "." && s != ".." && !s.contains('\0') && !s.contains('/') } @@ -630,6 +1004,9 @@ impl Default for Config { try_decode_base64: false, allow_xattr: true, keep_macos_xattr_file: false, + symlink: Symlink::NoFollow, + max_depth: None, + allow_symlink_escape: false, munge: Munge::Rename, read_only: false, input: Input::Stdin, diff --git a/src/format.rs b/src/format.rs index 24458a6..430d5d3 100644 --- a/src/format.rs +++ b/src/format.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::BTreeMap; use std::str::FromStr; use tracing::debug; @@ -187,7 +187,7 @@ where /// Should never be called when `typ == Typ::Bytes`. fn from_string(typ: Typ, v: String, config: &Config) -> Self; fn from_list_dir(files: Vec, config: &Config) -> Self; - fn from_named_dir(files: HashMap, config: &Config) -> Self; + fn from_named_dir(files: BTreeMap, config: &Config) -> Self; /// Loading fn from_reader(reader: Box) -> Self; @@ -307,7 +307,7 @@ pub mod json { Value::Array(files) } - fn from_named_dir(files: HashMap, _config: &Config) -> Self { + fn from_named_dir(files: BTreeMap, _config: &Config) -> Self { Value::Object(files.into_iter().collect()) } @@ -472,7 +472,7 @@ pub mod toml { Value(Toml::Array(files.into_iter().map(|v| v.0).collect())) } - fn from_named_dir(files: HashMap, _config: &Config) -> Self { + fn from_named_dir(files: BTreeMap, _config: &Config) -> Self { Value(Toml::Table( files.into_iter().map(|(f, v)| (f, v.0)).collect(), )) @@ -668,7 +668,7 @@ pub mod yaml { Value(Yaml::Array(vs.into_iter().map(|v| v.0).collect())) } - fn from_named_dir(fvs: HashMap, config: &Config) -> Self { + fn from_named_dir(fvs: BTreeMap, config: &Config) -> Self { Value(Yaml::Hash( fvs.into_iter() .map(|(k, v)| (Value::from_string(Typ::String, k, config).0, v.0)) diff --git a/src/fs.rs b/src/fs.rs index fba530f..675b240 100644 --- a/src/fs.rs +++ b/src/fs.rs @@ -1,5 +1,5 @@ use std::cell::Cell; -use std::collections::HashMap; +use std::collections::BTreeMap; use std::ffi::OsStr; use std::fmt::{Debug, Display}; use std::mem; @@ -85,7 +85,7 @@ pub enum Entry { // TODO 2021-06-14 need a 'written' flag to determine whether or not to // strip newlines during writeback File(Typ, Vec), - Directory(DirType, HashMap), + Directory(DirType, BTreeMap), Lazy(V), } @@ -159,8 +159,7 @@ where Node::Bytes(b) => (Entry::File(Typ::Bytes, b), Option::None), Node::String(t, s) => (Entry::File(t, s.into_bytes()), Option::None), Node::List(vs) => { - let mut children = HashMap::new(); - children.reserve(vs.len()); + let mut children = BTreeMap::new(); let num_elts = vs.len() as f64; let width = num_elts.log10().ceil() as usize; @@ -199,9 +198,7 @@ where ) } Node::Map(fvs) => { - let mut children = HashMap::new(); - children.reserve(fvs.len()); - + let mut children = BTreeMap::new(); let mut new_nodes = Vec::with_capacity(fvs.len()); for (field, child) in fvs.into_iter() { let original = field.clone(); @@ -338,7 +335,7 @@ where Some(reader) => reader, None => { // create an empty directory - let contents = HashMap::with_capacity(16); + let contents = BTreeMap::new(); inodes[1] = Some(Inode::new( fuser::FUSE_ROOT_ID, fuser::FUSE_ROOT_ID, @@ -525,7 +522,7 @@ where V::from_list_dir(entries, &self.config) } Entry::Directory(DirType::Named, files) => { - let mut entries = HashMap::with_capacity(files.len()); + let mut entries = BTreeMap::new(); for ( name, DirEntry { @@ -589,7 +586,7 @@ where U::from_list_dir(entries, &self.config) } Entry::Directory(DirType::Named, files) => { - let mut entries = HashMap::with_capacity(files.len()); + let mut entries = BTreeMap::new(); let files = files .iter() @@ -1386,7 +1383,7 @@ where } else { assert_eq!(file_type, libc::S_IFDIR as u32); ( - Entry::Directory(DirType::Named, HashMap::new()), + Entry::Directory(DirType::Named, BTreeMap::new()), FileType::Directory, ) }; @@ -1466,7 +1463,7 @@ where }; // create the inode entry - let entry = Entry::Directory(DirType::Named, HashMap::new()); + let entry = Entry::Directory(DirType::Named, BTreeMap::new()); let kind = FileType::Directory; // allocate the inode (sets dirty bit) diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..a4aa504 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,4 @@ +pub mod cli; +pub mod config; +pub mod format; +pub mod fs; diff --git a/tests/basic_object_exact.sh b/tests/basic_object_exact.sh index da0b297..e5db8c5 100755 --- a/tests/basic_object_exact.sh +++ b/tests/basic_object_exact.sh @@ -33,7 +33,7 @@ esac diff "${EXP}/name" "${MNT}/name" || fail name diff "${EXP}/eyes" "${MNT}/eyes" || fail eyes diff "${EXP}/fingernails" "${MNT}/fingernails" || fail fingernails -diff "${EXP}/human" "${MNT}/human" || fail huma +diff "${EXP}/human" "${MNT}/human" || fail human diff "${EXP}/problems" "${MNT}/problems" || fail problems cd - >/dev/null 2>&1 diff --git a/tests/pad_list.sh b/tests/pad_list.sh index 37f71ca..0f3871b 100755 --- a/tests/pad_list.sh +++ b/tests/pad_list.sh @@ -22,16 +22,16 @@ case $(ls) in (*) fail ls;; esac [ "$(cat 00)" -eq 0 ] || fail 0 -[ "$(cat 01)" -eq 1 ] || fail 1 -[ "$(cat 02)" -eq 2 ] || fail 2 -[ "$(cat 03)" -eq 3 ] || fail 3 -[ "$(cat 04)" -eq 4 ] || fail 4 -[ "$(cat 05)" -eq 5 ] || fail 5 -[ "$(cat 06)" -eq 6 ] || fail 6 -[ "$(cat 07)" -eq 7 ] || fail 7 -[ "$(cat 08)" -eq 8 ] || fail 8 -[ "$(cat 09)" -eq 9 ] || fail 9 -[ "$(cat 10)" -eq 10 ] || fail 10 +[ "$(cat 01)" -eq 1 ] || fail 1 +[ "$(cat 02)" -eq 2 ] || fail 2 +[ "$(cat 03)" -eq 3 ] || fail 3 +[ "$(cat 04)" -eq 4 ] || fail 4 +[ "$(cat 05)" -eq 5 ] || fail 5 +[ "$(cat 06)" -eq 6 ] || fail 6 +[ "$(cat 07)" -eq 7 ] || fail 7 +[ "$(cat 08)" -eq 8 ] || fail 8 +[ "$(cat 09)" -eq 9 ] || fail 9 +[ "$(cat 10)" -eq 10 ] || fail 10 cd - >/dev/null 2>&1 umount "$MNT" || fail unmount sleep 1 diff --git a/tests/unpack_pack_auto.sh b/tests/unpack_pack_auto.sh new file mode 100755 index 0000000..399b36f --- /dev/null +++ b/tests/unpack_pack_auto.sh @@ -0,0 +1,42 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm "$FILE" "$EXP" + rm -r "$MNT" + fi + exit 1 +} + +MNT=$(mktemp -d) +FILE=$(mktemp).json + +echo '{}' >"$FILE" + +EXP=$(mktemp) + +printf '{"favorite_number":47,"likes":{"cats":false,"dogs":true},"mistakes":null,"name":"Michael Greenberg","website":"https://mgree.github.io"}' >"$EXP" + +unpack "$FILE" --into "$MNT" || fail unpack + +ls "$MNT" +[ $(ls $MNT) ] && fail nonempty1 +[ $(ls $MNT | wc -l) -eq 0 ] || fail nonempty2 + +echo 47 >"$MNT"/favorite_number +mkdir "$MNT"/likes +echo true >"$MNT"/likes/dogs +echo false >"$MNT"/likes/cats +touch "$MNT"/mistakes +echo Michael Greenberg >"$MNT"/name +echo https://mgree.github.io >"$MNT"/website + +pack "$MNT" -o "$FILE" || fail pack + +cat "$FILE" +diff "$FILE" "$EXP" || fail diff + +rm "$FILE" "$EXP" +rm -r "$MNT" diff --git a/tests/unpack_pack_bad_root.sh b/tests/unpack_pack_bad_root.sh new file mode 100755 index 0000000..9645c99 --- /dev/null +++ b/tests/unpack_pack_bad_root.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + rm "$MSG" "$OUT" + fi + exit 1 +} + +MNT=$(mktemp -d) +OUT=$(mktemp) +MSG=$(mktemp) + +unpack --into "$MNT" ../json/null.json >"$OUT" 2>"$MSG" + +cat "$MSG" | grep -i -e "must be a directory" >/dev/null 2>&1 || fail error +[ -f "$OUT" ] && ! [ -s "$OUT" ] || fail output + +pack "$MNT" || fail pack + +rm -r "$MNT" || fail mount +rm "$MSG" "$OUT" diff --git a/tests/unpack_pack_bad_root_stdin.sh b/tests/unpack_pack_bad_root_stdin.sh new file mode 100755 index 0000000..e52a1ec --- /dev/null +++ b/tests/unpack_pack_bad_root_stdin.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + rm "$MSG" "$OUT" + fi + exit 1 +} + +MNT=$(mktemp -d) +OUT=$(mktemp) +MSG=$(mktemp) + +echo \"just a string\" | unpack --into "$MNT" >"$OUT" 2>"$MSG" + +cat "$MSG" | grep -i -e "must be a directory" >/dev/null 2>&1 || fail error +[ -f "$OUT" ] && ! [ -s "$OUT" ] || fail output + +rm -r "$MNT" || fail mount +rm "$MSG" "$OUT" diff --git a/tests/unpack_pack_basic_list.sh b/tests/unpack_pack_basic_list.sh new file mode 100755 index 0000000..dc7dd4c --- /dev/null +++ b/tests/unpack_pack_basic_list.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + fi + exit 1 +} + +MNT=$(mktemp -d) + +unpack --into "$MNT" ../json/list.json || fail unpack + +cd "$MNT" +case $(ls) in + (0*1*2*3) ;; + (*) fail ls;; +esac +[ "$(cat 0)" -eq 1 ] || fail 0 +[ "$(cat 1)" -eq 2 ] || fail 1 +[ "$(cat 2)" = "3" ] || fail 2 +[ "$(cat 3)" = "false" ] || fail 3 +cd - >/dev/null 2>&1 + +pack "$MNT" || fail pack +rm -r "$MNT" || fail mount diff --git a/tests/unpack_pack_basic_object.sh b/tests/unpack_pack_basic_object.sh new file mode 100755 index 0000000..c6a3d0e --- /dev/null +++ b/tests/unpack_pack_basic_object.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + fi + exit 1 +} + +MNT=$(mktemp -d) + +unpack --into "$MNT" ../json/object.json || fail unpack + +cd "$MNT" +case $(ls) in + (eyes*fingernails*human*name) ;; + (*) fail ls;; +esac +[ "$(cat name)" = "Michael Greenberg" ] || fail name +[ "$(cat eyes)" -eq 2 ] || fail eyes +[ "$(cat fingernails)" -eq 10 ] || fail fingernails +[ "$(cat human)" = "true" ] || fail human +cd - >/dev/null 2>&1 + +pack "$MNT" || fail pack +rm -r "$MNT" || fail mount diff --git a/tests/unpack_pack_basic_object_exact.sh b/tests/unpack_pack_basic_object_exact.sh new file mode 100755 index 0000000..d2a9009 --- /dev/null +++ b/tests/unpack_pack_basic_object_exact.sh @@ -0,0 +1,40 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + rm -r "$EXP" + fi + exit 1 +} + +MNT=$(mktemp -d) +EXP=$(mktemp -d) + +# generate files w/o newlines +printf "Michael Greenberg" >"${EXP}/name" +printf "2" >"${EXP}/eyes" +printf "10" >"${EXP}/fingernails" +printf "true" >"${EXP}/human" +printf "" >"${EXP}/problems" + +unpack --exact --into "$MNT" ../json/object_null.json || fail unpack + +cd "$MNT" +case $(ls) in + (eyes*fingernails*human*name*problems) ;; + (*) fail ls;; +esac +diff "${EXP}/name" "${MNT}/name" || fail name +diff "${EXP}/eyes" "${MNT}/eyes" || fail eyes +diff "${EXP}/fingernails" "${MNT}/fingernails" || fail fingernails +diff "${EXP}/human" "${MNT}/human" || fail human +diff "${EXP}/problems" "${MNT}/problems" || fail problems + +cd - >/dev/null 2>&1 + +pack "$MNT" || fail pack +rm -r "$MNT" || fail mount +rm -r "$EXP" diff --git a/tests/unpack_pack_basic_object_newline.sh b/tests/unpack_pack_basic_object_newline.sh new file mode 100755 index 0000000..6493983 --- /dev/null +++ b/tests/unpack_pack_basic_object_newline.sh @@ -0,0 +1,40 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + rm -r "$EXP" + fi + exit 1 +} + +MNT=$(mktemp -d) +EXP=$(mktemp -d) + +# generate files w/newlines +printf "Michael Greenberg\n" >"${EXP}/name" +printf "2\n" >"${EXP}/eyes" +printf "10\n" >"${EXP}/fingernails" +printf "true\n" >"${EXP}/human" +printf "" >"${EXP}/problems" + +unpack --into "$MNT" ../json/object_null.json || fail unpack + +cd "$MNT" +case $(ls) in + (eyes*fingernails*human*name*problems) ;; + (*) fail ls;; +esac +diff "${EXP}/name" "${MNT}/name" || fail name +diff "${EXP}/eyes" "${MNT}/eyes" || fail eyes +diff "${EXP}/fingernails" "${MNT}/fingernails" || fail fingernails +diff "${EXP}/human" "${MNT}/human" || fail human +diff "${EXP}/problems" "${MNT}/problems" || fail problems + +cd - >/dev/null 2>&1 + +pack "$MNT" || fail pack +rm -r "$MNT" || fail mount +rm -r "$EXP" diff --git a/tests/unpack_pack_basic_object_stdin.sh b/tests/unpack_pack_basic_object_stdin.sh new file mode 100755 index 0000000..88cffbb --- /dev/null +++ b/tests/unpack_pack_basic_object_stdin.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + fi + exit 1 +} + +MNT=$(mktemp -d) + +cat ../json/object.json | unpack --into "$MNT" || fail unpack + +cd "$MNT" +case $(ls) in + (eyes*fingernails*human*name) ;; + (*) fail ls;; +esac +[ "$(cat name)" = "Michael Greenberg" ] || fail name +[ "$(cat eyes)" -eq 2 ] || fail eyes +[ "$(cat fingernails)" -eq 10 ] || fail fingernails +[ "$(cat human)" = "true" ] || fail human +cd - >/dev/null 2>&1 + +pack "$MNT" || fail pack +rm -r "$MNT" || fail mount diff --git a/tests/unpack_pack_basic_toml.sh b/tests/unpack_pack_basic_toml.sh new file mode 100755 index 0000000..cf8c363 --- /dev/null +++ b/tests/unpack_pack_basic_toml.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + fi + exit 1 +} + +MNT=$(mktemp -d) + +unpack --into "$MNT" ../toml/eg.toml || fail unpack + +case $(ls "$MNT") in + (clients*database*owner*servers*title) ;; + (*) fail ls;; +esac +[ "$(cat $MNT/title)" = "TOML Example" ] || fail title +[ "$(cat $MNT/owner/dob)" = "1979-05-27T07:32:00-08:00" ] || fail dob + +pack "$MNT" || fail pack +rm -r "$MNT" || fail mount diff --git a/tests/unpack_pack_binary.sh b/tests/unpack_pack_binary.sh new file mode 100755 index 0000000..20fbaed --- /dev/null +++ b/tests/unpack_pack_binary.sh @@ -0,0 +1,57 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + rm "$TGT" + rm "$TGT2" + rm "$ICO" + fi + exit 1 +} + +if [ "$RUNNER_OS" = "Linux" ] || [ "$(uname)" = "Linux" ]; then + decode() { + base64 -d $1 >$2 + } +elif [ "$RUNNER_OS" = "macOS" ] || [ "$(uname)" = "Darwin" ]; then + decode() { + base64 -D -i $1 -o $2 + } +else + fail os +fi + +MNT=$(mktemp -d) +TGT=$(mktemp) +TGT2=$(mktemp) + +unpack --into "$MNT" ../json/object.json || fail unpack1 + +cp ../binary/twitter.ico "$MNT"/favicon +pack "$MNT" >"$TGT" || fail pack1 +rm -r "$MNT" + +# easiest to just test using ffs, but would be cool to get outside validation +[ -f "$TGT" ] || fail output1 +[ -s "$TGT" ] || fail output2 +grep favicon "$TGT" >/dev/null 2>&1 || fail text +unpack --into "$MNT" "$TGT" || fail unpack2 + +ICO=$(mktemp) + +ls "$MNT" | grep favicon >/dev/null 2>&1 || fail field +decode "$MNT"/favicon "$ICO" +diff ../binary/twitter.ico "$ICO" || fail diff + +pack --no-output "$MNT" >"$TGT2" || fail pack2 + +[ -f "$TGT2" ] || fail tgt2 +[ -s "$TGT2" ] && fail tgt2_nonempty + +rm -r "$MNT" || fail mount +rm "$TGT" +rm "$TGT2" +rm "$ICO" diff --git a/tests/unpack_pack_chmod.sh b/tests/unpack_pack_chmod.sh new file mode 100755 index 0000000..dc9c8fa --- /dev/null +++ b/tests/unpack_pack_chmod.sh @@ -0,0 +1,47 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + rm "$TGT" + rm "$TGT2" + fi + exit 1 +} + +MNT=$(mktemp -d) +TGT=$(mktemp) +TGT2=$(mktemp) + +unpack --into "$MNT" ../json/object.json || fail unpack1 + +echo 'echo hi' >"$MNT"/script +chmod +x "$MNT"/script +[ "$($MNT/script)" = "hi" ] || fail exec + +pack "$MNT" >"$TGT" || fail pack1 +rm -r "$MNT" + +# easiest to just test using ffs, but would be cool to get outside validation +[ -f "$TGT" ] || fail output1 +[ -s "$TGT" ] || fail output2 +grep -e echo "$TGT" >/dev/null 2>&1 || fail grep +unpack --type json --into "$MNT" "$TGT" || fail unpack2 + +case $(ls "$MNT") in + (eyes*fingernails*human*name*script) ;; + (*) fail ls;; +esac + +[ "$(cat $MNT/script)" = "echo hi" ] || fail contents + +pack --no-output >"$TGT2" "$MNT" || fail pack2 + +[ -f "$TGT2" ] || fail tgt2 +[ -s "$TGT2" ] && fail tgt2_nonempty + +rm -r "$MNT" || fail mount +rm "$TGT" +rm "$TGT2" diff --git a/tests/unpack_pack_exact_cleanup.sh b/tests/unpack_pack_exact_cleanup.sh new file mode 100755 index 0000000..3066188 --- /dev/null +++ b/tests/unpack_pack_exact_cleanup.sh @@ -0,0 +1,48 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + rm -r "$EXP" + rm "$JSON" + fi + exit 1 +} + +MNT=$(mktemp -d) +EXP=$(mktemp -d) +JSON=$(mktemp) + +# generate files w/newlines +printf "Michael Greenberg" >"${EXP}/name" +printf "2" >"${EXP}/eyes" +printf "10" >"${EXP}/fingernails" +printf "true" >"${EXP}/human" +printf "hi\n" >"${EXP}/greeting" +printf "bye" >"${EXP}/farewell" + +unpack --exact --into "$MNT" ../json/object.json || fail unpack1 + +echo hi >"$MNT"/greeting +printf "bye" >"$MNT"/farewell + +pack --exact "$MNT" -o "$JSON" || fail pack1 +rm -r "$MNT" + +# remount w/ --exact, confirm that they're not there (except for greeting) +unpack --exact --into "$MNT" "$JSON" || fail unpack2 + +case $(ls "$MNT") in + (eyes*farewell*fingernails*greeting*human*name) ;; + (*) fail ls;; +esac +for x in "$EXP"/* +do + diff "$x" "$MNT/$(basename $x)" || fail "$(basename $x)" +done + +pack "$MNT" || fail pack2 +rm -r "$MNT" || fail mount +rm -r "$EXP" diff --git a/tests/unpack_pack_exit_status.sh b/tests/unpack_pack_exit_status.sh new file mode 100755 index 0000000..80ab5d8 --- /dev/null +++ b/tests/unpack_pack_exit_status.sh @@ -0,0 +1,126 @@ +#!/bin/sh +# +# from https://github.com/mgree/ffs/issues/42 + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + umount "$D"/single + rm -r "$D" + fi + exit 1 +} + +TESTS="$(pwd)" + +D=$(mktemp -d) + +cp ../json/single.json "$D"/single.json +cp ../json/false.json "$D"/false.json + +cd "$D" + +cp single.json unreadable.json +chmod -r unreadable.json + +mkdir unwriteable +chmod -w unwriteable + +#### ERROR_STATUS_FUSE +## UNPACK +# mount exists but unempty +mkdir -p unempty/dir +unpack --into unempty single.json 2>single.err +[ $? -eq 1 ] || fail unempty_mount_status +[ -s single.err ] || fail unempty_mount_msg +rm single.err + +# inferred mount already exists, use --into +mkdir single +unpack single.json 2>single.err +[ $? -eq 1 ] || fail inferred_mount_exists_status +[ -s single.err ] || fail inferred_mount_exists_msg +rm single.err + +# mount unwriteable +cd unwriteable +unpack single.json 2>../single.err +[ $? -eq 1 ] || fail unwriteable_mount_status +cd .. +[ -s single.err ] || fail unwriteable_mount_msg +rm single.err + +# input file doesn't exist +unpack nonesuch.toml 2>nonesuch.err +[ $? -eq 1 ] || fail input_dne_status +[ -s nonesuch.err ] || fail input_dne_msg +rm nonesuch.err + +# input file unreadable +unpack unreadable.json 2>ur.err +[ $? -eq 1 ] || fail unreadable_status +[ -s ur.err ] || fail unreadable_msg +rmdir unreadable +rm ur.err + +# input is .., couldn't infer mount +unpack .. 2>dotdot.err +[ $? -eq 1 ] || fail dotdot_infer_mount_status +[ -s dotdot.err ] || fail dotdot_infer_mount_msg +rm dotdot.err + +# plain value input, already tested with null in bad_root.sh +unpack false.json 2>false.err +[ $? -eq 1 ] || fail false_bad_root_status +[ -s false.err ] || fail false_bad_root_msg +rm false.err + +## PACK +# input directory doesn't exist +pack nonesuch 2>nonesuch.err +[ $? -eq 1 ] || fail pack_no_input_dir_status +[ -s nonesuch.err ] || fail pack_no_input_dir_msg +rm nonesuch.err + +#### ERROR_STATUS_CLI +# unpack input is stdin but no mount specified +echo '{}' | unpack - 2>stdin.err +[ $? -eq 2 ] || fail stdin_nomount_status +[ -s stdin.err ] || fail stdin_nomount_msg +rm stdin.err + +# pack directory not specified +pack 2>nodir.err +[ $? -eq 2 ] || fail pack_no_dir_status +[ -s nodir.err ] || fail pack_no_dir_msg +rm nodir.err + +# bad shell completions +unpack --completions smoosh 2>comp.err +[ $? -eq 2 ] || fail unpack_comp_unsupported_shell_status +[ -s comp.err ] || fail unpack_comp_unsupported_shell_msg +rm comp.err + +pack --completions smoosh 2>comp.err +[ $? -eq 2 ] || fail pack_comp_unsupported_shell_status +[ -s comp.err ] || fail pack_comp_unsupported_shell_msg +rm comp.err + +# unknown unpack --type +unpack --type hieratic single.json 2>type.err +[ $? -eq 2 ] || fail unpack_unknown_type_status +[ -s type.err ] || fail unpack_unknown_type_msg +rm type.err + +# unknown pack --target +unpack --into unk_tgt single.json +pack --target hieratic unk_tgt 2>target.err +[ $? -eq 2 ] || fail pack_unknown_target_status +[ -s target.err ] || fail pack_unknown_target_msg +rm target.err + + +chmod +w unwriteable +cd "$TESTS" +rm -r "$D" || fail cleanup diff --git a/tests/unpack_pack_file_creation.sh b/tests/unpack_pack_file_creation.sh new file mode 100755 index 0000000..c5fabef --- /dev/null +++ b/tests/unpack_pack_file_creation.sh @@ -0,0 +1,40 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + fi + exit 1 +} + +MNT=$(mktemp -d) + +unpack --into "$MNT" ../json/object.json || fail unpack + +cd "$MNT" +case $(ls) in + (eyes*fingernails*human*name) ;; + (*) fail ls;; +esac +[ "$(cat name)" = "Michael Greenberg" ] || fail name +[ "$(cat eyes)" -eq 2 ] || fail eyes +[ "$(cat fingernails)" -eq 10 ] || fail fingernails +[ "$(cat human)" = "true" ] || fail human +touch jokes +[ -f jokes ] || fail touch +case $(ls) in + (eyes*fingernails*human*jokes*name) ;; + (*) fail ls2;; +esac +mkdir recipes +[ -d recipes ]|| fail mkdir +case $(ls) in + (eyes*fingernails*human*jokes*name*recipes) ;; + (*) fail ls3;; +esac +cd - >/dev/null 2>&1 + +pack "$MNT" || fail pack +rm -r "$MNT" || fail mount diff --git a/tests/unpack_pack_filename_spaces.sh b/tests/unpack_pack_filename_spaces.sh new file mode 100755 index 0000000..d525bfc --- /dev/null +++ b/tests/unpack_pack_filename_spaces.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + rm "$OUT" "$EXP" + fi + exit 1 +} + +MNT=$(mktemp -d) +OUT=$(mktemp) +EXP=$(mktemp) + +printf -- "---\nfield one: 1\nfield two: 2\nfield three: 3" >"$EXP" + +unpack --into "$MNT" --munge filter ../yaml/spaces.yaml || fail unpack + +case $(ls "$MNT") in + (field\ one*field\ two) ;; + (*) fail ls;; +esac +[ "$(cat $MNT/field\ one)" -eq 1 ] || fail one +[ "$(cat $MNT/field\ two)" -eq 2 ] || fail two +echo 3 >"$MNT"/field\ three + +pack --target yaml -o "$OUT" --munge filter "$MNT" || fail pack + +grep "field three: 3" $OUT >/dev/null 2>&1 || fail three + +sort $OUT >$OUT.yaml +sort $EXP >$EXP.yaml +diff $OUT.yaml $EXP.yaml || fail diff + +rm -r "$MNT" || fail mount +rm "$OUT" "$EXP" diff --git a/tests/unpack_pack_getxattr.sh b/tests/unpack_pack_getxattr.sh new file mode 100755 index 0000000..c400b87 --- /dev/null +++ b/tests/unpack_pack_getxattr.sh @@ -0,0 +1,44 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + fi + exit 1 +} + +if [ "$RUNNER_OS" = "Linux" ] || [ "$(uname)" = "Linux" ]; then + which getfattr || fail getfattr + getattr() { + attr=$1 + shift + getfattr -n "$attr" --only-values "$@" + } +elif [ "$RUNNER_OS" = "macOS" ] || [ "$(uname)" = "Darwin" ]; then + getattr() { + attr=$1 + shift + xattr -p "$attr" "$@" + } +else + fail os +fi + +typeof() { + getattr user.type "$@" +} + +MNT=$(mktemp -d) + +unpack --into "$MNT" ../json/object.json || fail unpack + +[ "$(typeof $MNT)" = "named" ] || fail root +[ "$(typeof $MNT/name)" = "string" ] || fail name +[ "$(typeof $MNT/eyes)" = "float" ] || fail eyes +[ "$(typeof $MNT/fingernails)" = "float" ] || fail fingernails +[ "$(typeof $MNT/human)" = "boolean" ] || fail human + +pack "$MNT" || fail pack +rm -r "$MNT" || fail mount diff --git a/tests/unpack_pack_infer_mount.sh b/tests/unpack_pack_infer_mount.sh new file mode 100755 index 0000000..e0e13df --- /dev/null +++ b/tests/unpack_pack_infer_mount.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$TMP"/object + rm -r "$TMP" + fi + exit 1 +} + +TMP=$(mktemp -d) + +cp ../json/object.json "$TMP" +cd "$TMP" +unpack object.json || fail unpack + +[ -d "object" ] || fail mountdir +case $(ls object) in + (eyes*fingernails*human*name) ;; + (*) fail ls;; +esac +[ "$(cat object/name)" = "Michael Greenberg" ] || fail name +[ "$(cat object/eyes)" -eq 2 ] || fail eyes +[ "$(cat object/fingernails)" -eq 10 ] || fail fingernails +[ "$(cat object/human)" = "true" ] || fail human +pack "$TMP"/object || fail pack +rm -r "$TMP"/object + +[ -d "object" ] && fail cleanup +cd - +rm -r "$TMP" diff --git a/tests/unpack_pack_infer_mount_relative.sh b/tests/unpack_pack_infer_mount_relative.sh new file mode 100755 index 0000000..cef1f7c --- /dev/null +++ b/tests/unpack_pack_infer_mount_relative.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$TMP"/nested/object + rm -r "$TMP" + fi + exit 1 +} + +TMP=$(mktemp -d) + +cp ../json/object.json "$TMP" +mkdir "$TMP"/nested +cd "$TMP"/nested + +unpack ../object.json || fail unpack + +[ -d "object" ] || fail mountdir +case $(ls object) in + (eyes*fingernails*human*name) ;; + (*) fail ls;; +esac +[ "$(cat object/name)" = "Michael Greenberg" ] || fail name +[ "$(cat object/eyes)" -eq 2 ] || fail eyes +[ "$(cat object/fingernails)" -eq 10 ] || fail fingernails +[ "$(cat object/human)" = "true" ] || fail human + +pack "$TMP"/nested/object || fail pack +rm -r "$TMP"/nested/object + +[ -d "object" ] && fail cleanup +cd - >/dev/null 2>&1 +rm -r "$TMP" diff --git a/tests/unpack_pack_json_roundtrip.sh b/tests/unpack_pack_json_roundtrip.sh new file mode 100755 index 0000000..3895d4b --- /dev/null +++ b/tests/unpack_pack_json_roundtrip.sh @@ -0,0 +1,32 @@ +#/bin/sh + +fail() { + echo FAILED: $1 + rm -r "$UNPACK_MNT0" + rm -r "$UNPACK_MNT1" + rm "$PACK_FILE0" + rm "$PACK_FILE1" + rm "$ERR_MSG" + exit 1 +} + +ERR_MSG=$(mktemp) +for f in ../json/*.json; do + UNPACK_MNT0=$(mktemp -d) + unpack $f --into "$UNPACK_MNT0" 2>"$ERR_MSG" + # skip the issue where it doesn't unpack into a directory structure + cat "$ERR_MSG" | grep -i -e "the unpacked form must be a directory" >/dev/null 2>&1 && continue + PACK_FILE0=$(mktemp) + UNPACK_MNT1=$(mktemp -d) + PACK_FILE1=$(mktemp) + pack "$UNPACK_MNT0" -t json >"$PACK_FILE0" || fail pack1 + unpack "$PACK_FILE0" -t json --into "$UNPACK_MNT1" || fail unpack2 + pack "$UNPACK_MNT1" -t json >"$PACK_FILE1" || fail pack2 + [ -z "$(diff $PACK_FILE0 $PACK_FILE1)" ] && [ -z "$(diff -r $UNPACK_MNT0 $UNPACK_MNT1)" ] || fail diff + rm -r "$UNPACK_MNT0" + rm -r "$UNPACK_MNT1" + rm "$PACK_FILE0" + rm "$PACK_FILE1" +done + +rm "$ERR_MSG" diff --git a/tests/unpack_pack_json_to_toml.sh b/tests/unpack_pack_json_to_toml.sh new file mode 100755 index 0000000..3ad7b16 --- /dev/null +++ b/tests/unpack_pack_json_to_toml.sh @@ -0,0 +1,39 @@ +#/bin/sh + +# convert json to toml with unpack pack + +# unpack from format 1 +# pack to format 2 +# unpack from format 2 +# diff -r unpacked1 unpacked2 + +fail() { + echo FAILED: $1 + rm -r "$UNPACK_MNT0" + rm -r "$UNPACK_MNT1" + rm "$PACK_FILE0" + rm "$ERR_MSG" + exit 1 +} + +ERR_MSG=$(mktemp) +# reasons for skipping: +# json_eg5.json has null values +# list.json and list2.json are lists at the top level, which toml doesn't support +# object_null.json has a null value +for f in $(find ../json -maxdepth 1 -name '*.json' ! -name 'json_eg5.json' ! -name 'list*.json' ! -name 'object_null.json'); do + UNPACK_MNT0=$(mktemp -d) + unpack $f --into "$UNPACK_MNT0" 2>"$ERR_MSG" + # skip the issue where it doesn't unpack into a directory structure + cat "$ERR_MSG" | grep -i -e "the unpacked form must be a directory" >/dev/null 2>&1 && continue + PACK_FILE0=$(mktemp) + UNPACK_MNT1=$(mktemp -d) + pack "$UNPACK_MNT0" -t toml >"$PACK_FILE0" || fail pack + unpack "$PACK_FILE0" -t toml --into "$UNPACK_MNT1" || fail unpack2 + [ -z "$(diff -r $UNPACK_MNT0 $UNPACK_MNT1)" ] || fail diff + rm -r "$UNPACK_MNT0" + rm -r "$UNPACK_MNT1" + rm "$PACK_FILE0" +done + +rm "$ERR_MSG" diff --git a/tests/unpack_pack_json_to_yaml.sh b/tests/unpack_pack_json_to_yaml.sh new file mode 100755 index 0000000..0c92277 --- /dev/null +++ b/tests/unpack_pack_json_to_yaml.sh @@ -0,0 +1,35 @@ +#/bin/sh + +# convert json to yaml with unpack pack + +# unpack from format 1 +# pack to format 2 +# unpack from format 2 +# diff -r unpacked1 unpacked2 + +fail() { + echo FAILED: $1 + rm -r "$UNPACK_MNT0" + rm -r "$UNPACK_MNT1" + rm "$PACK_FILE0" + rm "$ERR_MSG" + exit 1 +} + +ERR_MSG=$(mktemp) +for f in $(find ../json -maxdepth 1 -name '*.json'); do + UNPACK_MNT0=$(mktemp -d) + unpack $f --into "$UNPACK_MNT0" 2>"$ERR_MSG" + # skip the issue where it doesn't unpack into a directory structure + cat "$ERR_MSG" | grep -i -e "the unpacked form must be a directory" >/dev/null 2>&1 && continue + PACK_FILE0=$(mktemp) + UNPACK_MNT1=$(mktemp -d) + pack "$UNPACK_MNT0" -t yaml >"$PACK_FILE0" || fail pack + unpack "$PACK_FILE0" -t yaml --into "$UNPACK_MNT1" || fail unpack2 + [ -z "$(diff -r $UNPACK_MNT0 $UNPACK_MNT1)" ] || fail diff + rm -r "$UNPACK_MNT0" + rm -r "$UNPACK_MNT1" + rm "$PACK_FILE0" +done + +rm "$ERR_MSG" diff --git a/tests/unpack_pack_listxattr.sh b/tests/unpack_pack_listxattr.sh new file mode 100755 index 0000000..a1c6cef --- /dev/null +++ b/tests/unpack_pack_listxattr.sh @@ -0,0 +1,40 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + fi + exit 1 +} + +if [ "$RUNNER_OS" = "Linux" ] || [ "$(uname)" = "Linux" ]; then + which getfattr || fail getfattr + listattr() { + getfattr --match=- "$@" + } +elif [ "$RUNNER_OS" = "macOS" ] || [ "$(uname)" = "Darwin" ]; then + listattr() { + xattr -l "$@" + } +else + fail os +fi + +listattr_ok() { + listattr $1 | grep "user.type" +} + +MNT=$(mktemp -d) + +unpack --into "$MNT" ../json/object.json || fail unpack + +listattr_ok "$MNT" || fail root +listattr_ok "$MNT"/name || fail name +listattr_ok "$MNT"/eyes || fail eyes +listattr_ok "$MNT"/fingernails || fail fingernails +listattr_ok "$MNT"/human || fail human + +pack "$MNT" || fail pack +rm -r "$MNT" || fail mount diff --git a/tests/unpack_pack_macos_noxattr_cleanup.sh b/tests/unpack_pack_macos_noxattr_cleanup.sh new file mode 100755 index 0000000..fb8e295 --- /dev/null +++ b/tests/unpack_pack_macos_noxattr_cleanup.sh @@ -0,0 +1,128 @@ +#!/bin/sh + +if ! [ "$RUNNER_OS" = "macOS" ] && ! [ "$(uname)" = "Darwin" ] +then + echo "This test only runs under macOS; you're using ${RUNNER_OS-$(uname)}" >&2 + exit 0 +fi + +VERSION="$( sw_vers -productVersion | cut -d. -f1 )" +pre_ventura_test() { + if [ $VERSION -lt 13 ] + then + true + else + false + fi +} +non_macosdot_filesystem_test() { + TESTDIR=$(mktemp -d) + touch "$TESTDIR"/testfile + xattr -w xattr_test xattr_test "$TESTDIR"/testfile + if ! [ -e "$TESTDIR"/._testfile ] + then + rm -r "$TESTDIR" + true + else + rm -r "$TESTDIR" + false + fi +} + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -rf "$MNT" + rm "$OUT" + fi + exit 1 +} + +listattr() { + xattr -l "$@" +} +getattr() { + attr=$1 + shift + xattr -p "$attr" "$@" +} +setattr() { + attr="$1" + val="$2" + shift 2 + xattr -w "$attr" "$val" "$@" +} +rmattr() { + attr=$1 + shift + xattr -d "$attr" "$@" +} + +typeof() { + getattr user.type "$@" +} + +MNT=$(mktemp -d) +OUT=$(mktemp) + +unpack --into "$MNT" --no-xattr ../json/object.json || fail unpack1 + +[ "$(typeof $MNT)" = "named" ] && fail root +[ "$(typeof $MNT/name)" = "string" ] && fail name +[ "$(typeof $MNT/eyes)" = "float" ] && fail eyes +[ "$(typeof $MNT/fingernails)" = "float" ] && fail fingernails +[ "$(typeof $MNT/human)" = "boolean" ] && fail human + +setattr user.type list "$MNT" || fail set1 + +[ "$(typeof $MNT)" = "list" ] || fail "macos override" + +pack -o "$OUT" --no-xattr --target json "$MNT" || fail pack1 + +# for all the grep tests in this file, instead of looking for the literal "._.", +# look for any strings beginning with ._ +grep -e '"\._.*"' "$OUT" >/dev/null 2>&1 && fail metadata1 + +rm -rf "$MNT" +rm "$OUT" + +# now try to keep the metadata +unpack --into "$MNT" --no-xattr ../json/object.json || fail unpack2 +setattr user.type list "$MNT" + +pack -o "$OUT" --no-xattr --keep-macos-xattr --target json "$MNT" || fail pack2 + +# ffs creates a literal ._. file because it can't store the xattr of the root of the fuse filesystem +# outside the mount. Therefore, there is only one xattr (._.) in the output for the 2nd test for ffs. +# +# For OS version >= 13 and on filesystems that create ._ files for xattrs, the com.apple.provenance +# xattr is automatically created despite using --no-xattr for unpack. So, pack will find these pointless +# ._ files and include them. Technically, this means the grep command should be successful. +# However, what this means for OS version < 13 is that there should be no ._ files created by unpack and the +# setattr command above should only create the ._(name of unpacked directory) file, outside the directory +# in which the data is unpacked. So, pack will never see ._(name of unpacked directory) and will only +# output the 4 json attributes inside the json object (eyes, fingernails, human, name). +# So, the grep command should fail. +# +# If the grep fails, then if OS version < 13, the ._files were not created by default, so the test passes. +# Otherwise, if the current filesystem doesn't create ._files for xattrs at all, then the test passes. +# Otherwise, the test fails. +grep -e '"\._.*"' "$OUT" >/dev/null 2>&1 || pre_ventura_test || non_macosdot_filesystem_test || fail metadata2 + +rm -rf "$MNT" +rm "$OUT" + +# now try to keep the metadata but also have the FS store it +unpack --into "$MNT" ../json/object.json || fail unpack3 + +setattr user.type list "$MNT" + +pack -o "$OUT" --keep-macos-xattr --target json "$MNT" || fail pack3 + +# technically, the output of pack here still differs from that of ffs on macosdot filesystems because ffs doesn't +# create xattrs for (eyes, fingernails, human, name). +grep -e '"\._.*"' "$OUT" >/dev/null 2>&1 && fail metadata3 + +rm -rf "$MNT" || fail mount +rm "$OUT" diff --git a/tests/unpack_pack_max_depth.sh b/tests/unpack_pack_max_depth.sh new file mode 100755 index 0000000..ffa94be --- /dev/null +++ b/tests/unpack_pack_max_depth.sh @@ -0,0 +1,44 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm "$OUT" "$EXP" + rm -r "$MNT" + fi + exit 1 +} + +MNT=$(mktemp -d) +MNT2=$(mktemp -d) +OUT=$(mktemp) +mv "$OUT" "$OUT".json +OUT="$OUT".json + +mkdir -p "$MNT"/a/b/c/d/e/f/g/h/i/j +echo "file" >"$MNT"/a/b/c/d/e/f/g/h/i/j/file +mkdir -p "$MNT2"/symlink/test +cd "$MNT2"/symlink/test +ln -s "$MNT" link + +EXP=$(mktemp) + +printf '{"a":{"b":{"c":{"d":{"e":{"f":{}}}}}}}' >"$EXP" +pack --max-depth 6 -o "$OUT" -- "$MNT" || fail pack +diff "$OUT" "$EXP" || fail diff + +printf '{"a":{"b":{"c":{"d":{"e":{"f":{"g":{"h":{"i":{"j":{}}}}}}}}}}}' >"$EXP" +pack --max-depth 10 -o "$OUT" -- "$MNT" || fail pack2 +diff "$OUT" "$EXP" || fail diff2 + +printf '{"a":{"b":{"c":{"d":{"e":{"f":{"g":{"h":{"i":{"j":{"file":"file"}}}}}}}}}}}' >"$EXP" +pack --max-depth 11 -o "$OUT" -- "$MNT" || fail pack3 +diff "$OUT" "$EXP" || fail diff3 + +printf '{"symlink":{"test":{"link":{"a":{"b":{"c":{"d":{"e":{"f":{"g":{"h":{"i":{"j":{"file":"file"}}}}}}}}}}}}}}' >"$EXP" +pack --max-depth 14 -o "$OUT" -L --allow-symlink-escape -- "$MNT2" || fail pack4 +diff "$OUT" "$EXP" || fail diff4 + +rm "$OUT" "$EXP" +rm -r "$MNT" diff --git a/tests/unpack_pack_munge_filter.sh b/tests/unpack_pack_munge_filter.sh new file mode 100755 index 0000000..9500004 --- /dev/null +++ b/tests/unpack_pack_munge_filter.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + fi + exit 1 +} + +MNT=$(mktemp -d) + +unpack --into "$MNT" --munge filter ../json/obj_rename.json || fail unpack + +case $(ls "$MNT") in + (dot*dotdot) ;; + (*) fail ls;; +esac +[ "$(cat $MNT/dot)" = "third" ] || fail dot +[ "$(cat $MNT/dotdot)" = "fourth" ] || fail dotdot + +pack "$MNT" || fail pack +rm -r "$MNT" || fail mount diff --git a/tests/unpack_pack_newline_cleanup.sh b/tests/unpack_pack_newline_cleanup.sh new file mode 100755 index 0000000..5c188d9 --- /dev/null +++ b/tests/unpack_pack_newline_cleanup.sh @@ -0,0 +1,48 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + rm -r "$EXP" + rm "$JSON" + fi + exit 1 +} + +MNT=$(mktemp -d) +EXP=$(mktemp -d) +JSON=$(mktemp) + +# generate files w/newlines +printf "Michael Greenberg" >"${EXP}/name" +printf "2" >"${EXP}/eyes" +printf "10" >"${EXP}/fingernails" +printf "true" >"${EXP}/human" +printf "hi" >"${EXP}/greeting" +printf "bye" >"${EXP}/farewell" + +unpack --into "$MNT" ../json/object.json || fail unpack1 + +echo hi >"$MNT"/greeting +printf "bye" >"$MNT"/farewell + +pack -o "$JSON" "$MNT" || fail pack1 +rm -r "$MNT" + +# remount w/ --exact, confirm that they're not there +unpack --exact --into "$MNT" "$JSON" || fail unpack2 + +case $(ls "$MNT") in + (eyes*farewell*fingernails*greeting*human*name) ;; + (*) fail ls;; +esac +for x in "$EXP"/* +do + diff "$x" "$MNT/$(basename $x)" || fail "$(basename $x)" +done + +pack "$MNT" || fail pack2 +rm -r "$MNT" || fail mount +rm -r "$EXP" diff --git a/tests/unpack_pack_nlink.sh b/tests/unpack_pack_nlink.sh new file mode 100755 index 0000000..60ad0ee --- /dev/null +++ b/tests/unpack_pack_nlink.sh @@ -0,0 +1,48 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + fi + exit 1 +} + +if [ "$RUNNER_OS" = "Linux" ] || [ "$(uname)" = "Linux" ]; then + num_links() { + stat --format %h "$@" + } +elif [ "$RUNNER_OS" = "macOS" ] || [ "$(uname)" = "Darwin" ]; then + num_links() { + stat -f %l "$@" + } +else + fail os +fi + +MNT=$(mktemp -d) + +unpack --into "$MNT" ../json/nlink.json || fail unpack + +cd "$MNT" +case $(ls) in + (child1*child2*child3) ;; + (*) fail ls;; +esac +[ -d . ] && [ -d child1 ] && [ -f child2 ] && [ -d child3 ] || fail filetypes +# APFS on macOS counts directories differently +if [ "$RUNNER_OS" = "macOS" ] || [ "$(uname)" = "Darwin" ] +then + MACOS_DIR=1 +else + MACOS_DIR=0 +fi +[ $(num_links .) -eq $((4 + MACOS_DIR)) ] || fail root # parent + self + child1 + child3 +[ $(num_links child1) -eq $((2 + MACOS_DIR)) ] || fail child1 # parent + self +[ $(num_links child2) -eq 1 ] || fail child2 # parent +[ $(num_links child3) -eq $((2 + MACOS_DIR)) ] || fail child3 # parent + self +cd - >/dev/null 2>&1 + +pack "$MNT" || fail pack +rm -r "$MNT" || fail mount diff --git a/tests/unpack_pack_noxattr.sh b/tests/unpack_pack_noxattr.sh new file mode 100755 index 0000000..6d5b3f8 --- /dev/null +++ b/tests/unpack_pack_noxattr.sh @@ -0,0 +1,109 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + fi + if [ "$MNT2" ] + then + rm -r "$MNT2" + fi + exit 1 +} + +if [ "$RUNNER_OS" = "Linux" ] || [ "$(uname)" = "Linux" ]; then + which getfattr || fail getfattr + which setfattr || fail setfattr + getattr() { + attr=$1 + shift + getfattr -n "$attr" --only-values "$@" + } + setattr() { + attr="$1" + val="$2" + shift 2 + setfattr -n "$attr" -v "$val" "$@" + } + listattr() { + getfattr --match=- "$@" + } + rmattr() { + attr=$1 + shift + setfattr -x "$attr" "$@" + } +elif [ "$RUNNER_OS" = "macOS" ] || [ "$(uname)" = "Darwin" ]; then + listattr() { + xattr -l "$@" + } + getattr() { + attr=$1 + shift + xattr -p "$attr" "$@" + } + setattr() { + attr="$1" + val="$2" + shift 2 + xattr -w "$attr" "$val" "$@" + } + rmattr() { + attr=$1 + shift + xattr -d "$attr" "$@" + } + +else + fail os +fi + +typeof() { + getattr user.type "$@" +} + +MNT=$(mktemp -d) + +unpack --into "$MNT" --no-xattr ../json/object.json || fail unpack1 + +[ "$(typeof $MNT)" = "named" ] && fail root +[ "$(typeof $MNT/name)" = "string" ] && fail name +[ "$(typeof $MNT/eyes)" = "float" ] && fail eyes +[ "$(typeof $MNT/fingernails)" = "float" ] && fail fingernails +[ "$(typeof $MNT/human)" = "boolean" ] && fail human + + + +listattr_fails() { + ! listattr $1 | grep "user.type" +} + +listattr_fails "$MNT" || fail root +listattr_fails "$MNT"/name || fail name +listattr_fails "$MNT"/eyes || fail eyes +listattr_fails "$MNT"/fingernails || fail fingernails +listattr_fails "$MNT"/human || fail human + +# unlike ffs, we can set xattrs even if unpack didn't +setattr user.type list "$MNT" || fail "root user.type" +setattr user.fake list "$MNT" || fail "root user.fake" + +listattr "$MNT" | grep "user.type" || fail "root user.type missing" +listattr "$MNT" | grep "user.fake" || fail "root user.fake missing" + +rmattr user.type "$MNT" || fail "root user.type" +rmattr user.fake "$MNT" || fail "root user.fake" +rmattr user.type "$MNT"/name && fail "root user.type" + + +GOT="$(mktemp)" +pack "$MNT" >"$GOT" || fail pack1 +MNT2="$(mktemp -d)" +unpack --into "$MNT2" "$GOT" || fail unpack2 +diff -r "$MNT" "$MNT2" || fail "modified output" + +pack "$MNT2" || fail pack2 +rm -r "$MNT" || fail mount +rm -r "$MNT2" || fail mount2 diff --git a/tests/unpack_pack_output.sh b/tests/unpack_pack_output.sh new file mode 100755 index 0000000..3146d60 --- /dev/null +++ b/tests/unpack_pack_output.sh @@ -0,0 +1,57 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + rm "$TGT" + rm "$TGT2" + fi + exit 1 +} + +MNT=$(mktemp -d) +TGT=$(mktemp) +TGT2=$(mktemp) + +unpack --into "$MNT" ../json/object.json || fail unpack1 +mkdir "$MNT"/pockets +echo keys >"$MNT"/pockets/pants +echo pen >"$MNT"/pockets/shirt +cd - >/dev/null 2>&1 +pack "$MNT" >"$TGT" || fail pack1 +rm -r "$MNT" + +# easiest to just test using ffs, but would be cool to get outside validation +[ -f "$TGT" ] || fail output1 +[ -s "$TGT" ] || fail output2 +cat "$TGT" +stat "$TGT" +unpack --into "$MNT" "$TGT" || fail unpack2 + +case $(ls "$MNT") in + (eyes*fingernails*human*name*pockets) ;; + (*) fail ls1;; +esac +case $(ls "$MNT"/pockets) in + (pants*shirt) ;; + (*) fail ls2;; +esac + +[ "$(cat $MNT/name)" = "Michael Greenberg" ] || fail name +[ "$(cat $MNT/eyes)" -eq 2 ] || fail eyes +[ "$(cat $MNT/fingernails)" -eq 10 ] || fail fingernails +[ "$(cat $MNT/human)" = "true" ] || fail human +[ "$(cat $MNT/pockets/pants)" = "keys" ] || fail pants +[ "$(cat $MNT/pockets/shirt)" = "pen" ] || fail shirt + +pack --no-output "$MNT" >"$TGT2" || fail pack2 + +stat "$TGT2" +[ -f "$TGT2" ] || fail tgt2 +[ -s "$TGT2" ] && fail tgt2_nonempty + +rm -r "$MNT" || fail mount +rm "$TGT" +rm "$TGT2" diff --git a/tests/unpack_pack_override_infer.sh b/tests/unpack_pack_override_infer.sh new file mode 100755 index 0000000..394c268 --- /dev/null +++ b/tests/unpack_pack_override_infer.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + rm "$SRC" "$TGT" + fi + exit 1 +} + +MNT=$(mktemp -d) +SRC=$(mktemp) +TGT=$(mktemp) + +cp ../toml/single.toml "$SRC" + +unpack --type toml --into "$MNT" "$SRC" || fail unpack + +pack --target json -o "$TGT" "$MNT" || fail pack + +diff "$TGT" ../json/single.json || fail diff + +rm -r "$MNT" || fail mount +rm "$SRC" "$TGT" + diff --git a/tests/unpack_pack_pad_list.sh b/tests/unpack_pack_pad_list.sh new file mode 100755 index 0000000..41bcf22 --- /dev/null +++ b/tests/unpack_pack_pad_list.sh @@ -0,0 +1,35 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + fi + exit 1 +} + +MNT=$(mktemp -d) + +unpack --into "$MNT" ../json/list2.json || fail unpack + +cd "$MNT" +case $(ls) in + (00*01*02*03*04*05*06*07*08*09*10) ;; + (*) fail ls;; +esac +[ "$(cat 00)" -eq 0 ] || fail 0 +[ "$(cat 01)" -eq 1 ] || fail 1 +[ "$(cat 02)" -eq 2 ] || fail 2 +[ "$(cat 03)" -eq 3 ] || fail 3 +[ "$(cat 04)" -eq 4 ] || fail 4 +[ "$(cat 05)" -eq 5 ] || fail 5 +[ "$(cat 06)" -eq 6 ] || fail 6 +[ "$(cat 07)" -eq 7 ] || fail 7 +[ "$(cat 08)" -eq 8 ] || fail 8 +[ "$(cat 09)" -eq 9 ] || fail 9 +[ "$(cat 10)" -eq 10 ] || fail 10 +cd - >/dev/null 2>&1 + +pack "$MNT" || fail pack +rm -r "$MNT" || fail mount diff --git a/tests/unpack_pack_pretty_json.sh b/tests/unpack_pack_pretty_json.sh new file mode 100755 index 0000000..30226a3 --- /dev/null +++ b/tests/unpack_pack_pretty_json.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + rm "$OUT" + fi + exit 1 +} + +MNT=$(mktemp -d) +OUT=$(mktemp) + +unpack --into "$MNT" ../json/object.json || fail unpack + +echo mgree >"$MNT"/handle + +pack --target json -o "$OUT" --pretty "$MNT" || fail pack + +[ "$(cat $OUT | wc -l)" -eq 6 ] || fail lines +grep '^\s*"handle": "mgree",$' "$OUT" >/dev/null 2>&1 || fail handle + +rm -r "$MNT" || fail mount +rm "$OUT" diff --git a/tests/unpack_pack_pretty_toml.sh b/tests/unpack_pack_pretty_toml.sh new file mode 100755 index 0000000..b001dd3 --- /dev/null +++ b/tests/unpack_pack_pretty_toml.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + rm "$OUT" + fi + exit 1 +} + +MNT=$(mktemp -d) +OUT=$(mktemp) + +unpack --into "$MNT" ../toml/single.toml || fail unpack + +cat >"$MNT"/info <>"$LOG" 2>&1 || fail unpack1 + +echo hi >"$MNT"/greeting + +pack -q -o "$JSON" "$MNT" >>"$LOG" 2>&1 || fail pack1 +diff ../json/object.json "$JSON" >/dev/null && fail same +[ "$(cat $LOG)" = "" ] || fail quiet +rm -r "$MNT" + +unpack --into "$MNT" "$JSON" || fail unpack2 + +[ "$(cat $MNT/greeting)" = "hi" ] || fail updated + +pack "$MNT" || fail pack2 + +rm -r "$MNT" || fail mount +rm "$JSON" || fail copy diff --git a/tests/unpack_pack_removexattr.sh b/tests/unpack_pack_removexattr.sh new file mode 100755 index 0000000..16197f8 --- /dev/null +++ b/tests/unpack_pack_removexattr.sh @@ -0,0 +1,39 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + fi + exit 1 +} + +if [ "$RUNNER_OS" = "Linux" ] || [ "$(uname)" = "Linux" ]; then + which setfattr || fail setfattr + rmattr() { + attr=$1 + shift + setfattr -x "$attr" "$@" + } +elif [ "$RUNNER_OS" = "macOS" ] || [ "$(uname)" = "Darwin" ]; then + rmattr() { + attr=$1 + shift + xattr -d "$attr" "$@" + } +else + fail os +fi + +MNT=$(mktemp -d) + +unpack --into "$MNT" ../json/object.json || fail unpack + +rmattr user.type "$MNT" || fail "root user.type" +rmattr user.fake "$MNT" && fail "root user.fake" +rmattr user.type "$MNT"/name || fail "user.type" + +pack "$MNT" || fail pack + +rm -r "$MNT" || fail mount diff --git a/tests/unpack_pack_rename.sh b/tests/unpack_pack_rename.sh new file mode 100755 index 0000000..b577ecd --- /dev/null +++ b/tests/unpack_pack_rename.sh @@ -0,0 +1,43 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + fi + exit 1 +} + +MNT=$(mktemp -d) + +unpack --into "$MNT" ../json/object.json || fail unpack + +cd "$MNT" +case $(ls) in + (eyes*fingernails*human*name) ;; + (*) fail ls1;; +esac +mv name full_name +[ "$(cat full_name)" = "Michael Greenberg" ] || fail name1 +case $(ls) in + (eyes*fingernails*full_name*human) ;; + (*) fail ls2;; +esac +echo Prof. G >name +mv full_name name +case $(ls) in + (eyes*fingernails*human*name) ;; + (*) fail ls3;; +esac +[ "$(cat name)" = "Michael Greenberg" ] || fail name2 +mv nonesuch name && fail mv1 +case $(ls) in + (eyes*fingernails*human*name) ;; + (*) fail ls4;; +esac +cd - >/dev/null 2>&1 + +pack "$MNT" || fail pack + +rm -r "$MNT" || fail mount diff --git a/tests/unpack_pack_rename_fancy_restore.sh b/tests/unpack_pack_rename_fancy_restore.sh new file mode 100755 index 0000000..2eb51e3 --- /dev/null +++ b/tests/unpack_pack_rename_fancy_restore.sh @@ -0,0 +1,50 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + rm "$OUT" "$EXP" + fi + exit 1 +} + +MNT=$(mktemp -d) +OUT=$(mktemp) +EXP=$(mktemp) + +printf '{"he":{"dot":"shlishi"},"imnewhere":"derp","it":{".":"primo","..":"secondo"}}' >"$EXP" + +unpack --into "$MNT" ../json/obj_rename.json || fail unpack + +case $(ls "$MNT") in + (_.*_..*dot*dotdot) ;; + (*) fail ls;; +esac +[ "$(cat $MNT/_.)" = "first" ] || fail . +[ "$(cat $MNT/_..)" = "second" ] || fail .. +[ "$(cat $MNT/dot)" = "third" ] || fail dot +[ "$(cat $MNT/dotdot)" = "fourth" ] || fail dotdot + +echo primo >"$MNT"/_. +echo secondo >"$MNT"/_.. +echo shlishi >"$MNT"/dot +echo derp >"$MNT"/dotdot + +mkdir "$MNT"/it +mkdir "$MNT"/he + +mv "$MNT"/_. "$MNT"/it +mv "$MNT"/_.. "$MNT"/it + +mv "$MNT"/dot "$MNT"/he + +mv "$MNT"/dotdot "$MNT"/imnewhere + +pack --target json -o "$OUT" "$MNT" || fail pack + +diff "$OUT" "$EXP" || fail diff + +rm -r "$MNT" || fail mount +rm "$OUT" "$EXP" diff --git a/tests/unpack_pack_rename_object.sh b/tests/unpack_pack_rename_object.sh new file mode 100755 index 0000000..3d67b5a --- /dev/null +++ b/tests/unpack_pack_rename_object.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + fi + exit 1 +} + +MNT=$(mktemp -d) + +unpack --into "$MNT" ../json/obj_rename.json || fail unpack + +case $(ls "$MNT") in + (_.*_..*dot*dotdot) ;; + (*) fail ls;; +esac +[ "$(cat $MNT/_.)" = "first" ] || fail . +[ "$(cat $MNT/_..)" = "second" ] || fail .. +[ "$(cat $MNT/dot)" = "third" ] || fail dot +[ "$(cat $MNT/dotdot)" = "fourth" ] || fail dotdot + +pack "$MNT" || fail pack +rm -r "$MNT" || fail mount diff --git a/tests/unpack_pack_rename_restore.sh b/tests/unpack_pack_rename_restore.sh new file mode 100755 index 0000000..7315d3a --- /dev/null +++ b/tests/unpack_pack_rename_restore.sh @@ -0,0 +1,40 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + rm "$OUT" "$EXP" + fi + exit 1 +} + +MNT=$(mktemp -d) +OUT=$(mktemp) +EXP=$(mktemp) + +printf '{".":"primo","..":"secondo","dot":"terzo","dotdot":"quarto"}' >"$EXP" + +unpack --into "$MNT" ../json/obj_rename.json || fail unpack + +case $(ls "$MNT") in + (_.*_..*dot*dotdot) ;; + (*) fail ls;; +esac +[ "$(cat $MNT/_.)" = "first" ] || fail . +[ "$(cat $MNT/_..)" = "second" ] || fail .. +[ "$(cat $MNT/dot)" = "third" ] || fail dot +[ "$(cat $MNT/dotdot)" = "fourth" ] || fail dotdot + +echo primo >"$MNT"/_. +echo secondo >"$MNT"/_.. +echo terzo >"$MNT"/dot +echo quarto >"$MNT"/dotdot + +pack -o "$OUT" --target json "$MNT" || fail pack + +diff "$OUT" "$EXP" || fail diff + +rm -r "$MNT" || fail mount +rm "$OUT" "$EXP" diff --git a/tests/unpack_pack_rmdir.sh b/tests/unpack_pack_rmdir.sh new file mode 100755 index 0000000..b6dac50 --- /dev/null +++ b/tests/unpack_pack_rmdir.sh @@ -0,0 +1,52 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + fi + exit 1 +} + +MNT=$(mktemp -d) + +unpack --into "$MNT" ../json/object.json || fail unpack + +cd "$MNT" +case $(ls) in + (eyes*fingernails*human*name) ;; + (*) fail ls;; +esac +[ "$(cat name)" = "Michael Greenberg" ] || fail name +[ "$(cat eyes)" -eq 2 ] || fail eyes +[ "$(cat fingernails)" -eq 10 ] || fail fingernails +[ "$(cat human)" = "true" ] || fail human1 +rm human +case $(ls) in + (eyes*fingernails*name) ;; + (*) fail ls2;; +esac +mkdir pockets +case $(ls) in + (eyes*fingernails*name*pockets) ;; + (*) fail ls3;; +esac +rm pockets && fail rm1 +case $(ls) in + (eyes*fingernails*name*pockets) ;; + (*) fail ls4;; +esac +echo keys >pockets/pants +rmdir pockets && fail rm2 +rm pockets/pants +rmdir pockets || fail rmdir +case $(ls) in + (eyes*fingernails*name) ;; + (*) fail ls5;; +esac +cd - >/dev/null 2>&1 + +pack "$MNT" || fail pack + +rm -r "$MNT" || fail mount diff --git a/tests/unpack_pack_setxattr.sh b/tests/unpack_pack_setxattr.sh new file mode 100755 index 0000000..145f429 --- /dev/null +++ b/tests/unpack_pack_setxattr.sh @@ -0,0 +1,49 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + fi + exit 1 +} + +if [ "$RUNNER_OS" = "Linux" ] || [ "$(uname)" = "Linux" ]; then + which setfattr || fail setfattr + setattr() { + attr="$1" + val="$2" + shift 2 + setfattr -n "$attr" -v "$val" "$@" + } +elif [ "$RUNNER_OS" = "macOS" ] || [ "$(uname)" = "Darwin" ]; then + setattr() { + attr="$1" + val="$2" + shift 2 + xattr -w "$attr" "$val" "$@" + } +else + fail os +fi + +MNT=$(mktemp -d) +OUT=$(mktemp) +EXP=$(mktemp) + +# NB no newline. this is a little hardcoded for my taste, but yolo +printf '[2,10,"true","Michael Greenberg"]' >"$EXP" + +unpack --into "$MNT" ../json/object.json || fail unpack + +setattr user.type list "$MNT" || fail "root user.type" +setattr user.fake list "$MNT" || fail "root user.fake" +setattr user.type string "$MNT"/human || fail "human" + +pack --target json -o "$OUT" "$MNT" || fail pack + +[ "$(cat $OUT)" = "$(cat $EXP)" ] || fail "different strings" +diff "$OUT" "$EXP" || fail "different files" + +rm -r "$MNT" || fail mount diff --git a/tests/unpack_pack_symlink.sh b/tests/unpack_pack_symlink.sh new file mode 100755 index 0000000..762eb15 --- /dev/null +++ b/tests/unpack_pack_symlink.sh @@ -0,0 +1,289 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm "$EXP" "$OUT" + rm -r "$MNT" + fi + exit 1 +} + +if [ "$RUNNER_OS" = "Linux" ] || [ "$(uname)" = "Linux" ]; then + which getfattr || fail getfattr + which setfattr || fail setfattr + getattr() { + attr=$1 + shift + getfattr -h -n "$attr" --only-values "$@" + } + setattr() { + attr="$1" + val="$2" + shift 2 + setfattr -h -n "$attr" -v "$val" "$@" + } + listattr() { + getfattr -h --match=- "$@" + } + rmattr() { + attr=$1 + shift + setfattr -h -x "$attr" "$@" + } +elif [ "$RUNNER_OS" = "macOS" ] || [ "$(uname)" = "Darwin" ]; then + listattr() { + xattr -s -l "$@" + } + getattr() { + attr=$1 + shift + xattr -s -p "$attr" "$@" + } + setattr() { + attr="$1" + val="$2" + shift 2 + xattr -s -w "$attr" "$val" "$@" + } + rmattr() { + attr=$1 + shift + xattr -s -d "$attr" "$@" + } +else + fail os +fi + +typeof() { + getattr user.type "$@" +} + +MNT=$(mktemp -d) +EXP=$(mktemp) +OUT=$(mktemp) +mv "$OUT" "$OUT".json +OUT="$OUT".json + +# chain of symlinks and symlink to directory +# test0 +# ├── a +# ├── b -> a +# ├── c -> b +# ├── d -> c +# ├── e -> d +# ├── tree +# │ ├── about +# │ └── root +# └── treecopy -> tree + +cd "$MNT" +echo 'a' >a +ln -s a b +ln -s b c +ln -s c d +ln -s d e +mkdir tree +ln -s tree treecopy +cd tree +echo 'tree about' >about +echo 'tree root' >root + +printf '{"a":"a","tree":{"about":"tree about","root":"tree root"}}' >"$EXP" +pack -o "$OUT" -- "$MNT" || fail pack1 +diff "$EXP" "$OUT" || fail "test0 no-follow" + +printf '{"a":"a","b":"a","c":"a","d":"a","e":"a","tree":{"about":"tree about","root":"tree root"},"treecopy":{"about":"tree about","root":"tree root"}}' >"$EXP" +pack -o "$OUT" -L -- "$MNT" || fail pack2 +diff "$EXP" "$OUT" || fail "test0 follow" + +rm -r "$MNT" +mkdir "$MNT" + +# symlinks in list directories +# test1 +# ├── ascending +# │ ├── 0 -> 1 +# │ ├── 1 -> 2 +# │ ├── 2 -> 3 +# │ ├── 3 -> 4 +# │ └── 4 +# └── descending +# ├── 0 +# ├── 1 -> 0 +# ├── 2 -> 1 +# ├── 3 -> 2 +# └── 4 -> 3 + +cd "$MNT" +mkdir ascending descending +cd ascending +echo '4' >4 +ln -s 4 3 +ln -s 3 2 +ln -s 2 1 +ln -s 1 0 +cd ../descending +echo '0' >0 +ln -s 0 1 +ln -s 1 2 +ln -s 2 3 +ln -s 3 4 + +printf '{"ascending":[4],"descending":[0]}' >"$EXP" +pack -o "$OUT" -- "$MNT" || fail pack3 +diff "$EXP" "$OUT" || fail "test1 no-follow" + +printf '{"ascending":[4,4,4,4,4],"descending":[0,0,0,0,0]}' >"$EXP" +pack -o "$OUT" -L -- "$MNT" || fail pack4 +diff "$EXP" "$OUT" || fail "test1 follow" + +rm -r "$MNT" +mkdir "$MNT" + +# relative and absolute path symlinks to some path in mount +# test2 +# └── path +# └── to +# ├── other +# │ └── file +# │ └── data +# └── some +# └── link +# ├── abs -> "$MNT"/path/to/other/file/data +# └── rel -> ../../other/file/data + +cd "$MNT" +mkdir -p path/to/some/link path/to/other/file +touch path/to/other/file/data +cd path/to/some/link +ln -s ../../other/file/data rel +ln -s "$MNT"/path/to/other/file/data abs + +printf '{"path":{"to":{"other":{"file":{"data":null}},"some":{"link":{}}}}}' >"$EXP" +pack -o "$OUT" -- "$MNT" || fail pack5 +diff "$EXP" "$OUT" || fail "test2 no-follow" + +printf '{"path":{"to":{"other":{"file":{"data":null}},"some":{"link":{"abs":null,"rel":null}}}}}' >"$EXP" +pack -o "$OUT" -L -- "$MNT" || fail pack6 +diff "$EXP" "$OUT" || fail "test2 follow" + +rm -r "$MNT" +mkdir "$MNT" + +# symlink pointing to ancestor error +# test3 +# └── path +# └── to +# ├── other +# │ └── file +# │ └── data +# └── some +# └── link +# └── linkfile -> ../../some + +cd "$MNT" +mkdir -p path/to/some/link path/to/other/file +touch path/to/other/file/data +cd path/to/some/link +ln -s ../../some linkfile + +printf '{"path":{"to":{"other":{"file":{"data":null}},"some":{"link":{}}}}}' >"$EXP" +pack -o "$OUT" -- "$MNT" || fail pack7 +diff "$EXP" "$OUT" || fail "test3 no-follow" + +pack -L -- "$MNT" >/dev/null 2>"$OUT" && fail "pack8 symlink to ancestor error" +cat "$OUT" | grep "ancestor directory" >/dev/null 2>&1 || fail "test3 follow expected error" + +rm -r "$MNT" +mkdir "$MNT" + +# symlink loop +# test4 +# ├── a -> b +# ├── b -> a +# ├── c -> b +# ├── d -> c +# ├── e -> d +# └── f -> e + +cd "$MNT" +ln -s a b +ln -s b c +ln -s c d +ln -s d e +ln -s e f +ln -s b a + +printf '{}' >"$EXP" +pack -o "$OUT" -- "$MNT" || fail pack9 +diff "$EXP" "$OUT" || fail "test4 no-follow" + +pack -L -- "$MNT" >/dev/null 2>"$OUT" && fail "pack10 symlink loop error" +cat "$OUT" | grep "Symlink loop detected" >/dev/null 2>&1 || fail "test4 follow expected error" + +if [ "$RUNNER_OS" = "macOS" ] || [ "$(uname)" = "Darwin" ]; then + rm -r "$MNT" + mkdir "$MNT" + + # xattr propagates up the symlink chain unless redefined + # setting xattr for symlinks in linux doesn't work + # test5 + # ├── a + # ├── b -> a + # ├── c -> b + # ├── d -> c + # ├── e -> d + # └── f -> e + + cd "$MNT" + echo '4' >a + ln -s a b + ln -s b c + ln -s c d + ln -s d e + ln -s e f + setattr user.type integer a + setattr user.type string c + setattr user.type bytes e + + printf '{"a":4}' >"$EXP" + pack -o "$OUT" -- "$MNT" || fail pack11 + diff "$EXP" "$OUT" || fail "test5 no-follow" + + printf '{"a":4,"b":4,"c":"4","d":"4","e":"NAo=","f":"NAo="}' >"$EXP" + pack -o "$OUT" -L -- "$MNT" || fail pack12 + diff "$EXP" "$OUT" || fail "test5 follow" +fi + +rm -r "$MNT" +mkdir "$MNT" + +# test for allowing symlink to escape packed directory +# test6 +# ├── a +# │ ├── a +# │ ├── b +# │ └── c -> ../b/c +# └── b +# └── c + +cd "$MNT" +mkdir a b +echo "a" >a/a +echo "b" >a/b +echo "c" >b/c +cd a +ln -s ../b/c c + +pack -L -- "$MNT"/a >/dev/null 2>"$OUT" || fail pack13 +cat "$OUT" | grep "Specify --allow-symlink-escape" >/dev/null 2>&1 || fail "test6 follow but no escape" + +printf '{"a":"a","b":"b","c":"c"}' >"$EXP" +pack -L --allow-symlink-escape -- "$MNT"/a >"$OUT" || fail pack14 +diff "$EXP" "$OUT" || fail "test6 follow and escape" + + +rm "$EXP" "$OUT" +rm -r "$MNT" diff --git a/tests/unpack_pack_toml_output.sh b/tests/unpack_pack_toml_output.sh new file mode 100755 index 0000000..dd7af8d --- /dev/null +++ b/tests/unpack_pack_toml_output.sh @@ -0,0 +1,43 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + rm "$TOML" + fi + exit 1 +} + +MNT=$(mktemp -d) +TOML=$(mktemp) + +mv "$TOML" "$TOML".toml +TOML="$TOML".toml + +cp ../toml/eg.toml "$TOML" + +unpack --into "$MNT" "$TOML" || fail unpack1 + +case $(ls "$MNT") in + (clients*database*owner*servers*title) ;; + (*) fail ls;; +esac +[ "$(cat $MNT/title)" = "TOML Example" ] || fail title +[ "$(cat $MNT/owner/dob)" = "1979-05-27T07:32:00-08:00" ] || fail dob +echo aleph >"$MNT/clients/hosts/2" +echo tav >"$MNT/clients/hosts/3" +pack -o "$TOML" "$MNT" || fail pack1 +rm -r "$MNT" + +unpack --into "$MNT" "$TOML" || fail unpack2 + +[ "$(cat $MNT/clients/hosts/0)" = "alpha" ] || fail hosts0 +[ "$(cat $MNT/clients/hosts/1)" = "omega" ] || fail hosts1 +[ "$(cat $MNT/clients/hosts/2)" = "aleph" ] || fail hosts2 +[ "$(cat $MNT/clients/hosts/3)" = "tav" ] || fail hosts3 + +pack "$MNT" || fail pack2 +rm -r "$MNT" || fail mount +rm "$TOML" diff --git a/tests/unpack_pack_toml_roundtrip.sh b/tests/unpack_pack_toml_roundtrip.sh new file mode 100755 index 0000000..8b42665 --- /dev/null +++ b/tests/unpack_pack_toml_roundtrip.sh @@ -0,0 +1,32 @@ +#/bin/sh + +fail() { + echo FAILED: $1 + rm -r "$UNPACK_MNT0" + rm -r "$UNPACK_MNT1" + rm "$PACK_FILE0" + rm "$PACK_FILE1" + rm "$ERR_MSG" + exit 1 +} + +ERR_MSG=$(mktemp) +for f in ../toml/*.toml; do + UNPACK_MNT0=$(mktemp -d) + unpack $f --into "$UNPACK_MNT0" 2>"$ERR_MSG" + # skip the issue where it doesn't unpack into a directory structure + cat "$ERR_MSG" | grep -i -e "the unpacked form must be a directory" >/dev/null 2>&1 && continue + PACK_FILE0=$(mktemp) + UNPACK_MNT1=$(mktemp -d) + PACK_FILE1=$(mktemp) + pack "$UNPACK_MNT0" -t toml >"$PACK_FILE0" || fail pack1 + unpack "$PACK_FILE0" -t toml --into "$UNPACK_MNT1" || fail unpack2 + pack "$UNPACK_MNT1" -t toml >"$PACK_FILE1" || fail pack2 + [ -z "$(diff $PACK_FILE0 $PACK_FILE1)" ] && [ -z "$(diff -r $UNPACK_MNT0 $UNPACK_MNT1)" ] || fail diff + rm -r "$UNPACK_MNT0" + rm -r "$UNPACK_MNT1" + rm "$PACK_FILE0" + rm "$PACK_FILE1" +done + +rm "$ERR_MSG" diff --git a/tests/unpack_pack_toml_to_json.sh b/tests/unpack_pack_toml_to_json.sh new file mode 100755 index 0000000..203d628 --- /dev/null +++ b/tests/unpack_pack_toml_to_json.sh @@ -0,0 +1,36 @@ +#/bin/sh + +# convert toml to json with unpack pack + +# unpack from format 1 +# pack to format 2 +# unpack from format 2 +# diff -r unpacked1 unpacked2 + +fail() { + echo FAILED: $1 + rm -r "$UNPACK_MNT0" + rm -r "$UNPACK_MNT1" + rm "$PACK_FILE0" + rm "$ERR_MSG" + exit 1 +} + +ERR_MSG=$(mktemp) +for f in $(find ../toml -maxdepth 1 -name '*.toml'); do + UNPACK_MNT0=$(mktemp -d) + # using `--exact` because datetime object becomes a string in json and adds a newline when unpacked as json. + unpack $f --exact --into "$UNPACK_MNT0" 2>"$ERR_MSG" + # skip the issue where it doesn't unpack into a directory structure + cat "$ERR_MSG" | grep -i -e "the unpacked form must be a directory" >/dev/null 2>&1 && continue + PACK_FILE0=$(mktemp) + UNPACK_MNT1=$(mktemp -d) + pack "$UNPACK_MNT0" --exact -t json >"$PACK_FILE0" || fail pack1 + unpack "$PACK_FILE0" --exact -t json --into "$UNPACK_MNT1" || fail unpack2 + [ -z "$(diff -r $UNPACK_MNT0 $UNPACK_MNT1)" ] || fail diff + rm -r "$UNPACK_MNT0" + rm -r "$UNPACK_MNT1" + rm "$PACK_FILE0" +done + +rm "$ERR_MSG" diff --git a/tests/unpack_pack_toml_to_yaml.sh b/tests/unpack_pack_toml_to_yaml.sh new file mode 100755 index 0000000..e5d2225 --- /dev/null +++ b/tests/unpack_pack_toml_to_yaml.sh @@ -0,0 +1,36 @@ +#/bin/sh + +# convert toml to yaml with unpack pack + +# unpack from format 1 +# pack to format 2 +# unpack from format 2 +# diff -r unpacked1 unpacked2 + +fail() { + echo FAILED: $1 + rm -r "$UNPACK_MNT0" + rm -r "$UNPACK_MNT1" + rm "$PACK_FILE0" + rm "$ERR_MSG" + exit 1 +} + +ERR_MSG=$(mktemp) +for f in $(find ../toml -maxdepth 1 -name '*.toml'); do + UNPACK_MNT0=$(mktemp -d) + # using `--exact` because datetime object becomes a string in json and adds a newline when unpacked as json. + unpack $f --exact --into "$UNPACK_MNT0" 2>"$ERR_MSG" + # skip the issue where it doesn't unpack into a directory structure + cat "$ERR_MSG" | grep -i -e "the unpacked form must be a directory" >/dev/null 2>&1 && continue + PACK_FILE0=$(mktemp) + UNPACK_MNT1=$(mktemp -d) + pack "$UNPACK_MNT0" --exact -t yaml >"$PACK_FILE0" || fail pack1 + unpack "$PACK_FILE0" --exact -t yaml --into "$UNPACK_MNT1" || fail unpack2 + [ -z "$(diff -r $UNPACK_MNT0 $UNPACK_MNT1)" ] || fail diff + rm -r "$UNPACK_MNT0" + rm -r "$UNPACK_MNT1" + rm "$PACK_FILE0" +done + +rm "$ERR_MSG" diff --git a/tests/unpack_pack_touch.sh b/tests/unpack_pack_touch.sh new file mode 100755 index 0000000..0d47b02 --- /dev/null +++ b/tests/unpack_pack_touch.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + rm "$ERR" + fi + exit 1 +} + +MNT=$(mktemp -d) +ERR=$(mktemp) + +unpack --into "$MNT" ../json/object.json || fail unpack + +touch "$MNT"/name 2>"$ERR" >&2 || { cat "$ERR"; fail touch; } +[ -s "$ERR" ] && { cat "$ERR"; fail error ; } + +pack "$MNT" || fail pack +rm -r "$MNT" || fail mount +rm "$ERR" + diff --git a/tests/unpack_pack_truncate.sh b/tests/unpack_pack_truncate.sh new file mode 100755 index 0000000..e5a610c --- /dev/null +++ b/tests/unpack_pack_truncate.sh @@ -0,0 +1,47 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + rm "$TGT" + rm "$TGT2" + rm "$ERR" + fi + exit 1 +} + +MNT=$(mktemp -d) +TGT=$(mktemp) +TGT2=$(mktemp) +ERR=$(mktemp) + +unpack --into "$MNT" ../json/object.json || fail unpack1 +echo 'Mikey Indiana' >"$MNT"/name 2>"$ERR" +[ -s "$ERR" ] && fail non-empty error +pack "$MNT" >"$TGT" || fail pack1 +rm -r "$MNT" + +# easiest to just test using ffs, but would be cool to get outside validation +[ -f "$TGT" ] || fail output1 +[ -s "$TGT" ] || fail output2 +grep -e Indiana "$TGT" >/dev/null 2>&1 || fail grep +unpack --type json --into "$MNT" "$TGT" || fail unpack2 + +case $(ls "$MNT") in + (eyes*fingernails*human*name) ;; + (*) fail ls;; +esac + +[ "$(cat $MNT/name)" = "Mikey Indiana" ] || fail contents + +pack --no-output "$MNT" >"$TGT2" || fail pack2 + +[ -f "$TGT2" ] || fail tgt2 +[ -s "$TGT2" ] && fail tgt2_nonempty + +rm -r "$MNT" || fail mount +rm "$TGT" +rm "$TGT2" +rm "$ERR" diff --git a/tests/unpack_pack_umask.sh b/tests/unpack_pack_umask.sh new file mode 100755 index 0000000..f4bc778 --- /dev/null +++ b/tests/unpack_pack_umask.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + fi + exit 1 +} + +MNT=$(mktemp -d) + +umask 022 +unpack --into "$MNT" ../json/object.json || fail unpack1 +cd "$MNT" +ls -l eyes | grep -e 'rw-r--r--' >/dev/null 2>&1 || fail file1 +mkdir pockets +ls -ld pockets | grep -e 'rwxr-xr-x' >/dev/null 2>&1 || fail dir1 +cd - >/dev/null 2>&1 +pack "$MNT" || fail pack1 +rm -r "$MNT" + +umask 077 +unpack --into "$MNT" ../json/object.json || fail unpack2 +cd "$MNT" +ls -l eyes | grep -e 'rw-------' >/dev/null 2>&1 || fail file2 +mkdir pockets +ls -ld pockets | grep -e 'rwx------' >/dev/null 2>&1 || fail dir2 +cd - >/dev/null 2>&1 +pack "$MNT" || fail pack2 + +rm -r "$MNT" || fail mount diff --git a/tests/unpack_pack_unlink.sh b/tests/unpack_pack_unlink.sh new file mode 100755 index 0000000..1bf4325 --- /dev/null +++ b/tests/unpack_pack_unlink.sh @@ -0,0 +1,39 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + fi + exit 1 +} + +MNT=$(mktemp -d) + +unpack --into "$MNT" ../json/object.json || fail unpack + +cd "$MNT" +case $(ls) in + (eyes*fingernails*human*name) ;; + (*) fail ls;; +esac +[ "$(cat name)" = "Michael Greenberg" ] || fail name +[ "$(cat eyes)" -eq 2 ] || fail eyes +[ "$(cat fingernails)" -eq 10 ] || fail fingernails +[ "$(cat human)" = "true" ] || fail human1 +rm human +case $(ls) in + (eyes*fingernails*name) ;; + (*) fail ls2;; +esac +echo false >human +case $(ls) in + (eyes*fingernails*human*name) ;; + (*) fail ls3;; +esac +[ "$(cat human)" = "false" ] || fail human2 +cd - >/dev/null 2>&1 + +pack "$MNT" || fail pack +rm -r "$MNT" || fail mount diff --git a/tests/unpack_pack_unpadded_list.sh b/tests/unpack_pack_unpadded_list.sh new file mode 100755 index 0000000..78105b3 --- /dev/null +++ b/tests/unpack_pack_unpadded_list.sh @@ -0,0 +1,34 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + fi + exit 1 +} + +MNT=$(mktemp -d) + +unpack --unpadded --into "$MNT" ../json/list2.json || fail unpack +cd "$MNT" +case $(ls) in + (0*1*10*2*3*4*5*6*7*8*9) ;; + (*) fail ls;; +esac +[ "$(cat 0)" -eq 0 ] || fail 0 +[ "$(cat 1)" -eq 1 ] || fail 1 +[ "$(cat 2)" -eq 2 ] || fail 2 +[ "$(cat 3)" -eq 3 ] || fail 3 +[ "$(cat 4)" -eq 4 ] || fail 4 +[ "$(cat 5)" -eq 5 ] || fail 5 +[ "$(cat 6)" -eq 6 ] || fail 6 +[ "$(cat 7)" -eq 7 ] || fail 7 +[ "$(cat 8)" -eq 8 ] || fail 8 +[ "$(cat 9)" -eq 9 ] || fail 9 +[ "$(cat 10)" -eq 10 ] || fail 10 +cd - >/dev/null 2>&1 + +pack "$MNT" || fail pack +rm -r "$MNT" || fail mount diff --git a/tests/unpack_pack_write.sh b/tests/unpack_pack_write.sh new file mode 100755 index 0000000..6c2de3e --- /dev/null +++ b/tests/unpack_pack_write.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + rm -rf "$EXP" + fi + exit 1 +} + +MNT=$(mktemp -d) +EXP=$(mktemp -d) + +cat >"${EXP}/4" <4 +[ $(cat 4) = "hi" ] || fail write1 +echo hello >>4 +diff 4 "${EXP}/4" || fail write2 +cd - >/dev/null 2>&1 +pack "$MNT" || fail pack + +rm -r "$MNT" || fail mount +rm -rf "$EXP" + diff --git a/tests/unpack_pack_yaml_output.sh b/tests/unpack_pack_yaml_output.sh new file mode 100755 index 0000000..35b101a --- /dev/null +++ b/tests/unpack_pack_yaml_output.sh @@ -0,0 +1,41 @@ +#!/bin/sh + +fail() { + echo FAILED: $1 + if [ "$MNT" ] + then + rm -r "$MNT" + rm "$YAML" + fi + exit 1 +} + +MNT=$(mktemp -d) +YAML=$(mktemp) + +mv "$YAML" "$YAML".yaml +YAML="$YAML".yaml + +cp ../yaml/invoice.yaml "$YAML" + +unpack --into "$MNT" "$YAML" || fail unpack1 +case $(ls "$MNT") in + (bill-to*comments*date*invoice*product*ship-to*tax*total) ;; + (*) fail ls;; +esac +[ "$(cat $MNT/date)" = "2001-01-23" ] || fail date +[ "$(cat $MNT/product/0/description)" = "Basketball" ] || fail product +echo orange >"$MNT/product/0/color" +echo pink >"$MNT/product/1/color" +pack -o "$YAML" "$MNT" || fail pack1 +rm -r "$MNT" + +unpack --into "$MNT" "$YAML" || fail unpack2 +[ "$(cat $MNT/product/0/description)" = "Basketball" ] || fail desc1 +[ "$(cat $MNT/product/0/color)" = "orange" ] || fail color1 +[ "$(cat $MNT/product/1/description)" = "Super Hoop" ] || fail desc2 +[ "$(cat $MNT/product/1/color)" = "pink" ] || fail color2 + +pack "$MNT" || fail pack2 +rm -r "$MNT" || fail mount +rm "$YAML" diff --git a/tests/unpack_pack_yaml_roundtrip.sh b/tests/unpack_pack_yaml_roundtrip.sh new file mode 100755 index 0000000..45cc1a8 --- /dev/null +++ b/tests/unpack_pack_yaml_roundtrip.sh @@ -0,0 +1,32 @@ +#/bin/sh + +fail() { + echo FAILED: $1 + rm -r "$UNPACK_MNT0" + rm -r "$UNPACK_MNT1" + rm "$PACK_FILE0" + rm "$PACK_FILE1" + rm "$ERR_MSG" + exit 1 +} + +ERR_MSG=$(mktemp) +for f in ../yaml/*.yaml; do + UNPACK_MNT0=$(mktemp -d) + unpack $f --into "$UNPACK_MNT0" 2>"$ERR_MSG" + # skip the issue where it doesn't unpack into a directory structure + cat "$ERR_MSG" | grep -i -e "the unpacked form must be a directory" >/dev/null 2>&1 && continue + PACK_FILE0=$(mktemp) + UNPACK_MNT1=$(mktemp -d) + PACK_FILE1=$(mktemp) + pack "$UNPACK_MNT0" -t yaml >"$PACK_FILE0" || fail pack1 + unpack "$PACK_FILE0" -t yaml --into "$UNPACK_MNT1" || fail unpack2 + pack "$UNPACK_MNT1" -t yaml >"$PACK_FILE1" || fail pack2 + [ -z "$(diff $PACK_FILE0 $PACK_FILE1)" ] && [ -z "$(diff -r $UNPACK_MNT0 $UNPACK_MNT1)" ] || fail diff + rm -r "$UNPACK_MNT0" + rm -r "$UNPACK_MNT1" + rm "$PACK_FILE0" + rm "$PACK_FILE1" +done + +rm "$ERR_MSG" diff --git a/tests/unpack_pack_yaml_to_json.sh b/tests/unpack_pack_yaml_to_json.sh new file mode 100755 index 0000000..d0a2691 --- /dev/null +++ b/tests/unpack_pack_yaml_to_json.sh @@ -0,0 +1,37 @@ +#/bin/sh + +# convert yaml to json with unpack pack + +# unpack from format 1 +# pack to format 2 +# unpack from format 2 +# diff -r unpacked1 unpacked2 + +fail() { + echo FAILED: $1 + rm -r "$UNPACK_MNT0" + rm -r "$UNPACK_MNT1" + rm "$PACK_FILE0" + rm "$ERR_MSG" + exit 1 +} + +ERR_MSG=$(mktemp) +# reasons for skipping: +# invoice.yaml's floats with .00 become .0 in json. Everything else is perfect after unpacking the packed json version +for f in $(find ../yaml -maxdepth 1 -name '*.yaml' ! -name 'invoice.yaml'); do + UNPACK_MNT0=$(mktemp -d) + unpack $f --into "$UNPACK_MNT0" 2>"$ERR_MSG" + # skip the issue where it doesn't unpack into a directory structure + cat "$ERR_MSG" | grep -i -e "the unpacked form must be a directory" >/dev/null 2>&1 && continue + PACK_FILE0=$(mktemp) + UNPACK_MNT1=$(mktemp -d) + pack "$UNPACK_MNT0" -t json >"$PACK_FILE0" || fail pack1 + unpack "$PACK_FILE0" -t json --into "$UNPACK_MNT1" || fail unpack2 + [ -z "$(diff -r $UNPACK_MNT0 $UNPACK_MNT1)" ] || fail diff + rm -r "$UNPACK_MNT0" + rm -r "$UNPACK_MNT1" + rm "$PACK_FILE0" +done + +rm "$ERR_MSG" diff --git a/tests/unpack_pack_yaml_to_toml.sh b/tests/unpack_pack_yaml_to_toml.sh new file mode 100755 index 0000000..ff20348 --- /dev/null +++ b/tests/unpack_pack_yaml_to_toml.sh @@ -0,0 +1,38 @@ +#/bin/sh + +# convert yaml to toml with unpack pack + +# unpack from format 1 +# pack to format 2 +# unpack from format 2 +# diff -r unpacked1 unpacked2 + +fail() { + echo FAILED: $1 + rm -r "$UNPACK_MNT0" + rm -r "$UNPACK_MNT1" + rm "$PACK_FILE0" + rm "$ERR_MSG" + exit 1 +} + +ERR_MSG=$(mktemp) +# reasons for skipping: +# eg2.7.yaml unpacks into lists in json format because toml doesn't support it. +# invoice.yaml's floats get their decimals truncated. Everything else is perfect after unpacking the packed toml version +for f in $(find ../yaml -maxdepth 1 -name '*.yaml' ! -name 'eg2.7.yaml' ! -name 'invoice.yaml'); do + UNPACK_MNT0=$(mktemp -d) + unpack $f --exact --into "$UNPACK_MNT0" 2>"$ERR_MSG" + # skip the issue where it doesn't unpack into a directory structure + cat "$ERR_MSG" | grep -i -e "the unpacked form must be a directory" >/dev/null 2>&1 && continue + PACK_FILE0=$(mktemp) + UNPACK_MNT1=$(mktemp -d) + pack "$UNPACK_MNT0" --exact -t toml >"$PACK_FILE0" || fail pack1 + unpack "$PACK_FILE0" --exact -t toml --into "$UNPACK_MNT1" || fail unpack2 + [ -z "$(diff -r $UNPACK_MNT0 $UNPACK_MNT1)" ] || fail diff + rm -r "$UNPACK_MNT0" + rm -r "$UNPACK_MNT1" + rm "$PACK_FILE0" +done + +rm "$ERR_MSG"