mirror of
https://github.com/mgree/ffs.git
synced 2024-10-03 22:07:18 +03:00
Lazy loading (#50)
Lazy implementation is now the default. Use `--eager` to force ffs to construct the entire filesystem on startup. NB that lazy loading is not the same as lazy parsing. There's still plenty of savings left on the table. There is some unwelcome code duplication in saving to accommodate type-level jiggery pokery.
This commit is contained in:
Michael Greenberg
GitHub
parent
35f6bf8188
commit
b9c6644312
5
.github/workflows/build.yml
vendored
5
.github/workflows/build.yml
vendored
@ -47,8 +47,11 @@ jobs:
|
||||
run: |
|
||||
Rscript -e "install.packages('ggplot2', repos = 'https://cloud.r-project.org/')"
|
||||
PATH="$(pwd)/target/release:$PATH" ./run_bench.sh -n 3
|
||||
# grab latest directory (output of run_bench)
|
||||
DATADIR=bench/$(ls -ct bench/ | head -n 1)
|
||||
[ -d $DATADIR ] && ls $DATADIR | grep log >/dev/null || { echo "No log files found in $DATADIR. What's going on?"; tree bench; exit 1; }
|
||||
mkdir data
|
||||
for x in *.log *.png
|
||||
for x in $DATADIR/*
|
||||
do
|
||||
mv $x data/${x##*_}
|
||||
done
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
2281
bench/20210729_plos/bench.log
Normal file
2281
bench/20210729_plos/bench.log
Normal file
File diff suppressed because it is too large
Load Diff
BIN
bench/20210729_plos/bench.png
Normal file
BIN
bench/20210729_plos/bench.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 130 KiB |
1281
bench/20210729_plos/micro.log
Normal file
1281
bench/20210729_plos/micro.log
Normal file
File diff suppressed because it is too large
Load Diff
BIN
bench/20210729_plos/micro.png
Normal file
BIN
bench/20210729_plos/micro.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 226 KiB |
0
bench/20210929_lazy_vs_eager/README.md
Normal file
0
bench/20210929_lazy_vs_eager/README.md
Normal file
229
bench/20210929_lazy_vs_eager/eager_bench.log
Normal file
229
bench/20210929_lazy_vs_eager/eager_bench.log
Normal file
@ -0,0 +1,229 @@
|
||||
source,file,run,size,activity,ns
|
||||
gh,gh_licenses.json,1,2507,reading,4756731
|
||||
gh,gh_licenses.json,1,2507,loading,90969
|
||||
gh,gh_licenses.json,1,2507,saving,92397
|
||||
gh,gh_licenses.json,1,2507,writing,3439452
|
||||
gh,gh_events.json,1,54778,reading,86802412
|
||||
gh,gh_events.json,1,54778,loading,968594
|
||||
gh,gh_events.json,1,54778,saving,709096
|
||||
gh,gh_events.json,1,54778,writing,48779025
|
||||
json.org,json_eg3.json,1,602,reading,1329089
|
||||
json.org,json_eg3.json,1,602,loading,43664
|
||||
json.org,json_eg3.json,1,602,saving,36970
|
||||
json.org,json_eg3.json,1,602,writing,992639
|
||||
synthetic,list2.json,1,25,reading,909360
|
||||
synthetic,list2.json,1,25,loading,68594
|
||||
synthetic,list2.json,1,25,saving,20209
|
||||
synthetic,list2.json,1,25,writing,438095
|
||||
penn,historic-20181028.json,1,1468638,reading,2112686305
|
||||
penn,historic-20181028.json,1,1468638,loading,43091415
|
||||
penn,historic-20181028.json,1,1468638,saving,28111326
|
||||
penn,historic-20181028.json,1,1468638,writing,1753729157
|
||||
fda,waterpipe.json,1,1955,reading,2879997
|
||||
fda,waterpipe.json,1,1955,loading,84168
|
||||
fda,waterpipe.json,1,1955,saving,46846
|
||||
fda,waterpipe.json,1,1955,writing,1570303
|
||||
fda,tobacco100.json,1,63634,reading,93001838
|
||||
fda,tobacco100.json,1,63634,loading,1235534
|
||||
fda,tobacco100.json,1,63634,saving,691776
|
||||
fda,tobacco100.json,1,63634,writing,62262959
|
||||
penguin,dekeijzer.json,1,446,reading,1559848
|
||||
penguin,dekeijzer.json,1,446,loading,38312
|
||||
penguin,dekeijzer.json,1,446,saving,28728
|
||||
penguin,dekeijzer.json,1,446,writing,761622
|
||||
ncdc,ncdc_ewr_station.json,1,191,reading,1183948
|
||||
ncdc,ncdc_ewr_station.json,1,191,loading,37215
|
||||
ncdc,ncdc_ewr_station.json,1,191,saving,24209
|
||||
ncdc,ncdc_ewr_station.json,1,191,writing,496614
|
||||
gh,gh_apache2.json,1,12591,reading,19898816
|
||||
gh,gh_apache2.json,1,12591,loading,67766
|
||||
gh,gh_apache2.json,1,12591,saving,45535
|
||||
gh,gh_apache2.json,1,12591,writing,4651799
|
||||
fda,recall100.json,1,2117776,reading,3042598032
|
||||
fda,recall100.json,1,2117776,loading,60204079
|
||||
fda,recall100.json,1,2117776,saving,26203122
|
||||
fda,recall100.json,1,2117776,writing,2334859613
|
||||
penguin,linux.json,1,64401,reading,94101084
|
||||
penguin,linux.json,1,64401,loading,815883
|
||||
penguin,linux.json,1,64401,saving,567179
|
||||
penguin,linux.json,1,64401,writing,54275806
|
||||
gov.uk,bank-holidays.json,1,15124,reading,26542317
|
||||
gov.uk,bank-holidays.json,1,15124,loading,777081
|
||||
gov.uk,bank-holidays.json,1,15124,saving,448455
|
||||
gov.uk,bank-holidays.json,1,15124,writing,35643144
|
||||
fda,recall.json,1,22475,reading,34219366
|
||||
fda,recall.json,1,22475,loading,630647
|
||||
fda,recall.json,1,22475,saving,299473
|
||||
fda,recall.json,1,22475,writing,22035588
|
||||
json.org,json_eg2.json,1,242,reading,602274
|
||||
json.org,json_eg2.json,1,242,loading,60600
|
||||
json.org,json_eg2.json,1,242,saving,29190
|
||||
json.org,json_eg2.json,1,242,writing,739912
|
||||
ncdc,ncdc_07030_stations.json,1,1476,reading,2307805
|
||||
ncdc,ncdc_07030_stations.json,1,1476,loading,101051
|
||||
ncdc,ncdc_07030_stations.json,1,1476,saving,87151
|
||||
ncdc,ncdc_07030_stations.json,1,1476,writing,2872191
|
||||
ncdc,ncdc_zips.json,1,2851,reading,5267275
|
||||
ncdc,ncdc_zips.json,1,2851,loading,159818
|
||||
ncdc,ncdc_zips.json,1,2851,saving,103347
|
||||
ncdc,ncdc_zips.json,1,2851,writing,7219574
|
||||
rv,animal.json,1,7051,reading,11824778
|
||||
rv,animal.json,1,7051,loading,225834
|
||||
rv,animal.json,1,7051,saving,152135
|
||||
rv,animal.json,1,7051,writing,11140226
|
||||
json.org,json_eg1.json,1,583,reading,1100477
|
||||
json.org,json_eg1.json,1,583,loading,63616
|
||||
json.org,json_eg1.json,1,583,saving,32175
|
||||
json.org,json_eg1.json,1,583,writing,936669
|
||||
gh,mgree.json,1,1331,reading,2100567
|
||||
gh,mgree.json,1,1331,loading,47039
|
||||
gh,mgree.json,1,1331,saving,49455
|
||||
gh,mgree.json,1,1331,writing,1419751
|
||||
synthetic,object_null.json,1,95,reading,983035
|
||||
synthetic,object_null.json,1,95,loading,32155
|
||||
synthetic,object_null.json,1,95,saving,20586
|
||||
synthetic,object_null.json,1,95,writing,305246
|
||||
gh,gh_mgree_events.json,1,168832,reading,275532529
|
||||
gh,gh_mgree_events.json,1,168832,loading,1992652
|
||||
gh,gh_mgree_events.json,1,168832,saving,3098900
|
||||
gh,gh_mgree_events.json,1,168832,writing,195248107
|
||||
gh,gh_gpl3.json,1,36979,reading,56442782
|
||||
gh,gh_gpl3.json,1,36979,loading,70261
|
||||
gh,gh_gpl3.json,1,36979,saving,64687
|
||||
gh,gh_gpl3.json,1,36979,writing,9488543
|
||||
ncdc,ncdc_07030.json,1,111,reading,477245
|
||||
ncdc,ncdc_07030.json,1,111,loading,58723
|
||||
ncdc,ncdc_07030.json,1,111,saving,23973
|
||||
ncdc,ncdc_07030.json,1,111,writing,401189
|
||||
gh,gh_emoji.json,1,173641,reading,250919699
|
||||
gh,gh_emoji.json,1,173641,loading,1390594
|
||||
gh,gh_emoji.json,1,173641,saving,1746930
|
||||
gh,gh_emoji.json,1,173641,writing,92169271
|
||||
fda,recall10.json,1,212061,reading,344150239
|
||||
fda,recall10.json,1,212061,loading,5445647
|
||||
fda,recall10.json,1,212061,saving,2504900
|
||||
fda,recall10.json,1,212061,writing,215005273
|
||||
ncdc,ncdc_locationcategories.json,1,543,reading,2142873
|
||||
ncdc,ncdc_locationcategories.json,1,543,loading,96924
|
||||
ncdc,ncdc_locationcategories.json,1,543,saving,49473
|
||||
ncdc,ncdc_locationcategories.json,1,543,writing,1414060
|
||||
fda,registrationlisting.json,1,3228,reading,5320395
|
||||
fda,registrationlisting.json,1,3228,loading,110878
|
||||
fda,registrationlisting.json,1,3228,saving,91720
|
||||
fda,registrationlisting.json,1,3228,writing,3239285
|
||||
gh,ffs_issues.json,1,38158,reading,59715631
|
||||
gh,ffs_issues.json,1,38158,loading,789139
|
||||
gh,ffs_issues.json,1,38158,saving,717549
|
||||
gh,ffs_issues.json,1,38158,writing,35136405
|
||||
fda,event.json,1,1819,reading,2891121
|
||||
fda,event.json,1,1819,loading,84044
|
||||
fda,event.json,1,1819,saving,68000
|
||||
fda,event.json,1,1819,writing,1721457
|
||||
ncdc,ncdc_datasets.json,1,1625,reading,3599152
|
||||
ncdc,ncdc_datasets.json,1,1625,loading,186129
|
||||
ncdc,ncdc_datasets.json,1,1625,saving,68806
|
||||
ncdc,ncdc_datasets.json,1,1625,writing,3140412
|
||||
ncdc,ncdc_locations.json,1,2808,reading,6079131
|
||||
ncdc,ncdc_locations.json,1,2808,loading,219400
|
||||
ncdc,ncdc_locations.json,1,2808,saving,97780
|
||||
ncdc,ncdc_locations.json,1,2808,writing,5833940
|
||||
ncdc,ncdc_datacategories.json,1,1075,reading,2921852
|
||||
ncdc,ncdc_datacategories.json,1,1075,loading,106961
|
||||
ncdc,ncdc_datacategories.json,1,1075,saving,79414
|
||||
ncdc,ncdc_datacategories.json,1,1075,writing,3319340
|
||||
gh,mgree_repos.json,1,159905,reading,243081966
|
||||
gh,mgree_repos.json,1,159905,loading,2259708
|
||||
gh,mgree_repos.json,1,159905,saving,2323660
|
||||
gh,mgree_repos.json,1,159905,writing,132488363
|
||||
ncdc,ncdc_stations_big.json,1,200908,reading,301696497
|
||||
ncdc,ncdc_stations_big.json,1,200908,loading,7510908
|
||||
ncdc,ncdc_stations_big.json,1,200908,saving,4909671
|
||||
ncdc,ncdc_stations_big.json,1,200908,writing,425843149
|
||||
doi,doi_smoosh.json,1,537,reading,1751755
|
||||
doi,doi_smoosh.json,1,537,loading,65047
|
||||
doi,doi_smoosh.json,1,537,saving,46539
|
||||
doi,doi_smoosh.json,1,537,writing,1470142
|
||||
gh,ffs.json,1,5089,reading,9314126
|
||||
gh,ffs.json,1,5089,loading,122060
|
||||
gh,ffs.json,1,5089,saving,92831
|
||||
gh,ffs.json,1,5089,writing,5349474
|
||||
synthetic,object.json,1,77,reading,357792
|
||||
synthetic,object.json,1,77,loading,32961
|
||||
synthetic,object.json,1,77,saving,22649
|
||||
synthetic,object.json,1,77,writing,286831
|
||||
json.org,json_eg4.json,1,3468,reading,5924034
|
||||
json.org,json_eg4.json,1,3468,loading,130810
|
||||
json.org,json_eg4.json,1,3468,saving,151546
|
||||
json.org,json_eg4.json,1,3468,writing,3494471
|
||||
fda,tobacco_products.json,1,2345,reading,4310854
|
||||
fda,tobacco_products.json,1,2345,loading,136820
|
||||
fda,tobacco_products.json,1,2345,saving,59439
|
||||
fda,tobacco_products.json,1,2345,writing,3160259
|
||||
fda,tobacco10.json,1,6567,reading,13855670
|
||||
fda,tobacco10.json,1,6567,loading,280783
|
||||
fda,tobacco10.json,1,6567,saving,93980
|
||||
fda,tobacco10.json,1,6567,writing,5879484
|
||||
penguin,bsd.json,1,5415,reading,8631145
|
||||
penguin,bsd.json,1,5415,loading,108937
|
||||
penguin,bsd.json,1,5415,saving,78942
|
||||
penguin,bsd.json,1,5415,writing,4424858
|
||||
fda,event100.json,1,2945177,reading,4316049924
|
||||
fda,event100.json,1,2945177,loading,50924812
|
||||
fda,event100.json,1,2945177,saving,29717054
|
||||
fda,event100.json,1,2945177,writing,2158000040
|
||||
gh,kmt.json,1,4979,reading,12388129
|
||||
gh,kmt.json,1,4979,loading,128596
|
||||
gh,kmt.json,1,4979,saving,103578
|
||||
gh,kmt.json,1,4979,writing,6333481
|
||||
fda,tobacco.json,1,1144,reading,2604235
|
||||
fda,tobacco.json,1,1144,loading,98984
|
||||
fda,tobacco.json,1,1144,saving,39349
|
||||
fda,tobacco.json,1,1144,writing,1123780
|
||||
doi,doi_10.1000_1.json,1,369,reading,1651032
|
||||
doi,doi_10.1000_1.json,1,369,loading,60114
|
||||
doi,doi_10.1000_1.json,1,369,saving,52285
|
||||
doi,doi_10.1000_1.json,1,369,writing,1040857
|
||||
rv,human_male.json,1,603912,reading,880247537
|
||||
rv,human_male.json,1,603912,loading,14287478
|
||||
rv,human_male.json,1,603912,saving,10493764
|
||||
rv,human_male.json,1,603912,writing,1019649921
|
||||
rv,demons.json,1,194,reading,1281286
|
||||
rv,demons.json,1,194,loading,63289
|
||||
rv,demons.json,1,194,saving,25102
|
||||
rv,demons.json,1,194,writing,488744
|
||||
penguin,rust.json,1,161726,reading,241219038
|
||||
penguin,rust.json,1,161726,loading,749010
|
||||
penguin,rust.json,1,161726,saving,878072
|
||||
penguin,rust.json,1,161726,writing,40401368
|
||||
synthetic,obj_rename.json,1,69,reading,404152
|
||||
synthetic,obj_rename.json,1,69,loading,27378
|
||||
synthetic,obj_rename.json,1,69,saving,14246
|
||||
synthetic,obj_rename.json,1,69,writing,314361
|
||||
ncdc,ncdc_gsom_sample.json,1,2867,reading,5479014
|
||||
ncdc,ncdc_gsom_sample.json,1,2867,loading,139064
|
||||
ncdc,ncdc_gsom_sample.json,1,2867,saving,96473
|
||||
ncdc,ncdc_gsom_sample.json,1,2867,writing,6487512
|
||||
fda,registrationlisting100.json,1,214459,reading,326351594
|
||||
fda,registrationlisting100.json,1,214459,loading,4180987
|
||||
fda,registrationlisting100.json,1,214459,saving,3146411
|
||||
fda,registrationlisting100.json,1,214459,writing,254849442
|
||||
ncdc,ncdc_datatypes.json,1,4013,reading,6168447
|
||||
ncdc,ncdc_datatypes.json,1,4013,loading,169882
|
||||
ncdc,ncdc_datatypes.json,1,4013,saving,97933
|
||||
ncdc,ncdc_datatypes.json,1,4013,writing,6266393
|
||||
fda,recall50.json,1,1049501,reading,1516850010
|
||||
fda,recall50.json,1,1049501,loading,24478380
|
||||
fda,recall50.json,1,1049501,saving,12893951
|
||||
fda,recall50.json,1,1049501,writing,1109863656
|
||||
rv,agni.json,1,89267,reading,134850463
|
||||
rv,agni.json,1,89267,loading,2081502
|
||||
rv,agni.json,1,89267,saving,1530194
|
||||
rv,agni.json,1,89267,writing,145347639
|
||||
json.org,json_eg5.json,1,873,reading,2166091
|
||||
json.org,json_eg5.json,1,873,loading,80344
|
||||
json.org,json_eg5.json,1,873,saving,69325
|
||||
json.org,json_eg5.json,1,873,writing,2469190
|
||||
penguin,grisham.json,1,22488,reading,35736375
|
||||
penguin,grisham.json,1,22488,loading,1061233
|
||||
penguin,grisham.json,1,22488,saving,778041
|
||||
penguin,grisham.json,1,22488,writing,52360282
|
||||
BIN
bench/20210929_lazy_vs_eager/eager_bench.png
Normal file
BIN
bench/20210929_lazy_vs_eager/eager_bench.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 125 KiB |
129
bench/20210929_lazy_vs_eager/eager_micro.log
Normal file
129
bench/20210929_lazy_vs_eager/eager_micro.log
Normal file
@ -0,0 +1,129 @@
|
||||
source,file,kind,direction,magnitude,run,size,activity,ns
|
||||
micro,list_wide_128.json,list,wide,128,1,642,reading,1718515
|
||||
micro,list_wide_128.json,list,wide,128,1,642,loading,77333
|
||||
micro,list_wide_128.json,list,wide,128,1,642,saving,47883
|
||||
micro,list_wide_128.json,list,wide,128,1,642,writing,1604834
|
||||
micro,list_wide_32.json,list,wide,32,1,162,reading,484337
|
||||
micro,list_wide_32.json,list,wide,32,1,162,loading,65330
|
||||
micro,list_wide_32.json,list,wide,32,1,162,saving,41235
|
||||
micro,list_wide_32.json,list,wide,32,1,162,writing,549725
|
||||
micro,named_wide_64.json,named,wide,64,1,616,reading,1153006
|
||||
micro,named_wide_64.json,named,wide,64,1,616,loading,148303
|
||||
micro,named_wide_64.json,named,wide,64,1,616,saving,59715
|
||||
micro,named_wide_64.json,named,wide,64,1,616,writing,2546371
|
||||
micro,list_wide_4.json,list,wide,4,1,22,reading,242106
|
||||
micro,list_wide_4.json,list,wide,4,1,22,loading,46944
|
||||
micro,list_wide_4.json,list,wide,4,1,22,saving,13062
|
||||
micro,list_wide_4.json,list,wide,4,1,22,writing,179866
|
||||
micro,named_wide_256.json,named,wide,256,1,2536,reading,4614723
|
||||
micro,named_wide_256.json,named,wide,256,1,2536,loading,92342
|
||||
micro,named_wide_256.json,named,wide,256,1,2536,saving,179514
|
||||
micro,named_wide_256.json,named,wide,256,1,2536,writing,9419829
|
||||
micro,list_wide_1.json,list,wide,1,1,7,reading,886489
|
||||
micro,list_wide_1.json,list,wide,1,1,7,loading,34993
|
||||
micro,list_wide_1.json,list,wide,1,1,7,saving,12737
|
||||
micro,list_wide_1.json,list,wide,1,1,7,writing,143948
|
||||
micro,list_deep_32.json,list,deep,32,1,69,reading,332617
|
||||
micro,list_deep_32.json,list,deep,32,1,69,loading,66001
|
||||
micro,list_deep_32.json,list,deep,32,1,69,saving,38501
|
||||
micro,list_deep_32.json,list,deep,32,1,69,writing,571553
|
||||
micro,list_deep_4.json,list,deep,4,1,13,reading,899341
|
||||
micro,list_deep_4.json,list,deep,4,1,13,loading,43863
|
||||
micro,list_deep_4.json,list,deep,4,1,13,saving,14096
|
||||
micro,list_deep_4.json,list,deep,4,1,13,writing,188044
|
||||
micro,list_deep_16.json,list,deep,16,1,37,reading,899142
|
||||
micro,list_deep_16.json,list,deep,16,1,37,loading,56572
|
||||
micro,list_deep_16.json,list,deep,16,1,37,saving,21729
|
||||
micro,list_deep_16.json,list,deep,16,1,37,writing,316733
|
||||
micro,named_deep_4.json,named,deep,4,1,29,reading,983518
|
||||
micro,named_deep_4.json,named,deep,4,1,29,loading,43100
|
||||
micro,named_deep_4.json,named,deep,4,1,29,saving,21362
|
||||
micro,named_deep_4.json,named,deep,4,1,29,writing,282837
|
||||
micro,list_wide_2.json,list,wide,2,1,12,reading,229145
|
||||
micro,list_wide_2.json,list,wide,2,1,12,loading,53958
|
||||
micro,list_wide_2.json,list,wide,2,1,12,saving,20323
|
||||
micro,list_wide_2.json,list,wide,2,1,12,writing,171158
|
||||
micro,named_wide_8.json,named,wide,8,1,74,reading,963323
|
||||
micro,named_wide_8.json,named,wide,8,1,74,loading,35867
|
||||
micro,named_wide_8.json,named,wide,8,1,74,saving,17940
|
||||
micro,named_wide_8.json,named,wide,8,1,74,writing,451811
|
||||
micro,named_wide_128.json,named,wide,128,1,1256,reading,2104583
|
||||
micro,named_wide_128.json,named,wide,128,1,1256,loading,87556
|
||||
micro,named_wide_128.json,named,wide,128,1,1256,saving,88183
|
||||
micro,named_wide_128.json,named,wide,128,1,1256,writing,4587826
|
||||
micro,named_deep_1.json,named,deep,1,1,11,reading,891433
|
||||
micro,named_deep_1.json,named,deep,1,1,11,loading,30899
|
||||
micro,named_deep_1.json,named,deep,1,1,11,saving,12695
|
||||
micro,named_deep_1.json,named,deep,1,1,11,writing,172799
|
||||
micro,list_wide_64.json,list,wide,64,1,322,reading,1349280
|
||||
micro,list_wide_64.json,list,wide,64,1,322,loading,78534
|
||||
micro,list_wide_64.json,list,wide,64,1,322,saving,27724
|
||||
micro,list_wide_64.json,list,wide,64,1,322,writing,917649
|
||||
micro,named_wide_32.json,named,wide,32,1,296,reading,1360586
|
||||
micro,named_wide_32.json,named,wide,32,1,296,loading,38547
|
||||
micro,named_wide_32.json,named,wide,32,1,296,saving,40241
|
||||
micro,named_wide_32.json,named,wide,32,1,296,writing,1330957
|
||||
micro,named_deep_32.json,named,deep,32,1,203,reading,595340
|
||||
micro,named_deep_32.json,named,deep,32,1,203,loading,43597
|
||||
micro,named_deep_32.json,named,deep,32,1,203,saving,49755
|
||||
micro,named_deep_32.json,named,deep,32,1,203,writing,1526321
|
||||
micro,list_wide_16.json,list,wide,16,1,82,reading,314794
|
||||
micro,list_wide_16.json,list,wide,16,1,82,loading,50339
|
||||
micro,list_wide_16.json,list,wide,16,1,82,saving,15307
|
||||
micro,list_wide_16.json,list,wide,16,1,82,writing,301792
|
||||
micro,named_deep_8.json,named,deep,8,1,53,reading,939073
|
||||
micro,named_deep_8.json,named,deep,8,1,53,loading,31317
|
||||
micro,named_deep_8.json,named,deep,8,1,53,saving,27605
|
||||
micro,named_deep_8.json,named,deep,8,1,53,writing,419893
|
||||
micro,named_deep_16.json,named,deep,16,1,101,reading,1068042
|
||||
micro,named_deep_16.json,named,deep,16,1,101,loading,28966
|
||||
micro,named_deep_16.json,named,deep,16,1,101,saving,34335
|
||||
micro,named_deep_16.json,named,deep,16,1,101,writing,683679
|
||||
micro,named_wide_16.json,named,wide,16,1,146,reading,1302762
|
||||
micro,named_wide_16.json,named,wide,16,1,146,loading,57379
|
||||
micro,named_wide_16.json,named,wide,16,1,146,saving,29348
|
||||
micro,named_wide_16.json,named,wide,16,1,146,writing,736108
|
||||
micro,list_wide_8.json,list,wide,8,1,42,reading,931536
|
||||
micro,list_wide_8.json,list,wide,8,1,42,loading,47241
|
||||
micro,list_wide_8.json,list,wide,8,1,42,saving,13656
|
||||
micro,list_wide_8.json,list,wide,8,1,42,writing,218133
|
||||
micro,list_deep_1.json,list,deep,1,1,7,reading,906378
|
||||
micro,list_deep_1.json,list,deep,1,1,7,loading,58092
|
||||
micro,list_deep_1.json,list,deep,1,1,7,saving,13447
|
||||
micro,list_deep_1.json,list,deep,1,1,7,writing,163080
|
||||
micro,named_wide_4.json,named,wide,4,1,38,reading,907156
|
||||
micro,named_wide_4.json,named,wide,4,1,38,loading,29857
|
||||
micro,named_wide_4.json,named,wide,4,1,38,saving,14313
|
||||
micro,named_wide_4.json,named,wide,4,1,38,writing,298266
|
||||
micro,named_deep_2.json,named,deep,2,1,17,reading,918053
|
||||
micro,named_deep_2.json,named,deep,2,1,17,loading,21502
|
||||
micro,named_deep_2.json,named,deep,2,1,17,saving,48277
|
||||
micro,named_deep_2.json,named,deep,2,1,17,writing,261238
|
||||
micro,list_wide_256.json,list,wide,256,1,1282,reading,2623498
|
||||
micro,list_wide_256.json,list,wide,256,1,1282,loading,199524
|
||||
micro,list_wide_256.json,list,wide,256,1,1282,saving,105855
|
||||
micro,list_wide_256.json,list,wide,256,1,1282,writing,3321153
|
||||
micro,list_deep_8.json,list,deep,8,1,21,reading,900016
|
||||
micro,list_deep_8.json,list,deep,8,1,21,loading,47177
|
||||
micro,list_deep_8.json,list,deep,8,1,21,saving,16279
|
||||
micro,list_deep_8.json,list,deep,8,1,21,writing,243342
|
||||
micro,named_wide_2.json,named,wide,2,1,20,reading,936920
|
||||
micro,named_wide_2.json,named,wide,2,1,20,loading,31737
|
||||
micro,named_wide_2.json,named,wide,2,1,20,saving,18572
|
||||
micro,named_wide_2.json,named,wide,2,1,20,writing,212303
|
||||
micro,named_deep_64.json,named,deep,64,1,427,reading,3451995
|
||||
micro,named_deep_64.json,named,deep,64,1,427,loading,61418
|
||||
micro,named_deep_64.json,named,deep,64,1,427,saving,150072
|
||||
micro,named_deep_64.json,named,deep,64,1,427,writing,2973872
|
||||
micro,list_deep_64.json,list,deep,64,1,133,reading,1129096
|
||||
micro,list_deep_64.json,list,deep,64,1,133,loading,90452
|
||||
micro,list_deep_64.json,list,deep,64,1,133,saving,50118
|
||||
micro,list_deep_64.json,list,deep,64,1,133,writing,830353
|
||||
micro,named_wide_1.json,named,wide,1,1,11,reading,339666
|
||||
micro,named_wide_1.json,named,wide,1,1,11,loading,20189
|
||||
micro,named_wide_1.json,named,wide,1,1,11,saving,13764
|
||||
micro,named_wide_1.json,named,wide,1,1,11,writing,189796
|
||||
micro,list_deep_2.json,list,deep,2,1,9,reading,874282
|
||||
micro,list_deep_2.json,list,deep,2,1,9,loading,59254
|
||||
micro,list_deep_2.json,list,deep,2,1,9,saving,14115
|
||||
micro,list_deep_2.json,list,deep,2,1,9,writing,158582
|
||||
BIN
bench/20210929_lazy_vs_eager/eager_micro.png
Normal file
BIN
bench/20210929_lazy_vs_eager/eager_micro.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 108 KiB |
229
bench/20210929_lazy_vs_eager/lazy_bench.log
Normal file
229
bench/20210929_lazy_vs_eager/lazy_bench.log
Normal file
@ -0,0 +1,229 @@
|
||||
source,file,run,size,activity,ns
|
||||
gh,gh_gpl3.json,1,36979,reading,61320405
|
||||
gh,gh_gpl3.json,1,36979,loading,42017
|
||||
gh,gh_gpl3.json,1,36979,saving,50079
|
||||
gh,gh_gpl3.json,1,36979,writing,9986643
|
||||
fda,tobacco.json,1,1144,reading,2088605
|
||||
fda,tobacco.json,1,1144,loading,27363
|
||||
fda,tobacco.json,1,1144,saving,30169
|
||||
fda,tobacco.json,1,1144,writing,1017429
|
||||
rv,animal.json,1,7051,reading,11893192
|
||||
rv,animal.json,1,7051,loading,94506
|
||||
rv,animal.json,1,7051,saving,95070
|
||||
rv,animal.json,1,7051,writing,11651105
|
||||
synthetic,object.json,1,77,reading,498118
|
||||
synthetic,object.json,1,77,loading,56417
|
||||
synthetic,object.json,1,77,saving,15150
|
||||
synthetic,object.json,1,77,writing,285775
|
||||
ncdc,ncdc_gsom_sample.json,1,2867,reading,5224701
|
||||
ncdc,ncdc_gsom_sample.json,1,2867,loading,31954
|
||||
ncdc,ncdc_gsom_sample.json,1,2867,saving,72310
|
||||
ncdc,ncdc_gsom_sample.json,1,2867,writing,6735288
|
||||
gh,gh_events.json,1,54778,reading,81599566
|
||||
gh,gh_events.json,1,54778,loading,94033
|
||||
gh,gh_events.json,1,54778,saving,509829
|
||||
gh,gh_events.json,1,54778,writing,49834921
|
||||
fda,recall.json,1,22475,reading,34207262
|
||||
fda,recall.json,1,22475,loading,27327
|
||||
fda,recall.json,1,22475,saving,116892
|
||||
fda,recall.json,1,22475,writing,21859042
|
||||
ncdc,ncdc_locations.json,1,2808,reading,4343915
|
||||
ncdc,ncdc_locations.json,1,2808,loading,28674
|
||||
ncdc,ncdc_locations.json,1,2808,saving,49982
|
||||
ncdc,ncdc_locations.json,1,2808,writing,6366277
|
||||
fda,event100.json,1,2945177,reading,4199500880
|
||||
fda,event100.json,1,2945177,loading,34125
|
||||
fda,event100.json,1,2945177,saving,12261806
|
||||
fda,event100.json,1,2945177,writing,2269507967
|
||||
json.org,json_eg2.json,1,242,reading,1234874
|
||||
json.org,json_eg2.json,1,242,loading,28659
|
||||
json.org,json_eg2.json,1,242,saving,21889
|
||||
json.org,json_eg2.json,1,242,writing,644181
|
||||
penguin,linux.json,1,64401,reading,97255962
|
||||
penguin,linux.json,1,64401,loading,30181
|
||||
penguin,linux.json,1,64401,saving,342227
|
||||
penguin,linux.json,1,64401,writing,53590919
|
||||
rv,human_male.json,1,603912,reading,878221976
|
||||
rv,human_male.json,1,603912,loading,1388019
|
||||
rv,human_male.json,1,603912,saving,5072941
|
||||
rv,human_male.json,1,603912,writing,975501518
|
||||
json.org,json_eg3.json,1,602,reading,1699623
|
||||
json.org,json_eg3.json,1,602,loading,25646
|
||||
json.org,json_eg3.json,1,602,saving,31718
|
||||
json.org,json_eg3.json,1,602,writing,1050084
|
||||
penguin,bsd.json,1,5415,reading,8517810
|
||||
penguin,bsd.json,1,5415,loading,29907
|
||||
penguin,bsd.json,1,5415,saving,60855
|
||||
penguin,bsd.json,1,5415,writing,3802315
|
||||
gh,gh_mgree_events.json,1,168832,reading,246415105
|
||||
gh,gh_mgree_events.json,1,168832,loading,76633
|
||||
gh,gh_mgree_events.json,1,168832,saving,872396
|
||||
gh,gh_mgree_events.json,1,168832,writing,145354615
|
||||
gov.uk,bank-holidays.json,1,15124,reading,26568923
|
||||
gov.uk,bank-holidays.json,1,15124,loading,33502
|
||||
gov.uk,bank-holidays.json,1,15124,saving,218824
|
||||
gov.uk,bank-holidays.json,1,15124,writing,35390000
|
||||
fda,recall100.json,1,2117776,reading,2991325478
|
||||
fda,recall100.json,1,2117776,loading,30348
|
||||
fda,recall100.json,1,2117776,saving,9703964
|
||||
fda,recall100.json,1,2117776,writing,2287122804
|
||||
penguin,grisham.json,1,22488,reading,36170224
|
||||
penguin,grisham.json,1,22488,loading,37968
|
||||
penguin,grisham.json,1,22488,saving,431785
|
||||
penguin,grisham.json,1,22488,writing,51187093
|
||||
fda,event.json,1,1819,reading,3444766
|
||||
fda,event.json,1,1819,loading,25514
|
||||
fda,event.json,1,1819,saving,35653
|
||||
fda,event.json,1,1819,writing,1957068
|
||||
doi,doi_smoosh.json,1,537,reading,1008266
|
||||
doi,doi_smoosh.json,1,537,loading,28017
|
||||
doi,doi_smoosh.json,1,537,saving,29145
|
||||
doi,doi_smoosh.json,1,537,writing,1355392
|
||||
penn,historic-20181028.json,1,1468638,reading,2119654255
|
||||
penn,historic-20181028.json,1,1468638,loading,1102976
|
||||
penn,historic-20181028.json,1,1468638,saving,11160951
|
||||
penn,historic-20181028.json,1,1468638,writing,1741687725
|
||||
gh,mgree_repos.json,1,159905,reading,237843079
|
||||
gh,mgree_repos.json,1,159905,loading,58679
|
||||
gh,mgree_repos.json,1,159905,saving,831754
|
||||
gh,mgree_repos.json,1,159905,writing,131864580
|
||||
penguin,dekeijzer.json,1,446,reading,1585016
|
||||
penguin,dekeijzer.json,1,446,loading,26572
|
||||
penguin,dekeijzer.json,1,446,saving,20864
|
||||
penguin,dekeijzer.json,1,446,writing,815692
|
||||
json.org,json_eg1.json,1,583,reading,1268749
|
||||
json.org,json_eg1.json,1,583,loading,28686
|
||||
json.org,json_eg1.json,1,583,saving,27859
|
||||
json.org,json_eg1.json,1,583,writing,886649
|
||||
fda,registrationlisting100.json,1,214459,reading,311507703
|
||||
fda,registrationlisting100.json,1,214459,loading,29727
|
||||
fda,registrationlisting100.json,1,214459,saving,1632892
|
||||
fda,registrationlisting100.json,1,214459,writing,258404892
|
||||
json.org,json_eg5.json,1,873,reading,2103928
|
||||
json.org,json_eg5.json,1,873,loading,24857
|
||||
json.org,json_eg5.json,1,873,saving,34889
|
||||
json.org,json_eg5.json,1,873,writing,1825840
|
||||
doi,doi_10.1000_1.json,1,369,reading,1475886
|
||||
doi,doi_10.1000_1.json,1,369,loading,27217
|
||||
doi,doi_10.1000_1.json,1,369,saving,31791
|
||||
doi,doi_10.1000_1.json,1,369,writing,1070710
|
||||
ncdc,ncdc_datatypes.json,1,4013,reading,6769564
|
||||
ncdc,ncdc_datatypes.json,1,4013,loading,27457
|
||||
ncdc,ncdc_datatypes.json,1,4013,saving,95418
|
||||
ncdc,ncdc_datatypes.json,1,4013,writing,6251446
|
||||
gh,gh_licenses.json,1,2507,reading,4433571
|
||||
gh,gh_licenses.json,1,2507,loading,74754
|
||||
gh,gh_licenses.json,1,2507,saving,64047
|
||||
gh,gh_licenses.json,1,2507,writing,3211132
|
||||
fda,tobacco10.json,1,6567,reading,10447774
|
||||
fda,tobacco10.json,1,6567,loading,40774
|
||||
fda,tobacco10.json,1,6567,saving,47676
|
||||
fda,tobacco10.json,1,6567,writing,5293545
|
||||
fda,recall50.json,1,1049501,reading,1493750444
|
||||
fda,recall50.json,1,1049501,loading,30266
|
||||
fda,recall50.json,1,1049501,saving,4365694
|
||||
fda,recall50.json,1,1049501,writing,1123518242
|
||||
json.org,json_eg4.json,1,3468,reading,6373498
|
||||
json.org,json_eg4.json,1,3468,loading,28478
|
||||
json.org,json_eg4.json,1,3468,saving,67567
|
||||
json.org,json_eg4.json,1,3468,writing,3924030
|
||||
ncdc,ncdc_locationcategories.json,1,543,reading,1838057
|
||||
ncdc,ncdc_locationcategories.json,1,543,loading,29492
|
||||
ncdc,ncdc_locationcategories.json,1,543,saving,33773
|
||||
ncdc,ncdc_locationcategories.json,1,543,writing,1783274
|
||||
gh,ffs.json,1,5089,reading,8609836
|
||||
gh,ffs.json,1,5089,loading,61969
|
||||
gh,ffs.json,1,5089,saving,93027
|
||||
gh,ffs.json,1,5089,writing,4186523
|
||||
fda,recall10.json,1,212061,reading,300764409
|
||||
fda,recall10.json,1,212061,loading,31716
|
||||
fda,recall10.json,1,212061,saving,1262706
|
||||
fda,recall10.json,1,212061,writing,222510493
|
||||
ncdc,ncdc_datacategories.json,1,1075,reading,2515029
|
||||
ncdc,ncdc_datacategories.json,1,1075,loading,27961
|
||||
ncdc,ncdc_datacategories.json,1,1075,saving,48454
|
||||
ncdc,ncdc_datacategories.json,1,1075,writing,2939008
|
||||
fda,tobacco_products.json,1,2345,reading,4024483
|
||||
fda,tobacco_products.json,1,2345,loading,33026
|
||||
fda,tobacco_products.json,1,2345,saving,39123
|
||||
fda,tobacco_products.json,1,2345,writing,2576106
|
||||
fda,tobacco100.json,1,63634,reading,92941119
|
||||
fda,tobacco100.json,1,63634,loading,29386
|
||||
fda,tobacco100.json,1,63634,saving,304635
|
||||
fda,tobacco100.json,1,63634,writing,49627177
|
||||
ncdc,ncdc_ewr_station.json,1,191,reading,1181664
|
||||
ncdc,ncdc_ewr_station.json,1,191,loading,31627
|
||||
ncdc,ncdc_ewr_station.json,1,191,saving,14363
|
||||
ncdc,ncdc_ewr_station.json,1,191,writing,603710
|
||||
gh,kmt.json,1,4979,reading,8739107
|
||||
gh,kmt.json,1,4979,loading,76795
|
||||
gh,kmt.json,1,4979,saving,72342
|
||||
gh,kmt.json,1,4979,writing,4791472
|
||||
rv,demons.json,1,194,reading,1260344
|
||||
rv,demons.json,1,194,loading,67854
|
||||
rv,demons.json,1,194,saving,17486
|
||||
rv,demons.json,1,194,writing,455279
|
||||
rv,agni.json,1,89267,reading,133168005
|
||||
rv,agni.json,1,89267,loading,239421
|
||||
rv,agni.json,1,89267,saving,947251
|
||||
rv,agni.json,1,89267,writing,136245440
|
||||
fda,registrationlisting.json,1,3228,reading,5424061
|
||||
fda,registrationlisting.json,1,3228,loading,28682
|
||||
fda,registrationlisting.json,1,3228,saving,42267
|
||||
fda,registrationlisting.json,1,3228,writing,3382612
|
||||
synthetic,obj_rename.json,1,69,reading,954491
|
||||
synthetic,obj_rename.json,1,69,loading,34980
|
||||
synthetic,obj_rename.json,1,69,saving,15995
|
||||
synthetic,obj_rename.json,1,69,writing,349103
|
||||
gh,mgree.json,1,1331,reading,2563432
|
||||
gh,mgree.json,1,1331,loading,53139
|
||||
gh,mgree.json,1,1331,saving,39391
|
||||
gh,mgree.json,1,1331,writing,1490131
|
||||
ncdc,ncdc_07030_stations.json,1,1476,reading,3299967
|
||||
ncdc,ncdc_07030_stations.json,1,1476,loading,45400
|
||||
ncdc,ncdc_07030_stations.json,1,1476,saving,33814
|
||||
ncdc,ncdc_07030_stations.json,1,1476,writing,3593597
|
||||
synthetic,list2.json,1,25,reading,887161
|
||||
synthetic,list2.json,1,25,loading,51012
|
||||
synthetic,list2.json,1,25,saving,12682
|
||||
synthetic,list2.json,1,25,writing,396488
|
||||
ncdc,ncdc_07030.json,1,111,reading,1150898
|
||||
ncdc,ncdc_07030.json,1,111,loading,29696
|
||||
ncdc,ncdc_07030.json,1,111,saving,13669
|
||||
ncdc,ncdc_07030.json,1,111,writing,624790
|
||||
ncdc,ncdc_stations_big.json,1,200908,reading,297621635
|
||||
ncdc,ncdc_stations_big.json,1,200908,loading,29906
|
||||
ncdc,ncdc_stations_big.json,1,200908,saving,1596193
|
||||
ncdc,ncdc_stations_big.json,1,200908,writing,382516634
|
||||
ncdc,ncdc_zips.json,1,2851,reading,4760945
|
||||
ncdc,ncdc_zips.json,1,2851,loading,31141
|
||||
ncdc,ncdc_zips.json,1,2851,saving,52850
|
||||
ncdc,ncdc_zips.json,1,2851,writing,6180847
|
||||
ncdc,ncdc_datasets.json,1,1625,reading,3082976
|
||||
ncdc,ncdc_datasets.json,1,1625,loading,67240
|
||||
ncdc,ncdc_datasets.json,1,1625,saving,33903
|
||||
ncdc,ncdc_datasets.json,1,1625,writing,3373872
|
||||
penguin,rust.json,1,161726,reading,233761531
|
||||
penguin,rust.json,1,161726,loading,29150
|
||||
penguin,rust.json,1,161726,saving,413125
|
||||
penguin,rust.json,1,161726,writing,40922967
|
||||
gh,ffs_issues.json,1,38158,reading,57319742
|
||||
gh,ffs_issues.json,1,38158,loading,60266
|
||||
gh,ffs_issues.json,1,38158,saving,212641
|
||||
gh,ffs_issues.json,1,38158,writing,35494606
|
||||
fda,waterpipe.json,1,1955,reading,3856805
|
||||
fda,waterpipe.json,1,1955,loading,26625
|
||||
fda,waterpipe.json,1,1955,saving,29047
|
||||
fda,waterpipe.json,1,1955,writing,1500353
|
||||
gh,gh_apache2.json,1,12591,reading,19674974
|
||||
gh,gh_apache2.json,1,12591,loading,47699
|
||||
gh,gh_apache2.json,1,12591,saving,45453
|
||||
gh,gh_apache2.json,1,12591,writing,3457322
|
||||
gh,gh_emoji.json,1,173641,reading,250477225
|
||||
gh,gh_emoji.json,1,173641,loading,1002948
|
||||
gh,gh_emoji.json,1,173641,saving,1504798
|
||||
gh,gh_emoji.json,1,173641,writing,87263696
|
||||
synthetic,object_null.json,1,95,reading,1184428
|
||||
synthetic,object_null.json,1,95,loading,29649
|
||||
synthetic,object_null.json,1,95,saving,23328
|
||||
synthetic,object_null.json,1,95,writing,331025
|
||||
BIN
bench/20210929_lazy_vs_eager/lazy_bench.png
Normal file
BIN
bench/20210929_lazy_vs_eager/lazy_bench.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 123 KiB |
129
bench/20210929_lazy_vs_eager/lazy_micro.log
Normal file
129
bench/20210929_lazy_vs_eager/lazy_micro.log
Normal file
@ -0,0 +1,129 @@
|
||||
source,file,kind,direction,magnitude,run,size,activity,ns
|
||||
micro,list_deep_1.json,list,deep,1,1,7,reading,77411
|
||||
micro,list_deep_1.json,list,deep,1,1,7,loading,48041
|
||||
micro,list_deep_1.json,list,deep,1,1,7,saving,9725
|
||||
micro,list_deep_1.json,list,deep,1,1,7,writing,208293
|
||||
micro,list_wide_16.json,list,wide,16,1,82,reading,197327
|
||||
micro,list_wide_16.json,list,wide,16,1,82,loading,50356
|
||||
micro,list_wide_16.json,list,wide,16,1,82,saving,14955
|
||||
micro,list_wide_16.json,list,wide,16,1,82,writing,513614
|
||||
micro,list_wide_1.json,list,wide,1,1,7,reading,41303
|
||||
micro,list_wide_1.json,list,wide,1,1,7,loading,47363
|
||||
micro,list_wide_1.json,list,wide,1,1,7,saving,9518
|
||||
micro,list_wide_1.json,list,wide,1,1,7,writing,163677
|
||||
micro,named_deep_2.json,named,deep,2,1,17,reading,67763
|
||||
micro,named_deep_2.json,named,deep,2,1,17,loading,26929
|
||||
micro,named_deep_2.json,named,deep,2,1,17,saving,15303
|
||||
micro,named_deep_2.json,named,deep,2,1,17,writing,227808
|
||||
micro,named_wide_256.json,named,wide,256,1,2536,reading,3882963
|
||||
micro,named_wide_256.json,named,wide,256,1,2536,loading,124616
|
||||
micro,named_wide_256.json,named,wide,256,1,2536,saving,130790
|
||||
micro,named_wide_256.json,named,wide,256,1,2536,writing,9152278
|
||||
micro,list_deep_64.json,list,deep,64,1,133,reading,214251
|
||||
micro,list_deep_64.json,list,deep,64,1,133,loading,45853
|
||||
micro,list_deep_64.json,list,deep,64,1,133,saving,35144
|
||||
micro,list_deep_64.json,list,deep,64,1,133,writing,887716
|
||||
micro,named_wide_2.json,named,wide,2,1,20,reading,69966
|
||||
micro,named_wide_2.json,named,wide,2,1,20,loading,28250
|
||||
micro,named_wide_2.json,named,wide,2,1,20,saving,15993
|
||||
micro,named_wide_2.json,named,wide,2,1,20,writing,237669
|
||||
micro,list_deep_32.json,list,deep,32,1,69,reading,122918
|
||||
micro,list_deep_32.json,list,deep,32,1,69,loading,45041
|
||||
micro,list_deep_32.json,list,deep,32,1,69,saving,19007
|
||||
micro,list_deep_32.json,list,deep,32,1,69,writing,504770
|
||||
micro,named_deep_32.json,named,deep,32,1,203,reading,319594
|
||||
micro,named_deep_32.json,named,deep,32,1,203,loading,30709
|
||||
micro,named_deep_32.json,named,deep,32,1,203,saving,42750
|
||||
micro,named_deep_32.json,named,deep,32,1,203,writing,22185864
|
||||
micro,named_deep_16.json,named,deep,16,1,101,reading,206082
|
||||
micro,named_deep_16.json,named,deep,16,1,101,loading,26184
|
||||
micro,named_deep_16.json,named,deep,16,1,101,saving,23956
|
||||
micro,named_deep_16.json,named,deep,16,1,101,writing,654740
|
||||
micro,named_wide_128.json,named,wide,128,1,1256,reading,1907527
|
||||
micro,named_wide_128.json,named,wide,128,1,1256,loading,94912
|
||||
micro,named_wide_128.json,named,wide,128,1,1256,saving,81363
|
||||
micro,named_wide_128.json,named,wide,128,1,1256,writing,4632730
|
||||
micro,named_wide_8.json,named,wide,8,1,74,reading,158923
|
||||
micro,named_wide_8.json,named,wide,8,1,74,loading,40437
|
||||
micro,named_wide_8.json,named,wide,8,1,74,saving,19614
|
||||
micro,named_wide_8.json,named,wide,8,1,74,writing,413197
|
||||
micro,list_wide_128.json,list,wide,128,1,642,reading,958284
|
||||
micro,list_wide_128.json,list,wide,128,1,642,loading,89919
|
||||
micro,list_wide_128.json,list,wide,128,1,642,saving,48836
|
||||
micro,list_wide_128.json,list,wide,128,1,642,writing,1603550
|
||||
micro,named_wide_32.json,named,wide,32,1,296,reading,693077
|
||||
micro,named_wide_32.json,named,wide,32,1,296,loading,42744
|
||||
micro,named_wide_32.json,named,wide,32,1,296,saving,37735
|
||||
micro,named_wide_32.json,named,wide,32,1,296,writing,1310647
|
||||
micro,named_deep_8.json,named,deep,8,1,53,reading,122278
|
||||
micro,named_deep_8.json,named,deep,8,1,53,loading,25771
|
||||
micro,named_deep_8.json,named,deep,8,1,53,saving,22704
|
||||
micro,named_deep_8.json,named,deep,8,1,53,writing,490909
|
||||
micro,named_wide_4.json,named,wide,4,1,38,reading,110008
|
||||
micro,named_wide_4.json,named,wide,4,1,38,loading,37948
|
||||
micro,named_wide_4.json,named,wide,4,1,38,saving,12089
|
||||
micro,named_wide_4.json,named,wide,4,1,38,writing,276642
|
||||
micro,named_wide_16.json,named,wide,16,1,146,reading,250222
|
||||
micro,named_wide_16.json,named,wide,16,1,146,loading,28701
|
||||
micro,named_wide_16.json,named,wide,16,1,146,saving,23997
|
||||
micro,named_wide_16.json,named,wide,16,1,146,writing,662904
|
||||
micro,list_deep_4.json,list,deep,4,1,13,reading,46747
|
||||
micro,list_deep_4.json,list,deep,4,1,13,loading,46082
|
||||
micro,list_deep_4.json,list,deep,4,1,13,saving,18715
|
||||
micro,list_deep_4.json,list,deep,4,1,13,writing,195556
|
||||
micro,list_wide_256.json,list,wide,256,1,1282,reading,1953376
|
||||
micro,list_wide_256.json,list,wide,256,1,1282,loading,140498
|
||||
micro,list_wide_256.json,list,wide,256,1,1282,saving,94049
|
||||
micro,list_wide_256.json,list,wide,256,1,1282,writing,2952792
|
||||
micro,named_deep_1.json,named,deep,1,1,11,reading,68100
|
||||
micro,named_deep_1.json,named,deep,1,1,11,loading,29068
|
||||
micro,named_deep_1.json,named,deep,1,1,11,saving,10046
|
||||
micro,named_deep_1.json,named,deep,1,1,11,writing,162049
|
||||
micro,named_wide_64.json,named,wide,64,1,616,reading,956614
|
||||
micro,named_wide_64.json,named,wide,64,1,616,loading,56294
|
||||
micro,named_wide_64.json,named,wide,64,1,616,saving,45541
|
||||
micro,named_wide_64.json,named,wide,64,1,616,writing,2166256
|
||||
micro,list_wide_2.json,list,wide,2,1,12,reading,46945
|
||||
micro,list_wide_2.json,list,wide,2,1,12,loading,47035
|
||||
micro,list_wide_2.json,list,wide,2,1,12,saving,9517
|
||||
micro,list_wide_2.json,list,wide,2,1,12,writing,197836
|
||||
micro,list_deep_2.json,list,deep,2,1,9,reading,114821
|
||||
micro,list_deep_2.json,list,deep,2,1,9,loading,50216
|
||||
micro,list_deep_2.json,list,deep,2,1,9,saving,12237
|
||||
micro,list_deep_2.json,list,deep,2,1,9,writing,189978
|
||||
micro,list_wide_8.json,list,wide,8,1,42,reading,120506
|
||||
micro,list_wide_8.json,list,wide,8,1,42,loading,40533
|
||||
micro,list_wide_8.json,list,wide,8,1,42,saving,11830
|
||||
micro,list_wide_8.json,list,wide,8,1,42,writing,296876
|
||||
micro,named_deep_4.json,named,deep,4,1,29,reading,80846
|
||||
micro,named_deep_4.json,named,deep,4,1,29,loading,25848
|
||||
micro,named_deep_4.json,named,deep,4,1,29,saving,30188
|
||||
micro,named_deep_4.json,named,deep,4,1,29,writing,286524
|
||||
micro,list_deep_8.json,list,deep,8,1,21,reading,59378
|
||||
micro,list_deep_8.json,list,deep,8,1,21,loading,57111
|
||||
micro,list_deep_8.json,list,deep,8,1,21,saving,44890
|
||||
micro,list_deep_8.json,list,deep,8,1,21,writing,236407
|
||||
micro,named_deep_64.json,named,deep,64,1,427,reading,657390
|
||||
micro,named_deep_64.json,named,deep,64,1,427,loading,30568
|
||||
micro,named_deep_64.json,named,deep,64,1,427,saving,68322
|
||||
micro,named_deep_64.json,named,deep,64,1,427,writing,2255662
|
||||
micro,list_wide_32.json,list,wide,32,1,162,reading,696512
|
||||
micro,list_wide_32.json,list,wide,32,1,162,loading,80003
|
||||
micro,list_wide_32.json,list,wide,32,1,162,saving,26583
|
||||
micro,list_wide_32.json,list,wide,32,1,162,writing,813548
|
||||
micro,named_wide_1.json,named,wide,1,1,11,reading,289819
|
||||
micro,named_wide_1.json,named,wide,1,1,11,loading,40934
|
||||
micro,named_wide_1.json,named,wide,1,1,11,saving,20500
|
||||
micro,named_wide_1.json,named,wide,1,1,11,writing,228788
|
||||
micro,list_deep_16.json,list,deep,16,1,37,reading,911549
|
||||
micro,list_deep_16.json,list,deep,16,1,37,loading,48957
|
||||
micro,list_deep_16.json,list,deep,16,1,37,saving,28891
|
||||
micro,list_deep_16.json,list,deep,16,1,37,writing,401403
|
||||
micro,list_wide_4.json,list,wide,4,1,22,reading,261660
|
||||
micro,list_wide_4.json,list,wide,4,1,22,loading,50001
|
||||
micro,list_wide_4.json,list,wide,4,1,22,saving,13005
|
||||
micro,list_wide_4.json,list,wide,4,1,22,writing,254205
|
||||
micro,list_wide_64.json,list,wide,64,1,322,reading,786524
|
||||
micro,list_wide_64.json,list,wide,64,1,322,loading,82289
|
||||
micro,list_wide_64.json,list,wide,64,1,322,saving,27614
|
||||
micro,list_wide_64.json,list,wide,64,1,322,writing,859187
|
||||
BIN
bench/20210929_lazy_vs_eager/lazy_micro.png
Normal file
BIN
bench/20210929_lazy_vs_eager/lazy_micro.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 109 KiB |
0
bench/20210929_lazy_vs_force_vs_eager/README.md
Normal file
0
bench/20210929_lazy_vs_force_vs_eager/README.md
Normal file
2281
bench/20210929_lazy_vs_force_vs_eager/eager_bench.log
Normal file
2281
bench/20210929_lazy_vs_force_vs_eager/eager_bench.log
Normal file
File diff suppressed because it is too large
Load Diff
BIN
bench/20210929_lazy_vs_force_vs_eager/eager_bench.png
Normal file
BIN
bench/20210929_lazy_vs_force_vs_eager/eager_bench.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 129 KiB |
1281
bench/20210929_lazy_vs_force_vs_eager/eager_micro.log
Normal file
1281
bench/20210929_lazy_vs_force_vs_eager/eager_micro.log
Normal file
File diff suppressed because it is too large
Load Diff
BIN
bench/20210929_lazy_vs_force_vs_eager/eager_micro.png
Normal file
BIN
bench/20210929_lazy_vs_force_vs_eager/eager_micro.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 117 KiB |
2281
bench/20210929_lazy_vs_force_vs_eager/force_bench.log
Normal file
2281
bench/20210929_lazy_vs_force_vs_eager/force_bench.log
Normal file
File diff suppressed because it is too large
Load Diff
BIN
bench/20210929_lazy_vs_force_vs_eager/force_bench.png
Normal file
BIN
bench/20210929_lazy_vs_force_vs_eager/force_bench.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 125 KiB |
1281
bench/20210929_lazy_vs_force_vs_eager/force_micro.log
Normal file
1281
bench/20210929_lazy_vs_force_vs_eager/force_micro.log
Normal file
File diff suppressed because it is too large
Load Diff
BIN
bench/20210929_lazy_vs_force_vs_eager/force_micro.png
Normal file
BIN
bench/20210929_lazy_vs_force_vs_eager/force_micro.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 114 KiB |
2281
bench/20210929_lazy_vs_force_vs_eager/lazy_bench.log
Normal file
2281
bench/20210929_lazy_vs_force_vs_eager/lazy_bench.log
Normal file
File diff suppressed because it is too large
Load Diff
BIN
bench/20210929_lazy_vs_force_vs_eager/lazy_bench.png
Normal file
BIN
bench/20210929_lazy_vs_force_vs_eager/lazy_bench.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 131 KiB |
1281
bench/20210929_lazy_vs_force_vs_eager/lazy_micro.log
Normal file
1281
bench/20210929_lazy_vs_force_vs_eager/lazy_micro.log
Normal file
File diff suppressed because it is too large
Load Diff
BIN
bench/20210929_lazy_vs_force_vs_eager/lazy_micro.png
Normal file
BIN
bench/20210929_lazy_vs_force_vs_eager/lazy_micro.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 117 KiB |
@ -177,7 +177,7 @@ do
|
||||
|
||||
tempfile log
|
||||
tempfile out
|
||||
$FFS --time -m $mnt -o $out -t json $path 2>$log &
|
||||
$FFS $FFS_ARGS --time -m $mnt -o $out -t json $path 2>$log &
|
||||
PID=$!
|
||||
PIDS="$PIDS $PID"
|
||||
count=0
|
||||
|
||||
@ -20,7 +20,7 @@ _ffs() {
|
||||
|
||||
case "${cmd}" in
|
||||
ffs)
|
||||
opts=" -q -d -i -h -V -u -g -o -s -t -m --quiet --time --debug --exact --no-xattr --keep-macos-xattr --unpadded --readonly --no-output --in-place --pretty --help --version --completions --uid --gid --mode --dirmode --munge --output --source --target --mount --new <INPUT> "
|
||||
opts=" -q -d -i -h -V -u -g -o -s -t -m --quiet --time --debug --eager --exact --no-xattr --keep-macos-xattr --unpadded --readonly --no-output --in-place --pretty --help --version --completions --uid --gid --mode --dirmode --munge --output --source --target --mount --new <INPUT> "
|
||||
if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
|
||||
COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
|
||||
return 0
|
||||
|
||||
@ -12,6 +12,7 @@ complete -c ffs -n "__fish_use_subcommand" -l new -d 'Mounts an empty filesystem
|
||||
complete -c ffs -n "__fish_use_subcommand" -s q -l quiet -d 'Quiet mode (turns off all errors and warnings, enables `--no-output`)'
|
||||
complete -c ffs -n "__fish_use_subcommand" -l time -d 'Emit timing information on stderr in an \'event,time\' format; time is in nanoseconds'
|
||||
complete -c ffs -n "__fish_use_subcommand" -s d -l debug -d 'Give debug output on stderr'
|
||||
complete -c ffs -n "__fish_use_subcommand" -l eager -d 'Eagerly load data on startup'
|
||||
complete -c ffs -n "__fish_use_subcommand" -l exact -d 'Don\'t add newlines to the end of values that don\'t already have them (or strip them when loading)'
|
||||
complete -c ffs -n "__fish_use_subcommand" -l no-xattr -d 'Don\'t use extended attributes to track metadata (see `man xattr`)'
|
||||
complete -c ffs -n "__fish_use_subcommand" -l keep-macos-xattr -d 'Include ._* extended attribute/resource fork files on macOS'
|
||||
|
||||
@ -37,6 +37,7 @@ _ffs() {
|
||||
'--time[Emit timing information on stderr in an '\''event,time'\'' format; time is in nanoseconds]' \
|
||||
'-d[Give debug output on stderr]' \
|
||||
'--debug[Give debug output on stderr]' \
|
||||
'--eager[Eagerly load data on startup]' \
|
||||
'--exact[Don'\''t add newlines to the end of values that don'\''t already have them (or strip them when loading)]' \
|
||||
'--no-xattr[Don'\''t use extended attributes to track metadata (see `man xattr`)]' \
|
||||
'--keep-macos-xattr[Include ._* extended attribute/resource fork files on macOS]' \
|
||||
|
||||
@ -31,6 +31,11 @@ installed on your system to use *ffs*.
|
||||
|
||||
: Give debug output on stderr
|
||||
|
||||
--eager
|
||||
|
||||
: Eagerly load all data on startup. *ffs*'s default behavior is to lazily load
|
||||
data on startup, which avoids preparing data that won't be read or written.
|
||||
|
||||
--exact
|
||||
|
||||
: Don't add newlines to the end of values that don't already have them
|
||||
|
||||
1
json/big/.gitignore
vendored
Normal file
1
json/big/.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
||||
citylots.json
|
||||
@ -39,6 +39,11 @@ installed on your system to use \f[I]ffs\f[R].
|
||||
-d, --debug
|
||||
Give debug output on stderr
|
||||
.TP
|
||||
--eager
|
||||
Eagerly load all data on startup.
|
||||
\f[I]ffs\f[R]\[aq]s default behavior is to lazily load data on startup,
|
||||
which avoids preparing data that won\[aq]t be read or written.
|
||||
.TP
|
||||
--exact
|
||||
Don\[aq]t add newlines to the end of values that don\[aq]t already have
|
||||
them (or strip them when loading)
|
||||
|
||||
27
run_bench.sh
27
run_bench.sh
@ -4,6 +4,7 @@ set -e
|
||||
|
||||
TIMESTAMP=$(date +"%Y%m%d_%H:%M:%S")
|
||||
|
||||
NUM_RUNS_DEFAULT=10
|
||||
usage() {
|
||||
exec >&2
|
||||
printf "Usage: %s [-n NUM_RUNS]\n\n" "$(basename $0)"
|
||||
@ -34,14 +35,30 @@ shift $((OPTIND - 1))
|
||||
|
||||
cd bench
|
||||
|
||||
BENCH="../${TIMESTAMP}_bench.log"
|
||||
./bench.sh $ARGS >"$BENCH"
|
||||
mkdir ${TIMESTAMP}
|
||||
|
||||
./mk_micro.sh
|
||||
MICRO_RAW=$(mktemp)
|
||||
|
||||
printf "BENCHMARKING LAZY MODE\n"
|
||||
|
||||
BENCH_LAZY="${TIMESTAMP}/lazy_bench.log"
|
||||
./bench.sh $ARGS >"$BENCH_LAZY"
|
||||
|
||||
./bench.sh -d micro $ARGS >"$MICRO_RAW"
|
||||
MICRO="../${TIMESTAMP}_micro.log"
|
||||
./fixup_micro.sh "$MICRO_RAW" >"$MICRO"
|
||||
MICRO_LAZY="${TIMESTAMP}/lazy_micro.log"
|
||||
./fixup_micro.sh "$MICRO_RAW" >"$MICRO_LAZY"
|
||||
|
||||
printf "BENCHMARKING EAGER MODE\n"
|
||||
|
||||
BENCH_EAGER="${TIMESTAMP}/eager_bench.log"
|
||||
FFS_ARGS="--eager" ./bench.sh $ARGS >"$BENCH_EAGER"
|
||||
|
||||
FFS_ARGS="--eager" ./bench.sh -d micro $ARGS >"$MICRO_RAW"
|
||||
MICRO_EAGER="${TIMESTAMP}/eager_micro.log"
|
||||
./fixup_micro.sh "$MICRO_RAW" >"$MICRO_EAGER"
|
||||
|
||||
rm "$MICRO_RAW"
|
||||
|
||||
./generate_charts.R "$BENCH" "$MICRO"
|
||||
./generate_charts.R "$BENCH_LAZY" "$MICRO_LAZY"
|
||||
./generate_charts.R "$BENCH_EAGER" "$MICRO_EAGER"
|
||||
|
||||
@ -36,6 +36,11 @@ pub fn app() -> App<'static, 'static> {
|
||||
.long("debug")
|
||||
.short("d")
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("EAGER")
|
||||
.help("Eagerly load data on startup (data is lazily loaded by default)")
|
||||
.long("eager")
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("UID")
|
||||
.help("Sets the user id of the generated filesystem (defaults to current effective user id)")
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
use std::fs::File;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::str::FromStr;
|
||||
|
||||
@ -26,6 +27,7 @@ pub const ERROR_STATUS_CLI: i32 = 2;
|
||||
pub struct Config {
|
||||
pub input_format: Format,
|
||||
pub output_format: Format,
|
||||
pub eager: bool,
|
||||
pub uid: u32,
|
||||
pub gid: u32,
|
||||
pub filemode: u16,
|
||||
@ -63,7 +65,7 @@ impl std::fmt::Display for Input {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum Output {
|
||||
Quiet,
|
||||
Stdout,
|
||||
@ -104,12 +106,10 @@ impl FromStr for Munge {
|
||||
impl Config {
|
||||
/// Parses arguments from `std::env::Args`, via `cli::app().get_matches()`
|
||||
pub fn from_args() -> Self {
|
||||
let args = cli::app()
|
||||
.get_matches_safe()
|
||||
.unwrap_or_else(|e| {
|
||||
eprintln!("{}", e.message);
|
||||
std::process::exit(ERROR_STATUS_CLI)
|
||||
});
|
||||
let args = cli::app().get_matches_safe().unwrap_or_else(|e| {
|
||||
eprintln!("{}", e.message);
|
||||
std::process::exit(ERROR_STATUS_CLI)
|
||||
});
|
||||
|
||||
let mut config = Config::default();
|
||||
// generate completions?
|
||||
@ -148,6 +148,7 @@ impl Config {
|
||||
|
||||
// simple flags
|
||||
config.timing = args.is_present("TIMING");
|
||||
config.eager = args.is_present("EAGER");
|
||||
config.add_newlines = !args.is_present("EXACT");
|
||||
config.pad_element_names = !args.is_present("UNPADDED");
|
||||
config.read_only = args.is_present("READONLY");
|
||||
@ -573,6 +574,44 @@ impl Config {
|
||||
self.filemode
|
||||
}
|
||||
}
|
||||
|
||||
/// Generate a reader for input
|
||||
///
|
||||
/// A return of `None` means to start from an empty named directory
|
||||
pub fn input_reader(&self) -> Option<Box<dyn std::io::Read>> {
|
||||
match &self.input {
|
||||
Input::Stdin => Some(Box::new(std::io::stdin())),
|
||||
Input::File(file) => {
|
||||
let fmt = self.input_format;
|
||||
let file = std::fs::File::open(&file).unwrap_or_else(|e| {
|
||||
error!("Unable to open {} for {} input: {}", file.display(), fmt, e);
|
||||
std::process::exit(ERROR_STATUS_FUSE);
|
||||
});
|
||||
Some(Box::new(file))
|
||||
}
|
||||
Input::Empty => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Generate a writer for output
|
||||
///
|
||||
/// A return of `None` means no output should be provided
|
||||
pub fn output_writer(&self) -> Option<Box<dyn std::io::Write>> {
|
||||
match &self.output {
|
||||
Output::Stdout => {
|
||||
debug!("outputting on STDOUT");
|
||||
Some(Box::new(std::io::stdout()))
|
||||
}
|
||||
Output::File(path) => {
|
||||
debug!("output {}", path.display());
|
||||
Some(Box::new(File::create(path).unwrap()))
|
||||
}
|
||||
Output::Quiet => {
|
||||
debug!("no output path, skipping");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Config {
|
||||
@ -580,6 +619,7 @@ impl Default for Config {
|
||||
Config {
|
||||
input_format: Format::Json,
|
||||
output_format: Format::Json,
|
||||
eager: false,
|
||||
uid: 501,
|
||||
gid: 501,
|
||||
filemode: 0o644,
|
||||
|
||||
605
src/format.rs
605
src/format.rs
@ -1,20 +1,35 @@
|
||||
use std::collections::HashMap;
|
||||
use std::fs::File;
|
||||
use std::str::FromStr;
|
||||
|
||||
use tracing::{debug, error, info, instrument, warn};
|
||||
use tracing::debug;
|
||||
|
||||
use fuser::FileType;
|
||||
|
||||
use super::config::{Config, Input, Munge, Output, ERROR_STATUS_FUSE};
|
||||
use super::fs::{DirEntry, DirType, Entry, Inode, FS};
|
||||
use super::config::Config;
|
||||
|
||||
use ::toml as serde_toml;
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! time_ns {
|
||||
($msg:expr, $e:expr, $timing:expr) => {{
|
||||
let start = std::time::Instant::now();
|
||||
let v = $e;
|
||||
|
||||
let msg = $msg;
|
||||
let elapsed = start.elapsed().as_nanos();
|
||||
if $timing {
|
||||
eprintln!("{},{}", msg, elapsed);
|
||||
} else {
|
||||
info!("{} ({}ns)", msg, elapsed);
|
||||
}
|
||||
v
|
||||
}};
|
||||
}
|
||||
|
||||
/// The possible formats.
|
||||
///
|
||||
/// When extending, don't forget to also extend `cli::POSSIBLE_FORMATS`.
|
||||
#[derive(Copy, Clone, Debug)]
|
||||
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
|
||||
pub enum Format {
|
||||
Json,
|
||||
Toml,
|
||||
@ -126,155 +141,11 @@ impl Format {
|
||||
Format::Yaml => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Generates a filesystem `fs`, reading from `reader` according to a
|
||||
/// particular `Config`.
|
||||
///
|
||||
/// NB there is no check that `self == fs.config.input_format`!
|
||||
#[instrument(level = "info", skip(config))]
|
||||
pub fn load(&self, config: Config) -> FS {
|
||||
macro_rules! time_ns {
|
||||
($msg:expr, $e:expr) => {{
|
||||
let start = std::time::Instant::now();
|
||||
let v = $e;
|
||||
|
||||
let msg = $msg;
|
||||
let elapsed = start.elapsed().as_nanos();
|
||||
if config.timing {
|
||||
eprintln!("{},{}", msg, elapsed);
|
||||
} else {
|
||||
info!("{} ({}ns)", msg, elapsed);
|
||||
}
|
||||
|
||||
v
|
||||
}};
|
||||
}
|
||||
|
||||
info!("loading");
|
||||
let mut inodes: Vec<Option<Inode>> = Vec::new();
|
||||
|
||||
let reader: Box<dyn std::io::Read> = match &config.input {
|
||||
Input::Stdin => Box::new(std::io::stdin()),
|
||||
Input::File(file) => {
|
||||
let fmt = config.input_format;
|
||||
let file = std::fs::File::open(&file).unwrap_or_else(|e| {
|
||||
error!("Unable to open {} for {} input: {}", file.display(), fmt, e);
|
||||
std::process::exit(ERROR_STATUS_FUSE);
|
||||
});
|
||||
Box::new(file)
|
||||
}
|
||||
Input::Empty => {
|
||||
// let's just reserve some space for later and get cracking
|
||||
info!("reserving space in empty filesystem");
|
||||
inodes.resize_with(1024, || None);
|
||||
|
||||
// create an empty directory
|
||||
let contents = HashMap::with_capacity(16);
|
||||
inodes[1] = Some(Inode::new(
|
||||
fuser::FUSE_ROOT_ID,
|
||||
fuser::FUSE_ROOT_ID,
|
||||
Entry::Directory(DirType::Named, contents),
|
||||
&config,
|
||||
));
|
||||
return FS::new(inodes, config);
|
||||
}
|
||||
};
|
||||
|
||||
match self {
|
||||
Format::Json => {
|
||||
let v: serde_json::Value = time_ns!(
|
||||
"reading",
|
||||
serde_json::from_reader(reader).expect("JSON")
|
||||
);
|
||||
time_ns!("loading", fs_from_value(v, &config, &mut inodes));
|
||||
}
|
||||
Format::Toml => {
|
||||
let v = time_ns!("reading", toml::from_reader(reader).expect("TOML"));
|
||||
time_ns!("loading", fs_from_value(v, &config, &mut inodes));
|
||||
}
|
||||
Format::Yaml => {
|
||||
let v = time_ns!("reading", yaml::from_reader(reader).expect("YAML"));
|
||||
time_ns!("loading", fs_from_value(v, &config, &mut inodes));
|
||||
}
|
||||
};
|
||||
|
||||
FS::new(inodes, config)
|
||||
}
|
||||
|
||||
/// Given a filesystem `fs`, it outputs a file in the appropriate format,
|
||||
/// following `fs.config`.
|
||||
///
|
||||
/// NB there is no check that `self == fs.config.output_format`!
|
||||
#[instrument(level = "info", skip(fs))]
|
||||
pub fn save(&self, fs: &FS) {
|
||||
macro_rules! time_ns {
|
||||
($msg:expr, $e:expr) => {{
|
||||
let start = std::time::Instant::now();
|
||||
let v = $e;
|
||||
|
||||
let msg = $msg;
|
||||
let elapsed = start.elapsed().as_nanos();
|
||||
if fs.config.timing {
|
||||
eprintln!("{},{}", msg, elapsed);
|
||||
} else {
|
||||
info!("{} ({}ns)", msg, elapsed);
|
||||
}
|
||||
v
|
||||
}};
|
||||
}
|
||||
|
||||
let writer: Box<dyn std::io::Write> = match &fs.config.output {
|
||||
Output::Stdout => {
|
||||
debug!("outputting on STDOUT");
|
||||
Box::new(std::io::stdout())
|
||||
}
|
||||
Output::File(path) => {
|
||||
debug!("output {}", path.display());
|
||||
Box::new(File::create(path).unwrap())
|
||||
}
|
||||
Output::Quiet => {
|
||||
debug!("no output path, skipping");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
match self {
|
||||
Format::Json => {
|
||||
let v: serde_json::Value =
|
||||
time_ns!("saving", value_from_fs(fs, fuser::FUSE_ROOT_ID));
|
||||
debug!("outputting {}", v);
|
||||
time_ns!(
|
||||
"writing",
|
||||
if fs.config.pretty {
|
||||
serde_json::to_writer_pretty(writer, &v).unwrap();
|
||||
} else {
|
||||
serde_json::to_writer(writer, &v).unwrap();
|
||||
}
|
||||
);
|
||||
}
|
||||
Format::Toml => {
|
||||
let v: serde_toml::Value =
|
||||
time_ns!("saving", value_from_fs(fs, fuser::FUSE_ROOT_ID));
|
||||
debug!("outputting {}", v);
|
||||
time_ns!(
|
||||
"writing",
|
||||
if fs.config.pretty {
|
||||
toml::to_writer_pretty(writer, &v).unwrap();
|
||||
} else {
|
||||
toml::to_writer(writer, &v).unwrap();
|
||||
}
|
||||
);
|
||||
}
|
||||
Format::Yaml => {
|
||||
let v: yaml::Value = time_ns!("saving", value_from_fs(fs, fuser::FUSE_ROOT_ID));
|
||||
debug!("outputting {}", v);
|
||||
time_ns!("writing", yaml::to_writer(writer, &v).unwrap());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum Node<V> {
|
||||
/// The ffs data model; it represents just one layer---lists and maps are
|
||||
/// parameterized over the underlying value type V.
|
||||
pub enum Node<V> {
|
||||
String(Typ, String),
|
||||
Bytes(Vec<u8>),
|
||||
|
||||
@ -289,9 +160,9 @@ enum Node<V> {
|
||||
|
||||
/// Values that can be converted to a `Node`, which can be in turn processed by
|
||||
/// the worklist algorithm
|
||||
trait Nodelike
|
||||
pub trait Nodelike
|
||||
where
|
||||
Self: Sized,
|
||||
Self: Clone + std::fmt::Debug + Default + std::fmt::Display + Sized,
|
||||
{
|
||||
/// Number of "nodes" in the given value. This should correspond to the
|
||||
/// number of inodes needed to accommodate the value.
|
||||
@ -317,203 +188,19 @@ where
|
||||
fn from_string(typ: Typ, v: String, config: &Config) -> Self;
|
||||
fn from_list_dir(files: Vec<Self>, config: &Config) -> Self;
|
||||
fn from_named_dir(files: HashMap<String, Self>, config: &Config) -> Self;
|
||||
|
||||
/// Loading
|
||||
fn from_reader(reader: Box<dyn std::io::Read>) -> Self;
|
||||
|
||||
/// Saving, with optional pretty printing
|
||||
fn to_writer(&self, writer: Box<dyn std::io::Write>, pretty: bool);
|
||||
}
|
||||
|
||||
/// Given a `Nodelike` value `v`, initializes the vector `inodes` of (nullable)
|
||||
/// `Inodes` according to a given `config`.
|
||||
///
|
||||
/// The current implementation is eager: it preallocates enough inodes and then
|
||||
/// fills them in using a depth-first traversal.
|
||||
///
|
||||
/// Invariant: the index in the vector is the inode number. Inode 0 is invalid,
|
||||
/// and is left empty.
|
||||
fn fs_from_value<V>(v: V, config: &Config, inodes: &mut Vec<Option<Inode>>)
|
||||
where
|
||||
V: Nodelike + std::fmt::Display,
|
||||
{
|
||||
// reserve space for everyone else
|
||||
// won't work with streaming or lazy generation, but avoids having to resize the vector midway through
|
||||
inodes.resize_with(v.size() + 1, || None);
|
||||
info!("allocated {} inodes", inodes.len());
|
||||
|
||||
if v.kind() != FileType::Directory {
|
||||
error!("The root of the filesystem must be a directory, but '{}' only generates a single file.", v);
|
||||
std::process::exit(ERROR_STATUS_FUSE);
|
||||
}
|
||||
|
||||
let mut filtered = 0;
|
||||
let mut next_id = fuser::FUSE_ROOT_ID;
|
||||
// parent inum, inum, value
|
||||
let mut worklist: Vec<(u64, u64, V)> = vec![(next_id, next_id, v)];
|
||||
|
||||
next_id += 1;
|
||||
|
||||
while !worklist.is_empty() {
|
||||
let (parent, inum, v) = worklist.pop().unwrap();
|
||||
|
||||
let entry = match v.node(config) {
|
||||
Node::Bytes(b) => Entry::File(Typ::Bytes, b),
|
||||
Node::String(t, s) => Entry::File(t, s.into_bytes()),
|
||||
Node::List(vs) => {
|
||||
let mut children = HashMap::new();
|
||||
children.reserve(vs.len());
|
||||
|
||||
let num_elts = vs.len() as f64;
|
||||
let width = num_elts.log10().ceil() as usize;
|
||||
|
||||
for (i, child) in vs.into_iter().enumerate() {
|
||||
// TODO 2021-06-08 ability to add prefixes
|
||||
let name = if config.pad_element_names {
|
||||
format!("{:0width$}", i, width = width)
|
||||
} else {
|
||||
format!("{}", i)
|
||||
};
|
||||
|
||||
children.insert(
|
||||
name,
|
||||
DirEntry {
|
||||
kind: child.kind(),
|
||||
original_name: None,
|
||||
inum: next_id,
|
||||
},
|
||||
);
|
||||
worklist.push((inum, next_id, child));
|
||||
next_id += 1;
|
||||
}
|
||||
|
||||
Entry::Directory(DirType::List, children)
|
||||
}
|
||||
Node::Map(fvs) => {
|
||||
let mut children = HashMap::new();
|
||||
children.reserve(fvs.len());
|
||||
|
||||
for (field, child) in fvs.into_iter() {
|
||||
let original = field.clone();
|
||||
|
||||
let nfield = if !config.valid_name(&original) {
|
||||
match config.munge {
|
||||
Munge::Rename => {
|
||||
let mut nfield = config.normalize_name(field);
|
||||
|
||||
// TODO 2021-07-08 could be better to check fvs, but it's a vec now... :/
|
||||
while children.contains_key(&nfield) {
|
||||
nfield.push('_');
|
||||
}
|
||||
|
||||
nfield
|
||||
}
|
||||
Munge::Filter => {
|
||||
warn!("skipping '{}'", field);
|
||||
filtered += child.size();
|
||||
continue;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
field
|
||||
};
|
||||
|
||||
let original_name = if original != nfield {
|
||||
info!(
|
||||
"renamed {} to {} (inode {} with parent {})",
|
||||
original, nfield, next_id, parent
|
||||
);
|
||||
Some(original)
|
||||
} else {
|
||||
assert!(config.valid_name(&original));
|
||||
None
|
||||
};
|
||||
|
||||
children.insert(
|
||||
nfield,
|
||||
DirEntry {
|
||||
kind: child.kind(),
|
||||
original_name,
|
||||
inum: next_id,
|
||||
},
|
||||
);
|
||||
|
||||
worklist.push((inum, next_id, child));
|
||||
next_id += 1;
|
||||
}
|
||||
|
||||
Entry::Directory(DirType::Named, children)
|
||||
}
|
||||
};
|
||||
|
||||
inodes[inum as usize] = Some(Inode::new(parent, inum, entry, config));
|
||||
}
|
||||
|
||||
assert_eq!((inodes.len() - filtered) as u64, next_id);
|
||||
}
|
||||
|
||||
/// Walks `fs` starting at the inode with number `inum`, producing an
|
||||
/// appropriate value.
|
||||
fn value_from_fs<V>(fs: &FS, inum: u64) -> V
|
||||
where
|
||||
V: Nodelike,
|
||||
{
|
||||
match &fs.get(inum).unwrap().entry {
|
||||
Entry::File(typ, contents) => {
|
||||
// TODO 2021-07-01 use _t to try to force the type
|
||||
match String::from_utf8(contents.clone()) {
|
||||
Ok(mut contents) if typ != &Typ::Bytes => {
|
||||
if fs.config.add_newlines && contents.ends_with('\n') {
|
||||
contents.truncate(contents.len() - 1);
|
||||
}
|
||||
// TODO 2021-06-24 trim?
|
||||
V::from_string(*typ, contents, &fs.config)
|
||||
}
|
||||
Ok(_) | Err(_) => V::from_bytes(contents, &fs.config),
|
||||
}
|
||||
}
|
||||
Entry::Directory(DirType::List, files) => {
|
||||
let mut entries = Vec::with_capacity(files.len());
|
||||
|
||||
let mut files = files.iter().collect::<Vec<_>>();
|
||||
files.sort_unstable_by(|(name1, _), (name2, _)| name1.cmp(name2));
|
||||
for (name, DirEntry { inum, .. }) in files.iter() {
|
||||
if fs.config.ignored_file(name) {
|
||||
warn!("skipping ignored file '{}'", name);
|
||||
continue;
|
||||
}
|
||||
|
||||
let v = value_from_fs(fs, *inum);
|
||||
entries.push(v);
|
||||
}
|
||||
|
||||
V::from_list_dir(entries, &fs.config)
|
||||
}
|
||||
Entry::Directory(DirType::Named, files) => {
|
||||
let mut entries = HashMap::with_capacity(files.len());
|
||||
|
||||
for (
|
||||
name,
|
||||
DirEntry {
|
||||
inum,
|
||||
original_name,
|
||||
..
|
||||
},
|
||||
) in files.iter()
|
||||
{
|
||||
if fs.config.ignored_file(name) {
|
||||
warn!("skipping ignored file '{}'", name);
|
||||
continue;
|
||||
}
|
||||
|
||||
let v = value_from_fs(fs, *inum);
|
||||
|
||||
let name = original_name.as_ref().unwrap_or(name).into();
|
||||
entries.insert(name, v);
|
||||
}
|
||||
|
||||
V::from_named_dir(entries, &fs.config)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mod json {
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
/// JSON Nodelike implementation
|
||||
pub mod json {
|
||||
use super::*;
|
||||
use serde_json::Value;
|
||||
pub use serde_json::Value;
|
||||
|
||||
impl Nodelike for Value {
|
||||
/// `Value::Object` and `Value::Array` map to directories; everything else is a
|
||||
@ -623,72 +310,74 @@ mod json {
|
||||
fn from_named_dir(files: HashMap<String, Self>, _config: &Config) -> Self {
|
||||
Value::Object(files.into_iter().collect())
|
||||
}
|
||||
|
||||
fn to_writer(&self, writer: Box<dyn std::io::Write>, pretty: bool) {
|
||||
if pretty {
|
||||
serde_json::to_writer_pretty(writer, self).unwrap();
|
||||
} else {
|
||||
serde_json::to_writer(writer, self).unwrap();
|
||||
}
|
||||
}
|
||||
fn from_reader(reader: std::boxed::Box<dyn std::io::Read>) -> Self {
|
||||
serde_json::from_reader(reader).expect("JSON")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mod toml {
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
/// TOML Nodelike implementation
|
||||
pub mod toml {
|
||||
use super::*;
|
||||
use serde_toml::Value;
|
||||
use serde_toml::Value as Toml;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum Error<E> {
|
||||
Io(std::io::Error),
|
||||
Toml(E),
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Value(serde_toml::Value);
|
||||
|
||||
impl std::fmt::Display for Value {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::result::Result<(), std::fmt::Error> {
|
||||
self.0.fmt(f)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_reader(
|
||||
mut reader: Box<dyn std::io::Read>,
|
||||
) -> Result<Value, Error<serde_toml::de::Error>> {
|
||||
let mut text = String::new();
|
||||
let _len = reader.read_to_string(&mut text).map_err(Error::Io)?;
|
||||
serde_toml::from_str(&text).map_err(Error::Toml)
|
||||
impl Default for Value {
|
||||
fn default() -> Self {
|
||||
Value(Toml::String("".into()))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn to_writer(
|
||||
mut writer: Box<dyn std::io::Write>,
|
||||
v: &Value,
|
||||
) -> Result<(), Error<serde_toml::ser::Error>> {
|
||||
let text = serde_toml::to_string(v).map_err(Error::Toml)?;
|
||||
writer.write_all(text.as_bytes()).map_err(Error::Io)
|
||||
}
|
||||
|
||||
pub fn to_writer_pretty(
|
||||
mut writer: Box<dyn std::io::Write>,
|
||||
v: &Value,
|
||||
) -> Result<(), Error<serde_toml::ser::Error>> {
|
||||
let text = serde_toml::to_string_pretty(v).map_err(Error::Toml)?;
|
||||
writer.write_all(text.as_bytes()).map_err(Error::Io)
|
||||
fn toml_size(v: &Toml) -> usize {
|
||||
match v {
|
||||
Toml::Boolean(_)
|
||||
| Toml::Datetime(_)
|
||||
| Toml::Float(_)
|
||||
| Toml::Integer(_)
|
||||
| Toml::String(_) => 1,
|
||||
Toml::Array(vs) => vs.iter().map(|v| toml_size(v)).sum::<usize>() + 1,
|
||||
Toml::Table(fvs) => fvs.iter().map(|(_, v)| toml_size(v)).sum::<usize>() + 1,
|
||||
}
|
||||
}
|
||||
|
||||
impl Nodelike for Value {
|
||||
fn kind(&self) -> FileType {
|
||||
match self {
|
||||
Value::Table(_) | Value::Array(_) => FileType::Directory,
|
||||
match self.0 {
|
||||
Toml::Table(_) | Toml::Array(_) => FileType::Directory,
|
||||
_ => FileType::RegularFile,
|
||||
}
|
||||
}
|
||||
|
||||
fn size(&self) -> usize {
|
||||
match self {
|
||||
Value::Boolean(_)
|
||||
| Value::Datetime(_)
|
||||
| Value::Float(_)
|
||||
| Value::Integer(_)
|
||||
| Value::String(_) => 1,
|
||||
Value::Array(vs) => vs.iter().map(|v| v.size()).sum::<usize>() + 1,
|
||||
Value::Table(fvs) => fvs.iter().map(|(_, v)| v.size()).sum::<usize>() + 1,
|
||||
}
|
||||
toml_size(&self.0)
|
||||
}
|
||||
|
||||
fn node(self, config: &Config) -> Node<Self> {
|
||||
let nl = if config.add_newlines { "\n" } else { "" };
|
||||
|
||||
match self {
|
||||
Value::Boolean(b) => Node::String(Typ::Boolean, format!("{}{}", b, nl)),
|
||||
Value::Datetime(s) => Node::String(Typ::Datetime, s.to_string()),
|
||||
Value::Float(n) => Node::String(Typ::Float, format!("{}{}", n, nl)),
|
||||
Value::Integer(n) => Node::String(Typ::Integer, format!("{}{}", n, nl)),
|
||||
Value::String(s) => {
|
||||
match self.0 {
|
||||
Toml::Boolean(b) => Node::String(Typ::Boolean, format!("{}{}", b, nl)),
|
||||
Toml::Datetime(s) => Node::String(Typ::Datetime, s.to_string()),
|
||||
Toml::Float(n) => Node::String(Typ::Float, format!("{}{}", n, nl)),
|
||||
Toml::Integer(n) => Node::String(Typ::Integer, format!("{}{}", n, nl)),
|
||||
Toml::String(s) => {
|
||||
if config.try_decode_base64 {
|
||||
if let Ok(bytes) = base64::decode_config(&s, config.base64) {
|
||||
return Node::Bytes(bytes);
|
||||
@ -697,132 +386,125 @@ mod toml {
|
||||
|
||||
Node::String(Typ::String, if s.ends_with('\n') { s } else { s + nl })
|
||||
}
|
||||
Value::Array(vs) => Node::List(vs),
|
||||
Value::Table(fvs) => Node::Map(fvs.into_iter().collect()),
|
||||
Toml::Array(vs) => Node::List(vs.into_iter().map(Value).collect()),
|
||||
Toml::Table(fvs) => {
|
||||
Node::Map(fvs.into_iter().map(|(f, v)| (f, Value(v))).collect())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn from_string(typ: Typ, contents: String, _config: &Config) -> Self {
|
||||
match typ {
|
||||
let v = match typ {
|
||||
Typ::Auto => {
|
||||
if contents == "true" {
|
||||
Value::Boolean(true)
|
||||
Toml::Boolean(true)
|
||||
} else if contents == "false" {
|
||||
Value::Boolean(false)
|
||||
Toml::Boolean(false)
|
||||
} else if let Ok(n) = i64::from_str(&contents) {
|
||||
Value::Integer(n)
|
||||
Toml::Integer(n)
|
||||
} else if let Ok(n) = f64::from_str(&contents) {
|
||||
Value::Float(n)
|
||||
Toml::Float(n)
|
||||
} else if let Ok(datetime) = str::parse(&contents) {
|
||||
Value::Datetime(datetime)
|
||||
Toml::Datetime(datetime)
|
||||
} else {
|
||||
Value::String(contents)
|
||||
Toml::String(contents)
|
||||
}
|
||||
}
|
||||
Typ::Boolean => {
|
||||
if contents == "true" {
|
||||
Value::Boolean(true)
|
||||
Toml::Boolean(true)
|
||||
} else if contents == "false" {
|
||||
Value::Boolean(false)
|
||||
Toml::Boolean(false)
|
||||
} else {
|
||||
debug!("string '{}' tagged as boolean", contents);
|
||||
Value::String(contents)
|
||||
Toml::String(contents)
|
||||
}
|
||||
}
|
||||
Typ::Bytes => panic!("from_string called at typ::bytes"),
|
||||
Typ::Datetime => match str::parse(&contents) {
|
||||
Ok(datetime) => Value::Datetime(datetime),
|
||||
Ok(datetime) => Toml::Datetime(datetime),
|
||||
Err(e) => {
|
||||
debug!(
|
||||
"string '{}' tagged as datetime, didn't parse: {}",
|
||||
contents, e
|
||||
);
|
||||
Value::String(contents)
|
||||
Toml::String(contents)
|
||||
}
|
||||
},
|
||||
Typ::Float => {
|
||||
if let Ok(n) = f64::from_str(&contents) {
|
||||
Value::Float(n)
|
||||
Toml::Float(n)
|
||||
} else {
|
||||
debug!("string '{}' tagged as float", contents);
|
||||
Value::String(contents)
|
||||
Toml::String(contents)
|
||||
}
|
||||
}
|
||||
Typ::Integer => {
|
||||
if let Ok(n) = i64::from_str(&contents) {
|
||||
Value::Integer(n)
|
||||
Toml::Integer(n)
|
||||
} else {
|
||||
debug!("string '{}' tagged as float", contents);
|
||||
Value::String(contents)
|
||||
Toml::String(contents)
|
||||
}
|
||||
}
|
||||
Typ::Null => {
|
||||
if contents.is_empty() {
|
||||
Value::String(contents)
|
||||
Toml::String(contents)
|
||||
} else {
|
||||
debug!("string '{}' tagged as null", contents);
|
||||
Value::String(contents)
|
||||
Toml::String(contents)
|
||||
}
|
||||
}
|
||||
Typ::String => Value::String(contents),
|
||||
}
|
||||
Typ::String => Toml::String(contents),
|
||||
};
|
||||
|
||||
Value(v)
|
||||
}
|
||||
|
||||
fn from_bytes<T>(contents: T, config: &Config) -> Self
|
||||
where
|
||||
T: AsRef<[u8]>,
|
||||
{
|
||||
Value::String(base64::encode_config(contents, config.base64))
|
||||
Value(Toml::String(base64::encode_config(contents, config.base64)))
|
||||
}
|
||||
|
||||
fn from_list_dir(files: Vec<Self>, _config: &Config) -> Self {
|
||||
Value::Array(files)
|
||||
Value(Toml::Array(files.into_iter().map(|v| v.0).collect()))
|
||||
}
|
||||
|
||||
fn from_named_dir(files: HashMap<String, Self>, _config: &Config) -> Self {
|
||||
Value::Table(files.into_iter().collect())
|
||||
Value(Toml::Table(
|
||||
files.into_iter().map(|(f, v)| (f, v.0)).collect(),
|
||||
))
|
||||
}
|
||||
|
||||
fn from_reader(mut reader: Box<dyn std::io::Read>) -> Self {
|
||||
let mut text = String::new();
|
||||
let _len = reader.read_to_string(&mut text).unwrap();
|
||||
Value(serde_toml::from_str(&text).expect("TOML"))
|
||||
}
|
||||
|
||||
fn to_writer(&self, mut writer: Box<dyn std::io::Write>, pretty: bool) {
|
||||
let text = if pretty {
|
||||
serde_toml::to_string_pretty(&self.0).unwrap()
|
||||
} else {
|
||||
serde_toml::to_string(&self.0).unwrap()
|
||||
};
|
||||
writer.write_all(text.as_bytes()).unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mod yaml {
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
/// YAML Nodelike implementation
|
||||
pub mod yaml {
|
||||
use super::*;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use yaml_rust::{EmitError, ScanError, Yaml};
|
||||
use yaml_rust::Yaml;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Value(Yaml);
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum Error<E> {
|
||||
Io(std::io::Error),
|
||||
Yaml(E),
|
||||
}
|
||||
|
||||
pub fn from_reader(mut reader: Box<dyn std::io::Read>) -> Result<Value, Error<ScanError>> {
|
||||
let mut text = String::new();
|
||||
let _len = reader.read_to_string(&mut text).map_err(Error::Io)?;
|
||||
yaml_rust::YamlLoader::load_from_str(&text)
|
||||
.map(|vs| {
|
||||
Value(if vs.len() == 1 {
|
||||
vs.into_iter().next().unwrap()
|
||||
} else {
|
||||
Yaml::Array(vs)
|
||||
})
|
||||
})
|
||||
.map_err(Error::Yaml)
|
||||
}
|
||||
|
||||
pub fn to_writer(
|
||||
mut writer: Box<dyn std::io::Write>,
|
||||
v: &Value,
|
||||
) -> Result<(), Error<EmitError>> {
|
||||
let mut text = String::new();
|
||||
let mut emitter = yaml_rust::YamlEmitter::new(&mut text);
|
||||
emitter.dump(&v.0).map_err(Error::Yaml)?;
|
||||
writer.write_all(text.as_bytes()).map_err(Error::Io)
|
||||
}
|
||||
|
||||
impl std::fmt::Display for Value {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::result::Result<(), std::fmt::Error> {
|
||||
let mut emitter = yaml_rust::YamlEmitter::new(f);
|
||||
@ -835,6 +517,12 @@ mod yaml {
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Value {
|
||||
fn default() -> Self {
|
||||
Value(Yaml::Null)
|
||||
}
|
||||
}
|
||||
|
||||
fn yaml_size(v: &Yaml) -> usize {
|
||||
match v {
|
||||
Yaml::Real(_)
|
||||
@ -987,5 +675,26 @@ mod yaml {
|
||||
.collect(),
|
||||
))
|
||||
}
|
||||
|
||||
fn from_reader(mut reader: Box<dyn std::io::Read>) -> Self {
|
||||
let mut text = String::new();
|
||||
let _len = reader.read_to_string(&mut text).unwrap();
|
||||
yaml_rust::YamlLoader::load_from_str(&text)
|
||||
.map(|vs| {
|
||||
Value(if vs.len() == 1 {
|
||||
vs.into_iter().next().unwrap()
|
||||
} else {
|
||||
Yaml::Array(vs)
|
||||
})
|
||||
})
|
||||
.expect("YAML")
|
||||
}
|
||||
|
||||
fn to_writer(&self, mut writer: Box<dyn std::io::Write>, _pretty: bool) {
|
||||
let mut text = String::new();
|
||||
let mut emitter = yaml_rust::YamlEmitter::new(&mut text);
|
||||
emitter.dump(&self.0).unwrap();
|
||||
writer.write_all(text.as_bytes()).unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
548
src/fs.rs
548
src/fs.rs
@ -1,6 +1,8 @@
|
||||
use std::cell::Cell;
|
||||
use std::collections::HashMap;
|
||||
use std::ffi::OsStr;
|
||||
use std::fmt::{Debug, Display};
|
||||
use std::mem;
|
||||
use std::path::Path;
|
||||
use std::str::FromStr;
|
||||
use std::time::{Duration, SystemTime};
|
||||
@ -14,19 +16,23 @@ use fuser::{
|
||||
#[cfg(target_os = "macos")]
|
||||
use fuser::ReplyXTimes;
|
||||
|
||||
use tracing::{info, instrument, trace, warn};
|
||||
use tracing::{debug, error, info, instrument, trace, warn};
|
||||
|
||||
use super::config::{Config, Output};
|
||||
use super::format::Typ;
|
||||
use super::config::{Config, Munge, Output, ERROR_STATUS_FUSE};
|
||||
use super::format::{json, toml, yaml, Format, Node, Nodelike, Typ};
|
||||
use crate::time_ns;
|
||||
|
||||
/// A filesystem `FS` is just a vector of nullable inodes, where the index is
|
||||
/// the inode number.
|
||||
///
|
||||
/// NB that inode 0 is always invalid.
|
||||
#[derive(Debug)]
|
||||
pub struct FS {
|
||||
pub struct FS<V>
|
||||
where
|
||||
V: Nodelike + Clone + Debug + std::fmt::Display,
|
||||
{
|
||||
/// Vector of nullable inodes; the index is the inode number.
|
||||
pub inodes: Vec<Option<Inode>>,
|
||||
pub inodes: Vec<Option<Inode<V>>>,
|
||||
/// Configuration, which determines various file attributes.
|
||||
pub config: Config,
|
||||
/// Dirty bit: set to `true` when there are outstanding writes
|
||||
@ -40,7 +46,7 @@ const TTL: Duration = Duration::from_secs(300);
|
||||
|
||||
/// An inode, the core structure in the filesystem.
|
||||
#[derive(Debug)]
|
||||
pub struct Inode {
|
||||
pub struct Inode<V> {
|
||||
/// Inode number of the parent of the current inode.
|
||||
///
|
||||
/// For the root, it will be `FUSE_ROOT_ID`, i.e., itself.
|
||||
@ -63,7 +69,7 @@ pub struct Inode {
|
||||
/// Time of creation (macOS only)
|
||||
pub crtime: SystemTime,
|
||||
/// The actual file contents.
|
||||
pub entry: Entry,
|
||||
pub entry: Entry<V>,
|
||||
}
|
||||
|
||||
/// File contents. Either a `File` containing bytes or a `Directory`, mapping
|
||||
@ -75,16 +81,17 @@ pub struct Inode {
|
||||
/// generated (see `format::fs_from_value`). When writing a `DirType::List`
|
||||
/// directory back out, only the sort order of the name matters.
|
||||
#[derive(Debug)]
|
||||
pub enum Entry {
|
||||
pub enum Entry<V> {
|
||||
// TODO 2021-06-14 need a 'written' flag to determine whether or not to
|
||||
// strip newlines during writeback
|
||||
File(Typ, Vec<u8>),
|
||||
Directory(DirType, HashMap<String, DirEntry>),
|
||||
Lazy(V),
|
||||
}
|
||||
|
||||
/// Directory entries. We record the kind and inode (for faster
|
||||
/// `Filesystem::readdir`).
|
||||
#[derive(Debug)]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct DirEntry {
|
||||
pub kind: FileType,
|
||||
/// When loading from certain map types, names might get munged.
|
||||
@ -107,17 +114,11 @@ pub enum FSError {
|
||||
InvalidInode(u64),
|
||||
}
|
||||
|
||||
impl FS {
|
||||
pub fn new(inodes: Vec<Option<Inode>>, config: Config) -> Self {
|
||||
FS {
|
||||
inodes,
|
||||
config,
|
||||
dirty: Cell::new(false),
|
||||
synced: Cell::new(false),
|
||||
}
|
||||
}
|
||||
|
||||
fn fresh_inode(&mut self, parent: u64, entry: Entry, uid: u32, gid: u32, mode: u32) -> u64 {
|
||||
impl<V> FS<V>
|
||||
where
|
||||
V: Nodelike + Clone + Debug + Display + Default,
|
||||
{
|
||||
fn fresh_inode(&mut self, parent: u64, entry: Entry<V>, uid: u32, gid: u32, mode: u32) -> u64 {
|
||||
self.dirty.set(true);
|
||||
|
||||
let inum = self.inodes.len() as u64;
|
||||
@ -129,11 +130,177 @@ impl FS {
|
||||
inum
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
fn resolve_node(&mut self, inum: u64) -> Result<Option<Vec<u64>>, FSError>
|
||||
where
|
||||
V: Nodelike + std::fmt::Display + Default,
|
||||
{
|
||||
debug!("called");
|
||||
|
||||
let idx = inum as usize;
|
||||
|
||||
if idx >= self.inodes.len() || idx == 0 {
|
||||
return Err(FSError::NoSuchInode(inum));
|
||||
}
|
||||
|
||||
let inode = match &mut self.inodes[idx] {
|
||||
Some(inode) => inode,
|
||||
_ => return Err(FSError::InvalidInode(inum)),
|
||||
};
|
||||
|
||||
let v = match &mut inode.entry {
|
||||
Entry::Directory(..) | Entry::File(..) => return Ok(Option::None),
|
||||
Entry::Lazy(v) => mem::take(v),
|
||||
};
|
||||
let uid = inode.uid;
|
||||
let gid = inode.gid;
|
||||
|
||||
let (entry, new_nodes) = match v.node(&self.config) {
|
||||
Node::Bytes(b) => (Entry::File(Typ::Bytes, b), Option::None),
|
||||
Node::String(t, s) => (Entry::File(t, s.into_bytes()), Option::None),
|
||||
Node::List(vs) => {
|
||||
let mut children = HashMap::new();
|
||||
children.reserve(vs.len());
|
||||
let num_elts = vs.len() as f64;
|
||||
let width = num_elts.log10().ceil() as usize;
|
||||
|
||||
let mut new_nodes = Vec::with_capacity(vs.len());
|
||||
for (i, child) in vs.into_iter().enumerate() {
|
||||
// TODO 2021-06-08 ability to add prefixes
|
||||
let name = if self.config.pad_element_names {
|
||||
format!("{:0width$}", i, width = width)
|
||||
} else {
|
||||
format!("{}", i)
|
||||
};
|
||||
|
||||
let kind = child.kind();
|
||||
let child_id = self.fresh_inode(
|
||||
inum,
|
||||
Entry::Lazy(child),
|
||||
uid,
|
||||
gid,
|
||||
self.config.mode(kind) as u32,
|
||||
);
|
||||
|
||||
children.insert(
|
||||
name,
|
||||
DirEntry {
|
||||
kind,
|
||||
original_name: None,
|
||||
inum: child_id,
|
||||
},
|
||||
);
|
||||
new_nodes.push(child_id)
|
||||
}
|
||||
|
||||
(
|
||||
Entry::Directory(DirType::List, children),
|
||||
Option::Some(new_nodes),
|
||||
)
|
||||
}
|
||||
Node::Map(fvs) => {
|
||||
let mut children = HashMap::new();
|
||||
children.reserve(fvs.len());
|
||||
|
||||
let mut new_nodes = Vec::with_capacity(fvs.len());
|
||||
for (field, child) in fvs.into_iter() {
|
||||
let original = field.clone();
|
||||
|
||||
let nfield = if !self.config.valid_name(&original) {
|
||||
match self.config.munge {
|
||||
Munge::Rename => {
|
||||
let mut nfield = self.config.normalize_name(field);
|
||||
|
||||
// TODO 2021-07-08 could be better to check fvs, but it's a vec now... :/
|
||||
while children.contains_key(&nfield) {
|
||||
nfield.push('_');
|
||||
}
|
||||
|
||||
nfield
|
||||
}
|
||||
Munge::Filter => {
|
||||
warn!("skipping '{}'", field);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
field
|
||||
};
|
||||
|
||||
let kind = child.kind();
|
||||
let child_id = self.fresh_inode(
|
||||
inum,
|
||||
Entry::Lazy(child),
|
||||
uid,
|
||||
gid,
|
||||
self.config.mode(kind) as u32,
|
||||
);
|
||||
let original_name = if original != nfield {
|
||||
info!(
|
||||
"renamed {} to {} (inode {} with parent {})",
|
||||
original, nfield, child_id, inum
|
||||
);
|
||||
Some(original)
|
||||
} else {
|
||||
assert!(self.config.valid_name(&original));
|
||||
None
|
||||
};
|
||||
|
||||
children.insert(
|
||||
nfield,
|
||||
DirEntry {
|
||||
kind,
|
||||
original_name,
|
||||
inum: child_id,
|
||||
},
|
||||
);
|
||||
|
||||
new_nodes.push(child_id);
|
||||
}
|
||||
|
||||
(
|
||||
Entry::Directory(DirType::Named, children),
|
||||
Option::Some(new_nodes),
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
let inode = match &mut self.inodes[idx] {
|
||||
Some(inode) => inode,
|
||||
_ => return Err(FSError::InvalidInode(inum)),
|
||||
};
|
||||
inode.entry = entry;
|
||||
|
||||
if let Some(nodes) = &new_nodes {
|
||||
debug!("new_nodes = {:?}", nodes);
|
||||
}
|
||||
|
||||
Ok(new_nodes)
|
||||
}
|
||||
|
||||
fn resolve_nodes_transitively(&mut self, inum: u64) -> Result<(), FSError> {
|
||||
let mut worklist = match self.resolve_node(inum)? {
|
||||
Some(nodes) => nodes,
|
||||
None => return Ok(()),
|
||||
};
|
||||
|
||||
while !worklist.is_empty() {
|
||||
let node = worklist.pop().unwrap();
|
||||
if let Some(nodes) = self.resolve_node(node)? {
|
||||
worklist.extend(nodes);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check_access(&self, req: &Request) -> bool {
|
||||
req.uid() == 0 || req.uid() == self.config.uid
|
||||
}
|
||||
|
||||
pub fn get(&self, inum: u64) -> Result<&Inode, FSError> {
|
||||
pub fn get(&mut self, inum: u64) -> Result<&Inode<V>, FSError> {
|
||||
let _new_nodes = self.resolve_node(inum)?;
|
||||
|
||||
let idx = inum as usize;
|
||||
|
||||
if idx >= self.inodes.len() || idx == 0 {
|
||||
@ -146,7 +313,9 @@ impl FS {
|
||||
}
|
||||
}
|
||||
|
||||
fn get_mut(&mut self, inum: u64) -> Result<&mut Inode, FSError> {
|
||||
fn get_mut(&mut self, inum: u64) -> Result<&mut Inode<V>, FSError> {
|
||||
let _new_nodes = self.resolve_node(inum)?;
|
||||
|
||||
let idx = inum as usize;
|
||||
|
||||
if idx >= self.inodes.len() {
|
||||
@ -159,6 +328,69 @@ impl FS {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn new(config: Config) -> Self {
|
||||
info!("loading");
|
||||
let mut inodes: Vec<Option<Inode<V>>> = Vec::with_capacity(1024);
|
||||
// allocate space for dummy inode 0, root node
|
||||
inodes.resize_with(2, || None);
|
||||
|
||||
let reader = match config.input_reader() {
|
||||
Some(reader) => reader,
|
||||
None => {
|
||||
// create an empty directory
|
||||
let contents = HashMap::with_capacity(16);
|
||||
inodes[1] = Some(Inode::new(
|
||||
fuser::FUSE_ROOT_ID,
|
||||
fuser::FUSE_ROOT_ID,
|
||||
Entry::Directory(DirType::Named, contents),
|
||||
&config,
|
||||
));
|
||||
return FS {
|
||||
inodes,
|
||||
config,
|
||||
dirty: Cell::new(false),
|
||||
synced: Cell::new(false),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
let v = time_ns!("reading", V::from_reader(reader), config.timing);
|
||||
if v.kind() != FileType::Directory {
|
||||
error!("The root of the filesystem must be a directory, but '{}' only generates a single file.", v);
|
||||
std::process::exit(ERROR_STATUS_FUSE);
|
||||
}
|
||||
|
||||
let mut fs = FS {
|
||||
inodes,
|
||||
config,
|
||||
dirty: Cell::new(false),
|
||||
synced: Cell::new(false),
|
||||
};
|
||||
|
||||
time_ns!(
|
||||
"loading",
|
||||
{
|
||||
fs.inodes[fuser::FUSE_ROOT_ID as usize] = Option::Some(Inode::new(
|
||||
fuser::FUSE_ROOT_ID,
|
||||
fuser::FUSE_ROOT_ID,
|
||||
Entry::Lazy(v),
|
||||
&fs.config,
|
||||
));
|
||||
|
||||
if fs.config.eager {
|
||||
fs.resolve_nodes_transitively(fuser::FUSE_ROOT_ID)
|
||||
.expect("resolve_nodes_transitively");
|
||||
} else {
|
||||
// kick start the root directory
|
||||
fs.resolve_node(fuser::FUSE_ROOT_ID).expect("resolve_node");
|
||||
}
|
||||
},
|
||||
fs.config.timing
|
||||
);
|
||||
|
||||
fs
|
||||
}
|
||||
|
||||
/// Tries to synchronize the in-memory `FS` with its on-disk representation.
|
||||
///
|
||||
/// Depending on output conventions and the state of the `FS`, nothing may
|
||||
@ -170,7 +402,7 @@ impl FS {
|
||||
/// - if `self.config.output == Output::Stdout` and `last_sync == false`,
|
||||
/// nothing will happen (to prevent redundant writes to STDOUT)
|
||||
#[instrument(level = "debug", skip(self), fields(synced = self.synced.get(), dirty = self.dirty.get()))]
|
||||
pub fn sync(&self, last_sync: bool) {
|
||||
pub fn sync(&mut self, last_sync: bool) {
|
||||
info!("called");
|
||||
trace!("{:?}", self.inodes);
|
||||
|
||||
@ -187,21 +419,216 @@ impl FS {
|
||||
_ => (),
|
||||
};
|
||||
|
||||
self.config.output_format.save(self);
|
||||
self.save();
|
||||
self.dirty.set(false);
|
||||
self.synced.set(true);
|
||||
}
|
||||
|
||||
/// Actually output results, using `self.config.output`.
|
||||
///
|
||||
/// When `self.config.input == self.config.output`, then resolved lazy nodes
|
||||
/// can be directly returned. If the input and output formats are different,
|
||||
/// we eager resolve everything and then save.
|
||||
fn save(&mut self) {
|
||||
let writer = match self.config.output_writer() {
|
||||
Some(writer) => writer,
|
||||
None => return,
|
||||
};
|
||||
|
||||
if self.config.input_format == self.config.output_format {
|
||||
let v = time_ns!(
|
||||
"saving",
|
||||
self.as_value(fuser::FUSE_ROOT_ID),
|
||||
self.config.timing
|
||||
);
|
||||
|
||||
time_ns!(
|
||||
"writing",
|
||||
v.to_writer(writer, self.config.pretty),
|
||||
self.config.timing
|
||||
);
|
||||
} else {
|
||||
match self.config.output_format {
|
||||
Format::Json => {
|
||||
let v: json::Value = time_ns!(
|
||||
"saving",
|
||||
self.as_other_value(fuser::FUSE_ROOT_ID),
|
||||
self.config.timing
|
||||
);
|
||||
|
||||
time_ns!(
|
||||
"writing",
|
||||
v.to_writer(writer, self.config.pretty),
|
||||
self.config.timing
|
||||
);
|
||||
}
|
||||
Format::Toml => {
|
||||
let v: toml::Value = time_ns!(
|
||||
"saving",
|
||||
self.as_other_value(fuser::FUSE_ROOT_ID),
|
||||
self.config.timing
|
||||
);
|
||||
|
||||
time_ns!(
|
||||
"writing",
|
||||
v.to_writer(writer, self.config.pretty),
|
||||
self.config.timing
|
||||
);
|
||||
}
|
||||
Format::Yaml => {
|
||||
let v: yaml::Value = time_ns!(
|
||||
"saving",
|
||||
self.as_other_value(fuser::FUSE_ROOT_ID),
|
||||
self.config.timing
|
||||
);
|
||||
|
||||
time_ns!(
|
||||
"writing",
|
||||
v.to_writer(writer, self.config.pretty),
|
||||
self.config.timing
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// save as a value of the same type as the input
|
||||
// we need this special case to avoid type-level shenanigans
|
||||
fn as_value(&self, inum: u64) -> V {
|
||||
match &self.inodes[inum as usize].as_ref().unwrap().entry {
|
||||
Entry::Lazy(v) => v.clone(),
|
||||
Entry::File(typ, contents) => {
|
||||
// TODO 2021-07-01 use _t to try to force the type
|
||||
match String::from_utf8(contents.clone()) {
|
||||
Ok(mut contents) if typ != &Typ::Bytes => {
|
||||
if self.config.add_newlines && contents.ends_with('\n') {
|
||||
contents.truncate(contents.len() - 1);
|
||||
}
|
||||
// TODO 2021-06-24 trim?
|
||||
V::from_string(*typ, contents, &self.config)
|
||||
}
|
||||
Ok(_) | Err(_) => V::from_bytes(contents, &self.config),
|
||||
}
|
||||
}
|
||||
Entry::Directory(DirType::List, files) => {
|
||||
let mut entries = Vec::with_capacity(files.len());
|
||||
let mut files = files.iter().collect::<Vec<_>>();
|
||||
files.sort_unstable_by(|(name1, _), (name2, _)| name1.cmp(name2));
|
||||
for (name, DirEntry { inum, .. }) in files.iter() {
|
||||
if self.config.ignored_file(name) {
|
||||
warn!("skipping ignored file '{}'", name);
|
||||
continue;
|
||||
}
|
||||
let v = self.as_value(*inum);
|
||||
entries.push(v);
|
||||
}
|
||||
V::from_list_dir(entries, &self.config)
|
||||
}
|
||||
Entry::Directory(DirType::Named, files) => {
|
||||
let mut entries = HashMap::with_capacity(files.len());
|
||||
for (
|
||||
name,
|
||||
DirEntry {
|
||||
inum,
|
||||
original_name,
|
||||
..
|
||||
},
|
||||
) in files.iter()
|
||||
{
|
||||
if self.config.ignored_file(name) {
|
||||
warn!("skipping ignored file '{}'", name);
|
||||
continue;
|
||||
}
|
||||
let v = self.as_value(*inum);
|
||||
let name = original_name.as_ref().unwrap_or(name).into();
|
||||
entries.insert(name, v);
|
||||
}
|
||||
V::from_named_dir(entries, &self.config)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(level = "trace", skip(self))]
|
||||
fn as_other_value<U>(&mut self, inum: u64) -> U
|
||||
where
|
||||
U: Nodelike,
|
||||
{
|
||||
match &self.inodes[inum as usize].as_ref().unwrap().entry {
|
||||
Entry::Lazy(_) => {
|
||||
self.resolve_nodes_transitively(inum).unwrap();
|
||||
self.as_other_value(inum)
|
||||
}
|
||||
Entry::File(typ, contents) => {
|
||||
// TODO 2021-07-01 use _t to try to force the type
|
||||
match String::from_utf8(contents.clone()) {
|
||||
Ok(mut contents) if typ != &Typ::Bytes => {
|
||||
if self.config.add_newlines && contents.ends_with('\n') {
|
||||
contents.truncate(contents.len() - 1);
|
||||
}
|
||||
// TODO 2021-06-24 trim?
|
||||
U::from_string(*typ, contents, &self.config)
|
||||
}
|
||||
Ok(_) | Err(_) => U::from_bytes(contents, &self.config),
|
||||
}
|
||||
}
|
||||
Entry::Directory(DirType::List, files) => {
|
||||
let mut entries = Vec::with_capacity(files.len());
|
||||
let mut files = files
|
||||
.iter()
|
||||
.map(|(name, entry)| (name.clone(), entry.inum))
|
||||
.collect::<Vec<_>>();
|
||||
files.sort_unstable_by(|(name1, _), (name2, _)| name1.cmp(name2));
|
||||
for (name, inum) in files {
|
||||
if self.config.ignored_file(&name) {
|
||||
warn!("skipping ignored file '{}'", name);
|
||||
continue;
|
||||
}
|
||||
let v = self.as_other_value(inum);
|
||||
entries.push(v);
|
||||
}
|
||||
U::from_list_dir(entries, &self.config)
|
||||
}
|
||||
Entry::Directory(DirType::Named, files) => {
|
||||
let mut entries = HashMap::with_capacity(files.len());
|
||||
|
||||
let files = files
|
||||
.iter()
|
||||
.map(|(name, entry)| (name.clone(), entry.inum, entry.original_name.clone()))
|
||||
.collect::<Vec<_>>();
|
||||
for (name, inum, original_name) in files.iter() {
|
||||
if self.config.ignored_file(name) {
|
||||
warn!("skipping ignored file '{}'", name);
|
||||
continue;
|
||||
}
|
||||
let v = self.as_other_value(*inum);
|
||||
let name = original_name.as_ref().unwrap_or(name).into();
|
||||
entries.insert(name, v);
|
||||
}
|
||||
U::from_named_dir(entries, &self.config)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Inode {
|
||||
pub fn new(parent: u64, inum: u64, entry: Entry, config: &Config) -> Self {
|
||||
impl<V> Inode<V>
|
||||
where
|
||||
V: Nodelike,
|
||||
{
|
||||
pub fn new(parent: u64, inum: u64, entry: Entry<V>, config: &Config) -> Self {
|
||||
let mode = config.mode(entry.kind());
|
||||
let uid = config.uid;
|
||||
let gid = config.gid;
|
||||
Inode::with_mode(parent, inum, entry, uid, gid, mode)
|
||||
}
|
||||
|
||||
pub fn with_mode(parent: u64, inum: u64, entry: Entry, uid: u32, gid: u32, mode: u16) -> Self {
|
||||
pub fn with_mode(
|
||||
parent: u64,
|
||||
inum: u64,
|
||||
entry: Entry<V>,
|
||||
uid: u32,
|
||||
gid: u32,
|
||||
mode: u16,
|
||||
) -> Self {
|
||||
let now = SystemTime::now();
|
||||
|
||||
Inode {
|
||||
@ -232,6 +659,7 @@ impl Inode {
|
||||
.count() as u32
|
||||
}
|
||||
Entry::File(..) => 1,
|
||||
Entry::Lazy(..) => unreachable!("unresolved lazy value in Inode::attr"),
|
||||
};
|
||||
|
||||
FileAttr {
|
||||
@ -254,7 +682,10 @@ impl Inode {
|
||||
}
|
||||
}
|
||||
|
||||
impl Entry {
|
||||
impl<V> Entry<V>
|
||||
where
|
||||
V: Nodelike,
|
||||
{
|
||||
/// Computes the size of an entry
|
||||
///
|
||||
/// Files are simply their length (not capacity)
|
||||
@ -272,6 +703,7 @@ impl Entry {
|
||||
files.iter().map(|(name, _inum)| name.len() as u64).sum()
|
||||
}
|
||||
Entry::Directory(DirType::List, files) => files.len() as u64,
|
||||
Entry::Lazy(v) => v.size() as u64, // give an answer because we can... but should
|
||||
}
|
||||
}
|
||||
|
||||
@ -280,6 +712,7 @@ impl Entry {
|
||||
match self {
|
||||
Entry::File(..) => FileType::RegularFile,
|
||||
Entry::Directory(..) => FileType::Directory,
|
||||
Entry::Lazy(v) => v.kind(),
|
||||
}
|
||||
}
|
||||
|
||||
@ -287,6 +720,7 @@ impl Entry {
|
||||
match self {
|
||||
Entry::File(t, _) => t.to_string(),
|
||||
Entry::Directory(t, _) => t.to_string(),
|
||||
Entry::Lazy(_) => unreachable!("unresolved lazy value in Entry::typ"),
|
||||
}
|
||||
}
|
||||
|
||||
@ -308,6 +742,7 @@ impl Entry {
|
||||
}
|
||||
Err(..) => false,
|
||||
},
|
||||
Entry::Lazy(_) => todo!("Entry::try_set_typ"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -353,7 +788,10 @@ const ENOATTR: i32 = libc::ENODATA;
|
||||
#[cfg(target_os = "macos")]
|
||||
const ENOATTR: i32 = libc::ENOATTR;
|
||||
|
||||
impl Filesystem for FS {
|
||||
impl<V> Filesystem for FS<V>
|
||||
where
|
||||
V: Nodelike,
|
||||
{
|
||||
/// Synchronizes the `FS`, calling `FS::sync` with `last_sync == true`.
|
||||
#[instrument(level = "debug", skip(self), fields(dirty = self.dirty.get()))]
|
||||
fn destroy(&mut self) {
|
||||
@ -424,27 +862,29 @@ impl Filesystem for FS {
|
||||
Some(name) => name,
|
||||
};
|
||||
|
||||
match &dir.entry {
|
||||
let inum = match &dir.entry {
|
||||
Entry::Directory(_kind, files) => match files.get(filename) {
|
||||
None => {
|
||||
reply.error(libc::ENOENT);
|
||||
return;
|
||||
}
|
||||
Some(DirEntry { inum, .. }) => {
|
||||
let file = match self.get(*inum) {
|
||||
Err(_e) => {
|
||||
reply.error(libc::ENOENT);
|
||||
return;
|
||||
}
|
||||
Ok(inode) => inode,
|
||||
};
|
||||
|
||||
reply.entry(&TTL, &file.attr(), 0);
|
||||
}
|
||||
Some(DirEntry { inum, .. }) => *inum,
|
||||
},
|
||||
_ => {
|
||||
reply.error(libc::ENOTDIR);
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let file = match self.get(inum) {
|
||||
Err(_e) => {
|
||||
reply.error(libc::ENOENT);
|
||||
return;
|
||||
}
|
||||
Ok(inode) => inode,
|
||||
};
|
||||
|
||||
reply.entry(&TTL, &file.attr(), 0);
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, _req, reply))]
|
||||
@ -576,6 +1016,7 @@ impl Filesystem for FS {
|
||||
reply.error(libc::EISDIR);
|
||||
return;
|
||||
}
|
||||
Entry::Lazy(..) => unreachable!("unresolved lazy value found in setattr"),
|
||||
},
|
||||
Err(_) => {
|
||||
reply.error(libc::ENOENT);
|
||||
@ -858,6 +1299,7 @@ impl Filesystem for FS {
|
||||
}
|
||||
reply.ok()
|
||||
}
|
||||
Entry::Lazy(..) => unreachable!("unresolved lazy value in readdir"),
|
||||
}
|
||||
}
|
||||
|
||||
@ -934,6 +1376,7 @@ impl Filesystem for FS {
|
||||
return;
|
||||
}
|
||||
}
|
||||
Entry::Lazy(..) => unreachable!("unresolved lazy value in mknod"),
|
||||
},
|
||||
};
|
||||
|
||||
@ -967,6 +1410,7 @@ impl Filesystem for FS {
|
||||
},
|
||||
);
|
||||
}
|
||||
Entry::Lazy(..) => unreachable!("unresolved lazy value in mknod"),
|
||||
},
|
||||
};
|
||||
|
||||
@ -1017,6 +1461,7 @@ impl Filesystem for FS {
|
||||
return;
|
||||
}
|
||||
}
|
||||
Entry::Lazy(..) => unreachable!("unresolved lazy value in mkdir"),
|
||||
},
|
||||
};
|
||||
|
||||
@ -1043,6 +1488,7 @@ impl Filesystem for FS {
|
||||
},
|
||||
);
|
||||
}
|
||||
Entry::Lazy(..) => unreachable!("unresolved lazy value in mkdir"),
|
||||
},
|
||||
};
|
||||
|
||||
@ -1089,6 +1535,7 @@ impl Filesystem for FS {
|
||||
reply.error(libc::EISDIR);
|
||||
return;
|
||||
}
|
||||
Entry::Lazy(..) => unreachable!("unresolved lazy value in write"),
|
||||
};
|
||||
|
||||
// make space
|
||||
@ -1141,6 +1588,10 @@ impl Filesystem for FS {
|
||||
reply.error(libc::ENOTDIR);
|
||||
return;
|
||||
}
|
||||
Ok(Inode {
|
||||
entry: Entry::Lazy(..),
|
||||
..
|
||||
}) => unreachable!("unresolved lazy value in unlink"),
|
||||
};
|
||||
|
||||
// ensure it's a regular file
|
||||
@ -1198,6 +1649,10 @@ impl Filesystem for FS {
|
||||
reply.error(libc::ENOTDIR);
|
||||
return;
|
||||
}
|
||||
Ok(Inode {
|
||||
entry: Entry::Lazy(..),
|
||||
..
|
||||
}) => unreachable!("unresolved lazy value in rmdir"),
|
||||
};
|
||||
|
||||
// find the actual directory being deleted
|
||||
@ -1206,7 +1661,7 @@ impl Filesystem for FS {
|
||||
kind: FileType::Directory,
|
||||
inum,
|
||||
..
|
||||
}) => inum,
|
||||
}) => *inum,
|
||||
Some(_) => {
|
||||
reply.error(libc::ENOTDIR);
|
||||
return;
|
||||
@ -1218,7 +1673,7 @@ impl Filesystem for FS {
|
||||
};
|
||||
|
||||
// make sure it's empty
|
||||
match self.get(*inum) {
|
||||
match self.get(inum) {
|
||||
Ok(Inode {
|
||||
entry: Entry::Directory(_, dir_files),
|
||||
..
|
||||
@ -1430,6 +1885,11 @@ impl Filesystem for FS {
|
||||
reply.error(libc::EBADF);
|
||||
return;
|
||||
}
|
||||
Ok(Inode {
|
||||
entry: Entry::Lazy(..),
|
||||
..
|
||||
}) => unreachable!("unresolved lazy value in fallocate"),
|
||||
|
||||
Err(_e) => {
|
||||
reply.error(libc::ENODEV);
|
||||
return;
|
||||
|
||||
59
src/main.rs
59
src/main.rs
@ -6,14 +6,14 @@ mod format;
|
||||
mod fs;
|
||||
|
||||
use config::{Config, ERROR_STATUS_CLI, ERROR_STATUS_FUSE};
|
||||
use format::Format;
|
||||
use fs::FS;
|
||||
|
||||
use fuser::MountOption;
|
||||
|
||||
fn main() {
|
||||
let config = Config::from_args();
|
||||
let mut options = vec![
|
||||
MountOption::FSName(format!("{}", config.input)),
|
||||
];
|
||||
let mut options = vec![MountOption::FSName(format!("{}", config.input))];
|
||||
if config.read_only {
|
||||
options.push(MountOption::RO);
|
||||
}
|
||||
@ -30,17 +30,52 @@ fn main() {
|
||||
};
|
||||
let cleanup_mount = config.cleanup_mount;
|
||||
let input_format = config.input_format;
|
||||
let fs = input_format.load(config);
|
||||
|
||||
info!("mounting on {:?} with options {:?}", mount, options);
|
||||
let status = match fuser::mount2(fs, &mount, &options) {
|
||||
Ok(()) => {
|
||||
info!("unmounted");
|
||||
0
|
||||
let status = match input_format {
|
||||
Format::Json => {
|
||||
let fs: FS<format::json::Value> = FS::new(config);
|
||||
|
||||
info!("mounting on {:?} with options {:?}", mount, options);
|
||||
match fuser::mount2(fs, &mount, &options) {
|
||||
Ok(()) => {
|
||||
info!("unmounted");
|
||||
0
|
||||
}
|
||||
Err(e) => {
|
||||
error!("I/O error: {}", e);
|
||||
ERROR_STATUS_FUSE
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!("I/O error: {}", e);
|
||||
ERROR_STATUS_FUSE
|
||||
Format::Toml => {
|
||||
let fs: FS<format::toml::Value> = FS::new(config);
|
||||
|
||||
info!("mounting on {:?} with options {:?}", mount, options);
|
||||
match fuser::mount2(fs, &mount, &options) {
|
||||
Ok(()) => {
|
||||
info!("unmounted");
|
||||
0
|
||||
}
|
||||
Err(e) => {
|
||||
error!("I/O error: {}", e);
|
||||
ERROR_STATUS_FUSE
|
||||
}
|
||||
}
|
||||
}
|
||||
Format::Yaml => {
|
||||
let fs: FS<format::yaml::Value> = FS::new(config);
|
||||
|
||||
info!("mounting on {:?} with options {:?}", mount, options);
|
||||
match fuser::mount2(fs, &mount, &options) {
|
||||
Ok(()) => {
|
||||
info!("unmounted");
|
||||
0
|
||||
}
|
||||
Err(e) => {
|
||||
error!("I/O error: {}", e);
|
||||
ERROR_STATUS_FUSE
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
28
tests/eager.sh
Executable file
28
tests/eager.sh
Executable file
@ -0,0 +1,28 @@
|
||||
#!/bin/sh
|
||||
|
||||
fail() {
|
||||
echo FAILED: $1
|
||||
if [ "$MNT" ]
|
||||
then
|
||||
umount "$MNT"
|
||||
rmdir "$MNT"
|
||||
rm "$OUT" "$EXP"
|
||||
fi
|
||||
exit 1
|
||||
}
|
||||
|
||||
MNT=$(mktemp -d)
|
||||
|
||||
ffs -m "$MNT" --eager ../json/json_eg1.json &
|
||||
PID=$!
|
||||
sleep 2
|
||||
case $(ls "$MNT") in
|
||||
(glossary) ;;
|
||||
(*) fail ls;;
|
||||
esac
|
||||
umount "$MNT" || fail unmount
|
||||
sleep 1
|
||||
kill -0 $PID >/dev/null 2>&1 && fail process
|
||||
|
||||
rmdir "$MNT" || fail mount
|
||||
|
||||
Loading…
Reference in New Issue
Block a user