From a6229248e9e464d2d6370a366d90f362845b7e1f Mon Sep 17 00:00:00 2001 From: hsjobeki Date: Thu, 23 Feb 2023 16:19:19 +0100 Subject: [PATCH] move indexer into monorepo --- .gitignore | 37 +--- flake.lock | 128 +++---------- flake.nix | 101 ++++++++--- indexer/.gitignore | 4 + indexer/.travis.yml | 2 + indexer/Cargo.lock | 381 +++++++++++++++++++++++++++++++++++++++ indexer/Cargo.toml | 14 ++ indexer/makeMarkdown.js | 35 ++++ indexer/src/main.rs | 286 +++++++++++++++++++++++++++++ indexer/test/options.nix | 231 ++++++++++++++++++++++++ projects.toml | 16 +- website/.gitignore | 44 +++++ 12 files changed, 1119 insertions(+), 160 deletions(-) create mode 100644 indexer/.gitignore create mode 100644 indexer/.travis.yml create mode 100644 indexer/Cargo.lock create mode 100644 indexer/Cargo.toml create mode 100644 indexer/makeMarkdown.js create mode 100644 indexer/src/main.rs create mode 100644 indexer/test/options.nix create mode 100644 website/.gitignore diff --git a/.gitignore b/.gitignore index e06b3d2..0fe1497 100644 --- a/.gitignore +++ b/.gitignore @@ -1,47 +1,18 @@ # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. -# dependencies -node_modules -.pnp -.pnp.js + # testing coverage -website/models/data/* -!website/models/data/index.ts # nix .direnv/ result result-* - -# next.js -/.next/ -/out/ - -# production -/build - -# misc -.DS_Store -*.pem -result -result-* - -# debug -npm-debug.log* -yarn-debug.log* -yarn-error.log* -.pnpm-debug.log* - -# local env files -.env*.local .pre-commit-config.yaml -# vercel -.vercel +# dream2nix -# typescript -*.tsbuildinfo -next-env.d.ts +## dream2nix internal cache +.dream2nix \ No newline at end of file diff --git a/flake.lock b/flake.lock index 7f86654..2158d87 100644 --- a/flake.lock +++ b/flake.lock @@ -162,11 +162,11 @@ }, "flake-utils": { "locked": { - "lastModified": 1659877975, - "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=", + "lastModified": 1667395993, + "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", "owner": "numtide", "repo": "flake-utils", - "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0", + "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", "type": "github" }, "original": { @@ -190,21 +190,6 @@ "type": "github" } }, - "flake-utils_2": { - "locked": { - "lastModified": 1667395993, - "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, "flakeCompat": { "flake": false, "locked": { @@ -305,30 +290,6 @@ "type": "github" } }, - "nixdoc-fork": { - "inputs": { - "flake-utils": [ - "nixdoc-fork", - "rust-overlay", - "flake-utils" - ], - "nixpkgs": "nixpkgs_2", - "rust-overlay": "rust-overlay" - }, - "locked": { - "lastModified": 1675155227, - "narHash": "sha256-2yOid81G2AloUVoTKZfKQyalNFJ2EvvHxnzcFMDt1qU=", - "owner": "hsjobeki", - "repo": "nixdoc", - "rev": "6a6c165e2c1fa8b05b22ac683e2c71fff15fe87b", - "type": "github" - }, - "original": { - "owner": "hsjobeki", - "repo": "nixdoc", - "type": "github" - } - }, "nixpkgs": { "locked": { "lastModified": 1665580254, @@ -362,6 +323,21 @@ "type": "github" } }, + "nixpkgs-master": { + "locked": { + "lastModified": 1677161649, + "narHash": "sha256-ypilUl4lX0VCZUxpnq650SzVvcI2cwQmuhhTVWJ3af0=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "e3040b367c667525b614204d510ae0d09c59c8e0", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "ref": "master", + "type": "indirect" + } + }, "nixpkgs-stable": { "locked": { "lastModified": 1673800717, @@ -380,45 +356,16 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1673027386, - "narHash": "sha256-Wjt+oDhRLcf3opIjUKHGN+Xrp3w2ZDms6bO4pCLvsco=", + "lastModified": 1677063315, + "narHash": "sha256-qiB4ajTeAOVnVSAwCNEEkoybrAlA+cpeiBxLobHndE8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b3818a46e686f24561a28eaa9fcf35e18b8d8e89", - "type": "github" - }, - "original": { - "id": "nixpkgs", - "type": "indirect" - } - }, - "nixpkgs_3": { - "locked": { - "lastModified": 1665296151, - "narHash": "sha256-uOB0oxqxN9K7XGF1hcnY+PQnlQJ+3bP2vCn/+Ru/bbc=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "14ccaaedd95a488dd7ae142757884d8e125b3363", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs_4": { - "locked": { - "lastModified": 1671160997, - "narHash": "sha256-fcPZMRjAkUhrfXwoq2RPejfhtPnQ+aI5CTr4x8d0JPs=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "4172cdda7e56a48065475fb98c57b03b83c1fde4", + "rev": "988cc958c57ce4350ec248d2d53087777f9e1949", "type": "github" }, "original": { "id": "nixpkgs", + "ref": "nixos-unstable", "type": "indirect" } }, @@ -467,7 +414,7 @@ "pre-commit-hooks_2": { "inputs": { "flake-compat": "flake-compat", - "flake-utils": "flake-utils_2", + "flake-utils": "flake-utils", "gitignore": "gitignore", "nixpkgs": [ "nixpkgs" @@ -475,11 +422,11 @@ "nixpkgs-stable": "nixpkgs-stable" }, "locked": { - "lastModified": 1674550893, - "narHash": "sha256-HXI8AB96PP7UZ7iPANACXM8qc9eMz0ljxBEDM8JJKhY=", + "lastModified": 1677160285, + "narHash": "sha256-tBzpCjMP+P3Y3nKLYvdBkXBg3KvTMo3gvi8tLQaqXVY=", "owner": "cachix", "repo": "pre-commit-hooks.nix", - "rev": "7bdf85f6bbef581eb687838d19f2b35a4c9d77f0", + "rev": "2bd861ab81469428d9c823ef72c4bb08372dd2c4", "type": "github" }, "original": { @@ -508,8 +455,8 @@ "root": { "inputs": { "dream2nix": "dream2nix", - "nixdoc-fork": "nixdoc-fork", - "nixpkgs": "nixpkgs_4", + "nixpkgs": "nixpkgs_2", + "nixpkgs-master": "nixpkgs-master", "pre-commit-hooks": "pre-commit-hooks_2" } }, @@ -529,25 +476,6 @@ "repo": "rust-analyzer", "type": "github" } - }, - "rust-overlay": { - "inputs": { - "flake-utils": "flake-utils", - "nixpkgs": "nixpkgs_3" - }, - "locked": { - "lastModified": 1670380307, - "narHash": "sha256-7fJN5ndnE8YbrrtYdqMo3gDV/BW37M4wNBRhjdfP/XY=", - "owner": "oxalica", - "repo": "rust-overlay", - "rev": "fc98242f5f49d39b8fd3a611c146741a35dc012d", - "type": "github" - }, - "original": { - "owner": "oxalica", - "repo": "rust-overlay", - "type": "github" - } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 849f5ed..6436607 100644 --- a/flake.nix +++ b/flake.nix @@ -1,53 +1,94 @@ { inputs = { + nixpkgs.url = "nixpkgs/nixos-unstable"; + nixpkgs-master.url = "nixpkgs/master"; + dream2nix.url = "github:nix-community/dream2nix"; - nixdoc-fork.url = "github:hsjobeki/nixdoc"; + pre-commit-hooks = { url = "github:cachix/pre-commit-hooks.nix"; inputs.nixpkgs.follows = "nixpkgs"; }; }; - outputs = { self, nixpkgs, pre-commit-hooks, ... }@inp: + outputs = { self, nixpkgs, pre-commit-hooks, dream2nix, nixpkgs-master }: let system = "x86_64-linux"; - pkgs = inp.nixpkgs.legacyPackages.${system}; - inherit (builtins.fromJSON (builtins.readFile ./website/package.json)) name; + pkgs = nixpkgs.legacyPackages.${system}; + websiteName = (builtins.fromJSON (builtins.readFile ./website/package.json)).name; + inherit (self.packages.${system}) indexer nixpkgs-data; + prepareData = prefix: '' - cp -f ${inp.nixdoc-fork.packages.${system}.data.lib} ${prefix}./models/data/lib.json - cp -f ${inp.nixdoc-fork.packages.${system}.data.build_support} ${prefix}./models/data/trivial-builders.json + cp -f ${nixpkgs-data.lib} ${prefix}./models/data/lib.json + cp -f ${nixpkgs-data.build_support} ${prefix}./models/data/trivial-builders.json node ${prefix}./scripts/make-builtins.js ${prefix}./models/data ''; - in - (inp.dream2nix.lib.makeFlakeOutputs { - systems = [ system ]; - projects = ./projects.toml; - config.projectRoot = ./website; - source = ./website; - packageOverrides = { - ${name}.staticPage = { - preBuild = prepareData ""; - installPhase = '' - runHook preInstall + dream2nixOutput = dream2nix.lib.makeFlakeOutputs { + systems = [ system ]; + projects = ./projects.toml; + config.projectRoot = ./.; + source = ./.; - npm run export - mkdir -p $out/static - cp -r ./out/* $out/static/ - cp -r ./ $lib + packageOverrides = { + ${websiteName}.staticPage = { + preBuild = prepareData ""; + installPhase = '' + runHook preInstall - runHook postInstall - ''; + npm run export + mkdir -p $out/static + cp -r ./out/* $out/static/ + cp -r ./ $lib + + runHook postInstall + ''; + }; }; }; - }) - // { + in + { + packages.${system} = dream2nixOutput.packages.${system} // { + nixpkgs-data = pkgs.stdenv.mkDerivation { + pname = "data"; + version = "0.1.0"; + description = '' + wrapper around the indexer. + + Calls the indexer with ''/path. + and defines one output for every specified input path + + currently this list is manually maintained below. + ''; + src = nixpkgs-master; + outputs = [ "out" "lib" "build_support" ]; + nativeBuildInputs = [ indexer ]; + buildPhase = '' + echo "running nix metadata collect in nixpkgs/lib" + ${indexer}/bin/indexer --dir ./lib + ${indexer}/bin/indexer --dir ./pkgs/build-support + ''; + installPhase = '' + cat lib.json > $lib + cat build-support.json > $build_support + + mkdir $out + ln -s $lib $out/lib + ln -s $build_support $out/build_support + ''; + }; + + default = self.packages.${system}.noogle; + }; + devShells.${system}.default = pkgs.mkShell { - buildInputs = with pkgs; [ nodejs-18_x ]; + buildInputs = with pkgs; [ nodejs-18_x rustfmt rustc cargo clippy ]; + inputsFrom = [ indexer ]; shellHook = '' ${prepareData "./website/"} ${self.checks.${system}.pre-commit-check.shellHook} ''; }; + checks.${system} = { pre-commit-check = pre-commit-hooks.lib.${system}.run { src = ./.; @@ -55,7 +96,15 @@ nixpkgs-fmt.enable = true; statix.enable = true; markdownlint.enable = true; + + + }; + excludes = [ "indexer/test" ]; + settings = { + statix.ignore = [ "indexer/test" ]; + }; + }; }; }; diff --git a/indexer/.gitignore b/indexer/.gitignore new file mode 100644 index 0000000..0722ec3 --- /dev/null +++ b/indexer/.gitignore @@ -0,0 +1,4 @@ + +target/ +**/*.rs.bk +data.json \ No newline at end of file diff --git a/indexer/.travis.yml b/indexer/.travis.yml new file mode 100644 index 0000000..2a8d154 --- /dev/null +++ b/indexer/.travis.yml @@ -0,0 +1,2 @@ +language: nix +sudo: true diff --git a/indexer/Cargo.lock b/indexer/Cargo.lock new file mode 100644 index 0000000..2029177 --- /dev/null +++ b/indexer/Cargo.lock @@ -0,0 +1,381 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +dependencies = [ + "winapi", +] + +[[package]] +name = "arenatree" +version = "0.1.1" +source = "git+https://gitlab.com/jD91mZM2/arenatree#f9bf7efa9a5ef4c2dd9e2acc5a4cc79a987cb648" + +[[package]] +name = "arrayvec" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" +dependencies = [ + "nodrop", +] + +[[package]] +name = "atty" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652" +dependencies = [ + "libc", + "termion", + "winapi", +] + +[[package]] +name = "backtrace" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89a47830402e9981c5c41223151efcced65a0510c13097c769cede7efb34782a" +dependencies = [ + "backtrace-sys", + "cfg-if", + "libc", + "rustc-demangle", + "winapi", +] + +[[package]] +name = "backtrace-sys" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c66d56ac8dabd07f6aacdaf633f4b8262f5b3601a810a0dcddffd5c22c69daa0" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "bitflags" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" + +[[package]] +name = "cc" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f159dfd43363c4d08055a07703eb7a3406b0dac4d0584d96965a3262db3c9d16" + +[[package]] +name = "cfg-if" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "082bb9b28e00d3c9d39cc03e64ce4cea0f1bb9b3fde493f0cbc008472d22bdf4" + +[[package]] +name = "clap" +version = "2.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b957d88f4b6a63b9d70d5f454ac8011819c6efa7727858f458ab71c756ce2d3e" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "failure" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd377bcc1b1b7ce911967e3ec24fa19c3224394ec05b54aa7b083d498341ac7" +dependencies = [ + "backtrace", + "failure_derive", +] + +[[package]] +name = "failure_derive" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64c2d913fe8ed3b6c6518eedf4538255b989945c14c2a7d5cbff62a5e2120596" +dependencies = [ + "proc-macro2 0.4.20", + "quote 0.6.8", + "syn 0.15.15", + "synstructure", +] + +[[package]] +name = "indexer" +version = "0.1.0" +dependencies = [ + "rnix", + "serde", + "serde_json", + "structopt", +] + +[[package]] +name = "itoa" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" + +[[package]] +name = "libc" +version = "0.2.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76e3a3ef172f1a0b9a9ff0dd1491ae5e6c948b94479a3021819ba7d860c8645d" + +[[package]] +name = "nodrop" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2228dca57108069a5262f2ed8bd2e82496d2e074a06d1ccc7ce1687b6ae0a2" + +[[package]] +name = "proc-macro2" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b7eaaa90b4a90a932a9ea6666c95a389e424eff347f0f793979289429feee" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "proc-macro2" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd636425967c33af890042c483632d33fa7a18f19ad1d7ea72e8998c6ef8dea5" +dependencies = [ + "proc-macro2 0.4.20", +] + +[[package]] +name = "quote" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +dependencies = [ + "proc-macro2 1.0.47", +] + +[[package]] +name = "redox_syscall" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1" + +[[package]] +name = "redox_termios" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" +dependencies = [ + "redox_syscall", +] + +[[package]] +name = "rnix" +version = "0.4.1" +source = "git+https://gitlab.com/jD91mZM2/rnix.git?rev=10b86c94291b4864470158ef8750de85ddd8d4ba#10b86c94291b4864470158ef8750de85ddd8d4ba" +dependencies = [ + "arenatree", + "arrayvec", + "failure", + "smol_str", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcfe5b13211b4d78e5c2cadfebd7769197d95c639c35a50057eb4c05de811395" + +[[package]] +name = "ryu" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" + +[[package]] +name = "serde" +version = "1.0.148" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e53f64bb4ba0191d6d0676e1b141ca55047d83b74f5607e6d8eb88126c52c2dc" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.148" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55492425aa53521babf6137309e7d34c20bbfbbfcfe2c7f3a047fd1f6b92c0c" +dependencies = [ + "proc-macro2 1.0.47", + "quote 1.0.21", + "syn 1.0.105", +] + +[[package]] +name = "serde_json" +version = "1.0.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "smol_str" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ed6f19b800d76574926e458d5f8e2dbea86c2b58c08d33a982448f09ac8d0c" + +[[package]] +name = "strsim" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550" + +[[package]] +name = "structopt" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d77af7242f18c40fd19cb270985930f239ee1646cfb482050bbae9da1d18743b" +dependencies = [ + "clap", + "structopt-derive", +] + +[[package]] +name = "structopt-derive" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17ff01fe96de9d16e7372ae5f19dd7ece2c703b51043c3db9ea27f9e393ea311" +dependencies = [ + "proc-macro2 0.4.20", + "quote 0.6.8", + "syn 0.15.15", +] + +[[package]] +name = "syn" +version = "0.15.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a9c2bf1e53c21704a7cce1b2a42768f1ae32a6777108a0d7f1faa4bfe7f7c04" +dependencies = [ + "proc-macro2 0.4.20", + "quote 0.6.8", + "unicode-xid", +] + +[[package]] +name = "syn" +version = "1.0.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908" +dependencies = [ + "proc-macro2 1.0.47", + "quote 1.0.21", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73687139bf99285483c96ac0add482c3776528beac1d97d444f6e91f203a2015" +dependencies = [ + "proc-macro2 0.4.20", + "quote 0.6.8", + "syn 0.15.15", + "unicode-xid", +] + +[[package]] +name = "termion" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" +dependencies = [ + "libc", + "redox_syscall", + "redox_termios", +] + +[[package]] +name = "textwrap" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "307686869c93e71f94da64286f9a9524c0f308a9e1c87a583de8e9c9039ad3f6" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "unicode-ident" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" + +[[package]] +name = "unicode-width" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" + +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" + +[[package]] +name = "vec_map" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" + +[[package]] +name = "winapi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92c1eb33641e276cfa214a0522acad57be5c56b10cb348b3c5117db75f3ac4b0" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/indexer/Cargo.toml b/indexer/Cargo.toml new file mode 100644 index 0000000..93fff82 --- /dev/null +++ b/indexer/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "indexer" +version = "0.1.0" +authors = ["Johannes Kirschbauer "] +edition = "2021" + +[dependencies] +structopt = "0.2" +serde_json = "1.0.89" +serde = { version = "1.0.148", features = ["derive"] } + +[dependencies.rnix] +git = "https://gitlab.com/jD91mZM2/rnix.git" +rev = "10b86c94291b4864470158ef8750de85ddd8d4ba" diff --git a/indexer/makeMarkdown.js b/indexer/makeMarkdown.js new file mode 100644 index 0000000..cf91e8d --- /dev/null +++ b/indexer/makeMarkdown.js @@ -0,0 +1,35 @@ +// simple script +const data = require("./test.json"); +const fs = require("fs"); + +const all_docs = data.reduce( + (acc, doc) => `${acc} +## ${doc.name} + +### Description + +\`\`\`nix +${doc.description} +\`\`\` + +### Example + +\`\`\`nix +${doc.example} +\`\`\` + +### Type + +\`\`\`nix +${doc.fn_type} +\`\`\` + +`, + "" +); + +fs.writeFile("content.md", `# Functions\n${all_docs}`, (err) => { + if (err) { + console.error(err); + } +}); diff --git a/indexer/src/main.rs b/indexer/src/main.rs new file mode 100644 index 0000000..4e803d8 --- /dev/null +++ b/indexer/src/main.rs @@ -0,0 +1,286 @@ +extern crate rnix; +extern crate serde; +extern crate serde_json; +extern crate structopt; +use rnix::parser::{ASTKind, ASTNode, Arena, Data}; +use rnix::tokenizer::Trivia; +use rnix::tokenizer::{Meta, Span}; +use serde::{Deserialize, Serialize}; +use std::fs::{self, File}; +use std::path::PathBuf; +use structopt::StructOpt; + +/// Command line arguments for the indexer +#[derive(Debug, StructOpt)] +#[structopt(name = "indexer", about = "Generate Metadata from Nix files")] +struct Options { + /// directory to process. + #[structopt(short = "D", long = "dir", parse(from_os_str))] + dir: PathBuf, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct ManualEntry { + pub id: String, + pub line: Option, + pub category: String, + pub name: String, + pub fn_type: Option, + pub description: String, + pub example: Option, +} +#[derive(Debug)] +struct DocComment { + /// Primary documentation string. + doc: String, + + /// Optional type annotation for the thing being documented. + doc_type: Option, + + /// Usage example(s) (interpreted as a single code block) + example: Option, +} + +#[derive(Debug)] +struct DocItem { + name: String, + span: Span, + comment: DocComment, +} + +/// Retrieve documentation comments. For now only multiline comments are considered +fn retrieve_doc_comment(allow_single_line: bool, meta: &Meta) -> Option { + for item in meta.leading.iter() { + if let Trivia::Comment { + multiline, content, .. + } = item + { + if *multiline || allow_single_line { + return Some(content.to_string()); + } + } + } + return None; +} + +/// Transforms an AST node into a `DocItem` if it has a leading +/// documentation comment. +fn retrieve_doc_item(node: &ASTNode) -> Option { + // We are only interested in identifiers. + + if let Data::Ident(meta, name) = &node.data { + let comment = retrieve_doc_comment(false, meta)?; + return Some(DocItem { + span: node.span, + name: name.to_string(), + comment: parse_doc_comment(&comment), + }); + } + return None; +} + +fn get_indentation(line: &str) -> usize { + return line + .char_indices() + .find(|(_, ch)| !(ch.is_whitespace() && *ch != '\n')) + .map(|(i, _)| i) + .unwrap_or_else(|| line.len()); +} + +/// *Really* dumb, mutable, hacky doc comment "parser". +fn parse_doc_comment(raw: &str) -> DocComment { + enum ParseState { + Doc, + Type, + Example, + } + + let mut doc = String::new(); + let mut doc_type = String::new(); + let mut example = String::new(); + let mut state = ParseState::Doc; + + let mut type_ident = 0; + let mut doc_ident = 0; + let mut example_ident = 0; + + for line in raw.clone().lines() { + let mut line = line.clone().trim_end(); + let trimmed = line.clone().trim(); + if trimmed.starts_with("Type:") { + state = ParseState::Type; + line = &trimmed[5..]; // trim 'Type:' + } + + if trimmed.starts_with("Example:") { + state = ParseState::Example; + line = &trimmed[8..]; // trim 'Example:' + } + match state { + ParseState::Type => { + let curr_indent = get_indentation(line); + if curr_indent > 0 && type_ident == 0 { + type_ident = curr_indent; + } + } + ParseState::Doc => { + let curr_indent = get_indentation(line); + if curr_indent > 0 && doc_ident == 0 { + doc_ident = curr_indent; + } + } + ParseState::Example => { + let curr_indent = get_indentation(line); + if curr_indent > 0 && example_ident == 0 { + example_ident = curr_indent; + } + } + } + } + state = ParseState::Doc; + + for line in raw.lines() { + let mut line = line.trim_end(); + let trimmed = line.clone().trim(); + if trimmed.starts_with("Type:") { + state = ParseState::Type; + line = &trimmed[5..]; // trim 'Type:' + } + + if trimmed.starts_with("Example:") { + state = ParseState::Example; + line = &trimmed[8..]; // trim 'Example:' + } + match state { + // important: trim only trailing whitespaces; as leading ones might be markdown formatting or code examples. + ParseState::Type => { + let stripped = line.trim_end(); + let formatted = stripped.replacen(&" ".repeat(type_ident), "", 1); + doc_type.push_str(&formatted); + doc_type.push('\n'); + } + ParseState::Doc => { + let stripped = line.trim_end(); + let formatted = stripped.replacen(&" ".repeat(doc_ident), "", 1); + doc.push_str(&formatted); + doc.push('\n'); + } + ParseState::Example => { + let stripped = line.trim_end(); + let formatted = stripped.replacen(&" ".repeat(example_ident), "", 1); + example.push_str(&formatted); + example.push('\n'); + } + } + } + let f = |mut s: String| { + if s.is_empty() { + None + } else { + let len = s.trim_end_matches(&['\r', '\n'][..]).len(); + s.truncate(len); + return Some(s.trim_start().into()); + } + }; + // let doc_f = f(doc); + DocComment { + doc: f(doc).unwrap_or("".to_owned()), + doc_type: f(doc_type), + example: f(example), + } +} + +fn get_line(span: Span, src: &String) -> Option { + let mut line_nr: usize = 1; + for (count, char) in src.chars().enumerate() { + if char == '\n' { + line_nr = line_nr + 1; + } + if count == span.start as usize { + return Some(line_nr); + } + } + None +} + +/// Traverse a pattern argument, collecting its argument names. +/// Traverse a Nix lambda and collect the identifiers of arguments +/// until an unexpected AST node is encountered. +/// +/// This will collect the argument names for curried functions in the +/// `a: b: c: ...`-style, but does not currently work with pattern +/// functions (`{ a, b, c }: ...`). +/// +/// In the AST representation used by rnix, any lambda node has an +/// immediate child that is the identifier of its argument. The "body" +/// of the lambda is two steps to the right from that identifier, if +/// it is a lambda the function is curried and we can recurse. +/// Traverse the arena from a top-level SetEntry and collect, where +/// possible: +/// +/// 1. The identifier of the set entry itself. +/// 2. The attached doc comment on the entry. +/// 3. The argument names of any curried functions (pattern functions +/// not yet supported). +fn collect_entry_information<'a>(arena: &Arena<'a>, entry_node: &ASTNode) -> Option { + // The "root" of any attribute set entry is this `SetEntry` node. + // It has an `Attribute` child, which in turn has the identifier + // (on which the documentation comment is stored) as its child. + let attr_node = &arena[entry_node.node.child?]; + let ident_node = &arena[attr_node.node.child?]; + // At this point we can retrieve the `DocItem` from the identifier + // node - this already contains most of the information we are + // interested in. + let doc_item = retrieve_doc_item(ident_node)?; + + // From our entry we can walk two nodes to the right and check + // whether we are dealing with a lambda. If so, we can start + // collecting the function arguments - otherwise we're done. + // let assign_node = &arena[attr_node.node.sibling?]; + // let content_node = &arena[assign_node.node.sibling?]; + Some(doc_item) +} + +fn main() { + let opts = Options::from_args(); + let paths = fs::read_dir(&opts.dir).unwrap(); + let mut data: Vec = vec![]; + for path in paths { + let file_path = path.unwrap(); + let file_type = file_path.file_type().unwrap(); + let file = file_path.path(); + if file_type.is_file() && file.extension().unwrap() == "nix" { + // sources.push(file); + let src = fs::read_to_string(&file).unwrap(); + let nix = rnix::parse(&src).unwrap(); + let filename = file.file_stem().unwrap().to_str().unwrap(); + let parent = file + .parent() + .unwrap() + .file_name() + .unwrap() + .to_str() + .unwrap(); + + let entries: Vec = nix + .arena + .into_iter() + .filter(|node| node.kind == ASTKind::SetEntry) + .filter_map(|node| collect_entry_information(&nix.arena, node)) + .map(|d| ManualEntry { + id: format!("{}.{}.{}", parent, filename, d.name), + line: Some(get_line(d.span, &src)).unwrap_or(None), + category: file.display().to_string(), + name: d.name, + description: d.comment.doc, + fn_type: d.comment.doc_type, + example: d.comment.example, + }) + .collect(); + data.extend(entries); + } + } + let json_file = + File::create(opts.dir.file_name().unwrap().to_str().unwrap().to_owned() + ".json").unwrap(); + ::serde_json::to_writer(&json_file, &data).unwrap(); +} diff --git a/indexer/test/options.nix b/indexer/test/options.nix new file mode 100644 index 0000000..8165737 --- /dev/null +++ b/indexer/test/options.nix @@ -0,0 +1,231 @@ +{ pkgs +, lib +, options +, transformOptions ? lib.id +, # function for additional transformations of the options + documentType ? "appendix" +, # TODO deprecate "appendix" in favor of "none" + # and/or rename function to moduleOptionDoc for clean slate + # If you include more than one option list into a document, you need to + # provide different ids. + variablelistId ? "configuration-variable-list" +, # String to prefix to the option XML/HTML id attributes. + optionIdPrefix ? "opt-" +, revision ? "" +, # Specify revision for the options + # a set of options the docs we are generating will be merged into, as if by recursiveUpdate. + # used to split the options doc build into a static part (nixos/modules) and a dynamic part + # (non-nixos modules imported via configuration.nix, other module sources). + baseOptionsJSON ? null +, # instead of printing warnings for eg options with missing descriptions (which may be lost + # by nix build unless -L is given), emit errors instead and fail the build + warningsAreErrors ? true +, # allow docbook option docs if `true`. only markdown documentation is allowed when set to + # `false`, and a different renderer may be used with different bugs and performance + # characteristics but (hopefully) indistinguishable output. + allowDocBook ? true +, # whether lib.mdDoc is required for descriptions to be read as markdown. + markdownByDefault ? false +, +}: +let + /* + Generate JSON, XML and DocBook documentation for given NixOS options. + + Example: + + { pkgs, }: + + let + eval = import (pkgs.path + "/nixos/lib/eval-config.nix") { + baseModules = [ + ../module.nix + ]; + modules = []; + }; + in pkgs.nixosOptionsDoc { + options = eval.options; + } + + Type: + nixosOptionsDoc :: { + pkgs :: AttrSet, + lib :: AttrSet, + options :: AttrSet, + transformOptions? :: String, + documentType? :: String, + variablelistId? :: String, + optionIdPrefix? :: String, + revision? :: String, + baseOptionsJSON? :: AttrSet, + warningsAreErrors? :: Bool, + allowDocBook? :: Bool, + markdownByDefault? :: Bool, + } -> { + optionsNix :: AttrSet, + optionsAsciiDoc :: Derivation, + optionsCommonMark :: Derivation, + optionsJSON optionsXML :: Derivation, + optionsDocBook :: Derivation, + } + */ + rawOpts = lib.optionAttrSetToDocList options; + transformedOpts = map transformOptions rawOpts; + filteredOpts = lib.filter (opt: opt.visible && !opt.internal) transformedOpts; + optionsList = + lib.flip map filteredOpts + ( + opt: + opt + // lib.optionalAttrs (opt ? relatedPackages && opt.relatedPackages != [ ]) { relatedPackages = genRelatedPackages opt.relatedPackages opt.name; } + ); + + # Generate DocBook documentation for a list of packages. This is + # what `relatedPackages` option of `mkOption` from + # ../../../lib/options.nix influences. + # + # Each element of `relatedPackages` can be either + # - a string: that will be interpreted as an attribute name from `pkgs` and turned into a link + # to search.nixos.org, + # - a list: that will be interpreted as an attribute path from `pkgs` and turned into a link + # to search.nixos.org, + # - an attrset: that can specify `name`, `path`, `comment` + # (either of `name`, `path` is required, the rest are optional). + # + # NOTE: No checks against `pkgs` are made to ensure that the referenced package actually exists. + # Such checks are not compatible with option docs caching. + genRelatedPackages = packages: optName: + let + unpack = p: + if lib.isString p + then { name = p; } + else if lib.isList p + then { path = p; } + else p; + describe = args: + let + title = args.title or null; + name = args.name or (lib.concatStringsSep "." args.path); + in + '' + + + + ${lib.optionalString (title != null) "${title} aka "}pkgs.${name} + + + ${lib.optionalString (args ? comment) "${args.comment}"} + + ''; + in + "${lib.concatStringsSep "\n" (map (p: describe (unpack p)) packages)}"; + + optionsNix = builtins.listToAttrs (map + (o: { + name = o.name; + value = removeAttrs o [ "name" "visible" "internal" ]; + }) + optionsList); +in +rec { + inherit optionsNix; + + optionsAsciiDoc = pkgs.runCommand "options.adoc" { } '' + ${pkgs.python3Minimal}/bin/python ${./generateDoc.py} \ + --format asciidoc \ + ${optionsJSON}/share/doc/nixos/options.json \ + > $out + ''; + + optionsCommonMark = pkgs.runCommand "options.md" { } '' + ${pkgs.python3Minimal}/bin/python ${./generateDoc.py} \ + --format commonmark \ + ${optionsJSON}/share/doc/nixos/options.json \ + > $out + ''; + + optionsJSON = + pkgs.runCommand "options.json" + { + meta.description = "List of NixOS options in JSON format"; + nativeBuildInputs = [ + pkgs.brotli + ( + let + # python3Minimal can't be overridden with packages on Darwin, due to a missing framework. + # Instead of modifying stdenv, we take the easy way out, since most people on Darwin will + # just be hacking on the Nixpkgs manual (which also uses make-options-doc). + python = + if pkgs.stdenv.isDarwin + then pkgs.python3 + else pkgs.python3Minimal; + self = python.override { + inherit self; + includeSiteCustomize = true; + }; + in + self.withPackages (p: [ p.mistune ]) + ) + ]; + options = + builtins.toFile "options.json" + (builtins.unsafeDiscardStringContext (builtins.toJSON optionsNix)); + # merge with an empty set if baseOptionsJSON is null to run markdown + # processing on the input options + baseJSON = + if baseOptionsJSON == null + then builtins.toFile "base.json" "{}" + else baseOptionsJSON; + } + '' + # Export list of options in different format. + dst=$out/share/doc/nixos + mkdir -p $dst + + python ${./mergeJSON.py} \ + ${lib.optionalString warningsAreErrors "--warnings-are-errors"} \ + ${lib.optionalString (! allowDocBook) "--error-on-docbook"} \ + ${lib.optionalString markdownByDefault "--markdown-by-default"} \ + $baseJSON $options \ + > $dst/options.json + + brotli -9 < $dst/options.json > $dst/options.json.br + + mkdir -p $out/nix-support + echo "file json $dst/options.json" >> $out/nix-support/hydra-build-products + echo "file json-br $dst/options.json.br" >> $out/nix-support/hydra-build-products + ''; + + # Convert options.json into an XML file. + # The actual generation of the xml file is done in nix purely for the convenience + # of not having to generate the xml some other way + optionsXML = pkgs.runCommand "options.xml" { } '' + export NIX_STORE_DIR=$TMPDIR/store + export NIX_STATE_DIR=$TMPDIR/state + ${pkgs.nix}/bin/nix-instantiate \ + --eval --xml --strict ${./optionsJSONtoXML.nix} \ + --argstr file ${optionsJSON}/share/doc/nixos/options.json \ + > "$out" + ''; + + optionsDocBook = pkgs.runCommand "options-docbook.xml" { } '' + optionsXML=${optionsXML} + if grep /nixpkgs/nixos/modules $optionsXML; then + echo "The manual appears to depend on the location of Nixpkgs, which is bad" + echo "since this prevents sharing via the NixOS channel. This is typically" + echo "caused by an option default that refers to a relative path (see above" + echo "for hints about the offending path)." + exit 1 + fi + + ${pkgs.python3Minimal}/bin/python ${./sortXML.py} $optionsXML sorted.xml + ${pkgs.libxslt.bin}/bin/xsltproc \ + --stringparam documentType '${documentType}' \ + --stringparam revision '${revision}' \ + --stringparam variablelistId '${variablelistId}' \ + --stringparam optionIdPrefix '${optionIdPrefix}' \ + -o intermediate.xml ${./options-to-docbook.xsl} sorted.xml + ${pkgs.libxslt.bin}/bin/xsltproc \ + -o "$out" ${./postprocess-option-descriptions.xsl} intermediate.xml + ''; +} diff --git a/projects.toml b/projects.toml index db74265..479502c 100644 --- a/projects.toml +++ b/projects.toml @@ -10,7 +10,7 @@ [noogle] name = "noogle" -relPath = "" +relPath = "website" subsystem = "nodejs" translator = "package-lock" builder = "strict-builder" @@ -18,3 +18,17 @@ translators = ["package-lock", "package-json"] [noogle.subsystemInfo] nodejs = 18 + +[indexer] +name = "indexer" +relPath = "indexer" +subsystem = "rust" +translator = "cargo-lock" +translators = ["cargo-lock", "cargo-toml"] + +[indexer.subsystemInfo] +workspaceMembers = [] +[[indexer.subsystemInfo.crates]] +name = "indexer" +relPath = "indexer" +version = "0.1.0" diff --git a/website/.gitignore b/website/.gitignore new file mode 100644 index 0000000..5c22230 --- /dev/null +++ b/website/.gitignore @@ -0,0 +1,44 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# dependencies +node_modules +.pnp +.pnp.js + +# testing +coverage +/models/data/* +!/models/data/index.ts + +# nix +.direnv/ + + +# next.js +/.next/ +/out/ + +# production +/build + +# misc +.DS_Store +*.pem +result +result-* + +# debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* +.pnpm-debug.log* + +# local env files +.env*.local + +# vercel +.vercel + +# typescript +*.tsbuildinfo +next-env.d.ts