git.iko.soy/helix
on master help sign in
fetch and build grammars with nix in flake

This commit replaces the out-of-date builder in the flake which relied on submodules for fetching and the compiler for building. Now we disable fetching and building explicitly with the environment variable and then use builtins.fetchGit and a derivation mostly derived from upstream to compile the grammars. Anecdotally, this is still quite slow as builtins.fetchGit does not seem to do shallow clones. I'm not sure I see a way around it though without recording sha256s, which seems cumbersome.

commit37520f46ae891f77f81f4049cbb7dc2dbe2d5fc3 authorMichael Davis <mcarsondavis@gmail.com> dateMon 14 Feb 2022, 21:50 (5 years ago) parentb157c5a
2 files changed+113−20
compare with master
unifiedsplit
flake.nix+26−20 grammars.nix+87
flake.nix+26−20
@@ -33,26 +33,32 @@
3333 package = "helix";
3434 };
3535 overrides = {
36− crateOverrides = common: _: rec {
37− helix-term = prev: {
38− # disable fetching and building of tree-sitter grammars in the helix-term build.rs
39− HELIX_DISABLE_AUTO_GRAMMAR_BUILD = "1";
40− buildInputs = (prev.buildInputs or [ ]) ++ [ common.cCompiler.cc.lib ];
41− nativeBuildInputs = (prev.nativeBuildInputs or [ ]) ++ [ common.pkgs.makeWrapper ];
42− preConfigure = ''
43− ${prev.preConfigure}
44− rm -r helix-syntax/languages
45− ln -s ${helix}/helix-syntax/languages helix-syntax/languages
46− ln -s "$PWD/helix-syntax/languages" languages
47− mkdir -p runtime/grammars
48− '';
49− postInstall = ''
50− ${prev.postInstall or ""}
51− mkdir -p $out/lib
52− cp -r runtime $out/lib
53− wrapProgram "$out/bin/hx" --set HELIX_RUNTIME "$out/lib/runtime"
54− '';
55− };
36+ crateOverrides = common: _: {
37+ helix-term = prev:
38+ let
39+ inherit (common) pkgs;
40+ grammars = pkgs.callPackage ./grammars.nix { };
41+ runtimeDir = pkgs.runCommand "helix-runtime" { } ''
42+ mkdir -p $out
43+ ln -s ${common.root}/runtime/* $out
44+ rm -r $out/grammars
45+ ln -s ${grammars} $out/grammars
46+ '';
47+ in
48+ {
49+ # disable fetching and building of tree-sitter grammars in the helix-term build.rs
50+ HELIX_DISABLE_AUTO_GRAMMAR_BUILD = "1";
51+ # link languages and theme toml files since helix-term expects them (for tests)
52+ preConfigure = "ln -s ${common.root}/{languages.toml,theme.toml,base16_theme.toml} ..";
53+ buildInputs = (prev.buildInputs or [ ]) ++ [ common.cCompiler.cc.lib ];
54+ nativeBuildInputs = [ pkgs.makeWrapper ];
55+
56+ postFixup = ''
57+ if [ -f "$out/bin/hx" ]; then
58+ wrapProgram "$out/bin/hx" --set HELIX_RUNTIME "${runtimeDir}"
59+ fi
60+ '';
61+ };
5662 };
5763 shell = common: prev: {
5864 packages = prev.packages ++ (with common.pkgs; [ lld_13 lldb cargo-tarpaulin cargo-flamegraph ]);
grammars.nixnew file+87
@@ -0,0 +1,87 @@
1+{ stdenv, lib, runCommand, yj }:
2+let
3+ # HACK: nix < 2.6 has a bug in the toml parser, so we convert to JSON
4+ # before parsing
5+ languages-json = runCommand "languages-toml-to-json" { } ''
6+ ${yj}/bin/yj -t < ${./languages.toml} > $out
7+ '';
8+ languagesConfig =
9+ builtins.fromJSON (builtins.readFile (builtins.toPath languages-json));
10+ isGitGrammar = (grammar:
11+ builtins.hasAttr "source" grammar && builtins.hasAttr "git" grammar.source
12+ && builtins.hasAttr "rev" grammar.source);
13+ gitGrammars = builtins.filter isGitGrammar languagesConfig.grammar;
14+ buildGrammar = grammar:
15+ let
16+ source = builtins.fetchGit {
17+ url = grammar.source.git;
18+ rev = grammar.source.rev;
19+ allRefs = true;
20+ };
21+ in stdenv.mkDerivation rec {
22+ # see https://github.com/NixOS/nixpkgs/blob/fbdd1a7c0bc29af5325e0d7dd70e804a972eb465/pkgs/development/tools/parsing/tree-sitter/grammar.nix
23+
24+ pname = "helix-tree-sitter-${grammar.name}";
25+ version = grammar.source.rev;
26+
27+ src = if builtins.hasAttr "subpath" grammar.source then
28+ "${source}/${grammar.source.subpath}"
29+ else
30+ source;
31+
32+ dontUnpack = true;
33+ dontConfigure = true;
34+
35+ FLAGS = [
36+ "-I${src}/src"
37+ "-g"
38+ "-O3"
39+ "-fPIC"
40+ "-fno-exceptions"
41+ "-Wl,-z,relro,-z,now"
42+ ];
43+
44+ NAME = grammar.name;
45+
46+ buildPhase = ''
47+ runHook preBuild
48+
49+ if [[ -e "$src/src/scanner.cc" ]]; then
50+ $CXX -c "$src/src/scanner.cc" -o scanner.o $FLAGS
51+ elif [[ -e "$src/src/scanner.c" ]]; then
52+ $CC -c "$src/src/scanner.c" -o scanner.o $FLAGS
53+ fi
54+
55+ $CC -c "$src/src/parser.c" -o parser.o $FLAGS
56+ $CXX -shared -o $NAME.so *.o
57+
58+ ls -al
59+
60+ runHook postBuild
61+ '';
62+
63+ installPhase = ''
64+ runHook preInstall
65+ mkdir $out
66+ mv $NAME.so $out/
67+ runHook postInstall
68+ '';
69+
70+ # Strip failed on darwin: strip: error: symbols referenced by indirect symbol table entries that can't be stripped
71+ fixupPhase = lib.optionalString stdenv.isLinux ''
72+ runHook preFixup
73+ $STRIP $out/$NAME.so
74+ runHook postFixup
75+ '';
76+ };
77+ builtGrammars = builtins.map (grammar: {
78+ inherit (grammar) name;
79+ artifact = buildGrammar grammar;
80+ }) gitGrammars;
81+ grammarLinks = builtins.map (grammar:
82+ "ln -s ${grammar.artifact}/${grammar.name}.so $out/${grammar.name}.so")
83+ builtGrammars;
84+in runCommand "consolidated-helix-grammars" { } ''
85+ mkdir -p $out
86+ ${builtins.concatStringsSep "\n" grammarLinks}
87+''