diff --git a/build_system/clerk_driver.ml b/build_system/clerk_driver.ml index c6af3179..9a1104c3 100644 --- a/build_system/clerk_driver.ml +++ b/build_system/clerk_driver.ml @@ -20,7 +20,7 @@ open Catala_utils open Ninja_utils module Nj = Ninja_utils -(* Retrieve current version from dune *) +(* Version is synchronised with the catala version *) let version = Catala_utils.Cli.version @@ -28,7 +28,7 @@ let version = let files_or_folders = Arg.( - non_empty + value & pos_right 0 file [] & info [] ~docv:"FILE(S)" ~doc:"File(s) or folder(s) to process") @@ -41,6 +41,11 @@ let command = let debug = Arg.(value & flag & info ["debug"; "d"] ~doc:"Prints debug information") +let chdir = + Arg.(value & opt (some string) None + & info ["C"] ~docv:"DIR" + ~doc:"Change to the given directory before processing") + let reset_test_outputs = Arg.( value @@ -55,7 +60,7 @@ let catalac = value & opt (some string) None & info ["e"; "exe"] ~docv:"EXE" - ~doc:"Catala compiler executable, defaults to `catala`") + ~doc:"Catala compiler executable.") let ninja_output = Arg.( @@ -65,7 +70,7 @@ let ninja_output = ~doc: "$(i,FILE) is the file that will contain the build.ninja file \ output. If not specified, the build.ninja file will be output in \ - the temporary directory of the system.") + the temporary directory of the system and cleaned up on exit.") let scope = Arg.( @@ -80,7 +85,7 @@ let makeflags = Arg.( value & opt (some string) None - & info ["makeflags"] ~docv:"LANG" + & info ["makeflags"] ~docv:"FLAG" ~doc: "Provides the contents of a $(i, MAKEFLAGS) variable to pass on to \ Ninja. Currently recognizes the -i and -j options.") @@ -88,19 +93,31 @@ let makeflags = let catala_opts = Arg.( value - & opt (some string) None - & info ["c"; "catala-opts"] ~docv:"LANG" - ~doc:"Options to pass to the Catala compiler") + & opt_all string [] + & info ["c"; "catala-opts"] ~docv:"FLAG" + ~doc:"Option to pass to the Catala compiler. Can be repeated.") + +let color = + Arg.(value + & opt ~vopt:Cli.Always Cli.when_opt Auto + & info ["color"] + ~env:(Cmd.Env.info "CATALA_COLOR") + ~doc: + "Allow output of colored and styled text. Use $(i,auto), to \ + enable when the standard output is to a terminal, $(i,never) to \ + disable.") let clerk_t f = Term.( const f $ files_or_folders $ command + $ chdir $ catalac $ catala_opts $ makeflags $ debug + $ color $ scope $ reset_test_outputs $ ninja_output) @@ -159,7 +176,7 @@ type expected_output_descr = { id : string; (** Id of this precise unit test that will be associated to an expected output *) - cmd : string; + cmd : string list; (** Catala command to launch to run the test, excluding "catala" at the begin, and the name of the file to test *) } @@ -192,6 +209,7 @@ let scan_catala_file (file : File.t) (lang : Cli.backend_lang) : parse lines (n+1) @@ match line with | L.LINE_INCLUDE f -> + let f = if Filename.is_relative f then File.(file /../ f) else f in { acc with included_files = f :: acc.included_files } | L.LINE_MODULE_DEF m -> { acc with module_def = Some m } @@ -205,17 +223,17 @@ let scan_catala_file (file : File.t) (lang : Cli.backend_lang) : { id; tested_filename = file; output_dir = File.(file /../ "output" / ""); - cmd = "" } + cmd = [] } in let err n = - Format.asprintf "\"\"" File.format file n + [Format.asprintf "" File.format file n] in match Seq.uncons lines with | Some ((str, L.LINE_ANY), lines) when String.starts_with ~prefix:"catala " str -> let cmd = String.trim (String.remove_prefix ~prefix:"catala " str) in let cmd, lines, n = parse_block lines (n+1) [cmd] in - { test with cmd = String.concat " " cmd }, + { test with cmd = List.flatten (List.map (String.split_on_char ' ') cmd) }, lines, (n+1) | Some (_, lines) -> { test with cmd = err n}, lines, n+1 @@ -250,302 +268,367 @@ let scan_tree (dir : File.t) : catala_build_item Seq.t = | Some lang -> Some (scan_catala_file f lang)) dir -(**{1 Testing}*) +(** {1 System analysis} *) -(** Var references used in the Clerk file *) -module Var = struct - let tested_file = Nj.Expr.Var "tested_file" - let catala_cmd = Nj.Expr.Var "catala_cmd" - let expected_output = Nj.Expr.Var "expected_output" - let test_file_or_folder = Nj.Expr.Var "test_file_or_folder" +(** Some functions that poll the surrounding systems (think [./configure]) *) +module Poll = struct + + (** Scans for a parent directory being the root of the Catala source repo *) + let catala_project_root: File.t option Lazy.t = + let rec aux dir = + if Sys.file_exists File.(dir / "catala.opam") then Some dir + else + let dir' = File.dirname dir in + if dir' = dir then None else aux dir' + in + lazy (aux (Sys.getcwd ())) + + let exec_dir: File.t = + (* Do not use Sys.executable_name, which may resolve symlinks: we want the original path. + (e.g. _build/install/default/bin/foo is a symlink) *) + Filename.dirname Sys.argv.(0) + + let clerk_exe: File.t Lazy.t = lazy Sys.executable_name + + let catala_exe: File.t Lazy.t = lazy ( + let f = File.(exec_dir / "catala") in + if Sys.file_exists f then f else + match Lazy.force catala_project_root with + | Some root -> File.(root/"_build"/"default"/"compiler"/"catala.exe") + | None -> "catala" (* Dynamically resolved from PATH *) + ) + + let build_dir: File.t Lazy.t = lazy ( + File.(Sys.getcwd () / "_build") + ) + (* TODO: probably detect the project root and put this there instead *) + + (** Locates the main [lib] directory containing the OCaml libs *) + let ocaml_libdir: File.t Lazy.t = + lazy + (try String.trim (File.process_out "opam" ["var"; "lib"]) + with Failure _ -> ( + try String.trim (File.process_out "ocamlc" ["-where"]) + with Failure _ -> ( + match File.(check_directory (exec_dir /../ "lib")) with + | Some d -> d + | None -> + Message.raise_error + "Could not locate the OCaml library directory, make sure OCaml or \ + opam is installed"))) + + (** Locates the directory containing the OCaml runtime to link to *) + let ocaml_runtime_dir: File.t Lazy.t = + lazy + (let d = + match Lazy.force catala_project_root with + | Some root -> + (* Relative dir when running from catala source *) + File.( + root + / "_build" + / "install" + / "default" + / "lib" + / "catala" + / "runtime_ocaml") + | None -> ( + match + File.check_directory + File.(exec_dir /../ "lib" / "catala" / "runtime_ocaml") + with + | Some d -> d + | None -> File.(Lazy.force ocaml_libdir / "catala" / "runtime")) + (* FIXME check this, not "runtime_ocaml" ?? *) + in + match File.check_directory d with + | Some dir -> + Message.emit_debug "Catala runtime libraries found at @{%s@}." dir; + dir + | None -> + Message.raise_error + "@[Could not locate the Catala runtime library.@ Make sure that \ + either catala is correctly installed,@ or you are running from the \ + root of a compiled source tree.@]") + + let ocaml_link_flags: string list Lazy.t = lazy ( + let link_libs = + [ + "biniou"; + "easy-format"; + "yojson"; + "ppx_yojson_conv_lib"; + "zarith"; + "dates_calc"; + ] + in + let link_libs_flags = + List.concat_map + (fun lib -> + match File.(check_directory (Lazy.force ocaml_libdir / lib)) with + | None -> + Message.raise_error + "Required OCaml library not found at %a.@ Try `opam \ + install %s'" + File.format File.(Lazy.force ocaml_libdir / lib) + lib + | Some d -> + ["-I"; d; String.map (function '-' -> '_' | c -> c) lib ^ ".cmxa"]) + link_libs + in + let runtime_dir = Lazy.force ocaml_runtime_dir in + link_libs_flags @ [File.(runtime_dir / "runtime_ocaml.cmxa")]) + + let has_command cmd = + let check_cmd = Printf.sprintf "type %s >/dev/null 2>&1" cmd in + Sys.command check_cmd = 0 + + let diff_command = lazy ( + if has_command "patdiff" then + ["patdiff"; "-alt-old"; "reference"; "-alt-new"; "current-output"] + else + ["diff"; "-u"; "-b"; "--color"; "--label"; "referenec"; "--label"; "current-output"] + ) - let name = function - | Nj.Expr.Var n -> n - | _ -> invalid_arg "Clerk_driver.Var.name" end -let pipe_diff_cmd = - let open Nj.Expr in - let has_patdiff = Sys.command "type patdiff >/dev/null 2>&1" = 0 in - if has_patdiff then - Seq - [ - Lit "|"; - Lit "patdiff"; - Seq [Lit "-alt-new"; Lit "current-output"]; - Var.tested_file; - Lit "/dev/stdin"; - ] - else Seq [Lit "| diff -u -b --color"; Var.tested_file; Lit "-"] +(**{1 Building rules}*) -let inline_test_rule catala_exe catala_opts = - let open Nj.Expr in - Nj.Rule.make "inline_tests" - ~command: - (Seq - [ - Lit Sys.argv.(0); - Lit "runtest"; - Lit ("--exe=" ^ catala_exe); - Lit ("--catala-opts=\"" ^ String.escaped catala_opts ^ "\""); - Var.tested_file; - pipe_diff_cmd; - ]) - ~description:(Seq [Lit "INLINE TESTS of file"; Var.tested_file]) +(** Ninja variable names *) +module Var = struct + include Nj.Var -let inline_reset_rule catala_exe catala_opts = - let open Nj.Expr in - Nj.Rule.make "inline_tests_reset" - ~command: - (Seq - [ - Lit Sys.argv.(0); - Lit "runtest"; - Lit ("--exe=" ^ catala_exe); - Lit ("--catala-opts=" ^ catala_opts); - Lit "--reset"; - Var.tested_file; - ]) - ~description:(Seq [Lit "RESET INLINE TESTS of file"; Var.tested_file]) + (** Global vars: always defined, at toplevel *) -let add_reset_rules_aux - ~(redirect : string) - ~(rule_name : string) - (catala_exe_opts : string) - (rules : Rule.t Nj.RuleMap.t) : Rule.t Nj.RuleMap.t = - let reset_common_cmd_exprs = - Nj.Expr. - [ - Var.catala_cmd; - Var.tested_file; - Lit redirect; - Var.expected_output; - Lit "2>&1"; - Lit "|| true"; - ] + let ninja_required_version = make "ninja_required_version" + let builddir = make "builddir" + let clerk_exe = make "CLERK_EXE" + let catala_exe = make "CATALA_EXE" + let catala_flags = make "CATALA_FLAGS" + let clerk_flags = make "CLERK_FLAGS" + let ocamlopt_exe = make "OCAMLOPT_EXE" + let ocamlopt_flags = make "OCAMLOPT_FLAGS" + let runtime_ocaml_libs = make "RUNTIME_OCAML_LIBS" + let diff = make "DIFF" + + let catala_tests = make "tests" + (** Accumulator for all tests *) + + let module_dir module_name = make ("module_dir_" ^ module_name) + let module_src module_name = make ("module_src_" ^ module_name) + (** Source file of a given Catala module *) + + (** Rule vars, Used in specific rules *) + + let input = make "in" + let output = make "out" + let modules_src = make "modules_src" + let include_flags = make "include_flags" + + (* let scope = make "scope" *) + let test_id = make "test-id" + let test_out = make "test-out" + let test_command = make "test-command" + + let ( ! ) = Var.v +end + +let base_bindings catala_exe catala_flags = + [ + Nj.binding Var.ninja_required_version ["1.7"]; (* use of implicit outputs *) + Nj.binding Var.builddir [Lazy.force Poll.build_dir]; + + Nj.binding Var.clerk_exe [Lazy.force Poll.clerk_exe]; + Nj.binding Var.catala_exe [match catala_exe with Some e -> e | None -> Lazy.force Poll.catala_exe]; + Nj.binding Var.catala_flags ("--build-dir" :: Var.(!builddir) :: catala_flags); + Nj.binding Var.clerk_flags ("-e" :: Var.(!catala_exe) :: (List.map (fun f -> "--catala-opts=" ^ f) catala_flags)); + Nj.binding Var.ocamlopt_exe ["ocamlopt"]; + Nj.binding Var.ocamlopt_flags ["-I"; Lazy.force Poll.ocaml_runtime_dir]; + Nj.binding Var.runtime_ocaml_libs (Lazy.force Poll.ocaml_link_flags); + Nj.binding Var.diff (Lazy.force Poll.diff_command); +] + +let static_base_rules = + let open Var in + [ + (* Nj.rule "interpret-scope" + * ~command:[!catala_exe; "interpret"; !catala_flags; !modules_src; !input; "-s"; !scope] + * ~description:[""; "run"; !scope; "⇐"; !input]; *) + + Nj.rule "ocaml" + ~command:[!catala_exe; "ocaml"; !catala_flags; !modules_src; !input; "-o"; !output] + ~description:[""; "ocaml"; "⇒"; !output]; + + Nj.rule "ocaml-module" + ~command:[!ocamlopt_exe; "-shared"; !include_flags; !ocamlopt_flags; !input; "-o"; !output] + ~description:[""; "⇒"; !output]; + + Nj.rule "ocaml-exec" + ~command:[!ocamlopt_exe; !ocamlopt_flags; !runtime_ocaml_libs; !input; "-o"; !output] + ~description:[""; "⇒"; !output]; + + Nj.rule "out-test" + ~command:[!catala_exe; !test_command; !catala_flags; !input; "2>&1"; "|"; !diff; !test_out; "/dev/stdin"] + ~description:[""; "test"; !test_id; "⇐"; !input; "("^ !test_command ^ ")"]; + + Nj.rule "out-reset" + ~command:[!catala_exe; !test_command; !catala_flags; !input; ">"; !output; "2>&1"; "||"; "true"] + ~description:[""; "reset"; !test_id; "⇐"; !input; "("^ !test_command ^ ")"]; + + Nj.rule "inline-tests" + ~command:[!clerk_exe; "runtest"; !clerk_flags; !input; "2>&1"; "|"; !diff; !input; "/dev/stdin"] + ~description:[""; "inline-tests"; "⇐"; !input]; + + Nj.rule "inline-reset" + ~command:[!clerk_exe; "runtest"; !clerk_flags; !input; "--reset"] + ~description:[""; "inline-reset"; "⇐"; !input] + ] + +let gen_module_def (item: catala_build_item) : Nj.ninja = + match item.module_def with + | None -> Seq.empty + | Some modname -> + List.to_seq [ + Nj.binding (Var.module_dir modname) [Filename.dirname item.file_name]; + Nj.binding (Var.module_src modname) [Filename.basename item.file_name]; + ] + +let gen_build_statements (item: catala_build_item) : Nj.ninja = + let open File in + let ( ! ) = Var.( ! ) in + let src = item.file_name in + let modules = List.rev item.used_modules in + let header = + Nj.comment ("\nDefinitions from " ^ src) in - let reset_rule = - Nj.Rule.make rule_name - ~command:Nj.Expr.(Seq (Lit catala_exe_opts :: reset_common_cmd_exprs)) - ~description: - Nj.Expr.( - Seq - [ - Lit "RESET"; - Lit "file"; - Var.tested_file; - Lit "with the"; - Var.catala_cmd; - Lit "command"; - ]) + let ocaml = + Nj.build "ocaml" + ~inputs:[src] + ~implicit_in:((if modules = [] then [] else [!Var.modules_src]) @ item.included_files) + ~outputs:[!Var.builddir / src -.- "ml"] + ~vars:(if modules = [] then [] else + [Var.modules_src, + List.map (fun m -> !(Var.module_dir m) / !(Var.module_src m)) modules]) in - Nj.RuleMap.(rules |> add reset_rule.name reset_rule) - -let add_test_rules_aux - ~(rule_name : string) - (catala_exe_opts : string) - (rules : Rule.t Nj.RuleMap.t) : Rule.t Nj.RuleMap.t = - let test_rule = - Nj.Rule.make rule_name - ~command: - Nj.Expr.( - Seq - (Lit catala_exe_opts - :: [ - Var.catala_cmd; - Var.tested_file; - Lit "2>&1 | colordiff -u -b"; - Var.expected_output; - Lit "-"; - ])) - ~description: - Nj.Expr.( - Seq - [ - Lit "TEST on file"; - Var.tested_file; - Lit "with the"; - Var.catala_cmd; - Lit "command"; - ]) - in - Nj.RuleMap.(rules |> add test_rule.name test_rule) - -(** [add_reset_rules catala_exe_opts rules] adds ninja rules used to reset test - files into [rules] and returns it.*) -let add_reset_rules (catala_exe_opts : string) (rules : Rule.t Nj.RuleMap.t) : - Rule.t Nj.RuleMap.t = - add_reset_rules_aux ~rule_name:"reset_rule" ~redirect:">" catala_exe_opts - rules - -(** [add_test_rules catala_exe_opts rules] adds ninja rules used to test files - into [rules] and returns it.*) -let add_test_rules (catala_exe_opts : string) (rules : Rule.t Nj.RuleMap.t) : - Rule.t Nj.RuleMap.t = - add_test_rules_aux ~rule_name:"test_rule" catala_exe_opts rules - -(** [ninja_start catala_exe] returns the inital [ninja] data structure with - rules needed to reset and test files. *) -let ninja_start (catala_exe : string) (catala_opts : string) : ninja = - let catala_exe_opts = catala_exe ^ " " ^ catala_opts in - let add_rule r rules = Nj.RuleMap.add r.Nj.Rule.name r rules in - let run_and_display_final_message = - Nj.Rule.make "run_and_display_final_message" - ~command:Nj.Expr.(Seq [Lit ":"]) - ~description: - Nj.Expr.(Seq [Lit "All tests"; Var.test_file_or_folder; Lit "passed!"]) - in - { - rules = - Nj.RuleMap.( - empty - |> add_reset_rules catala_exe_opts - |> add_test_rules catala_exe_opts - |> add_rule (inline_test_rule catala_exe catala_opts) - |> add_rule (inline_reset_rule catala_exe catala_opts) - |> add run_and_display_final_message.name run_and_display_final_message); - builds = Nj.BuildMap.empty; - } - -let collect_inline_ninja_builds - (ninja : ninja) - (tested_file : string) - _lang - (reset_test_outputs : bool) : (string * ninja) option = - if not (Clerk_runtest.has_inline_tests tested_file) then None - else - let ninja = - let vars = [Var.(name tested_file), Nj.Expr.Lit tested_file] in - let rule_to_call = - if reset_test_outputs then "inline_tests_reset" else "inline_tests" + (* let interpret_deps = + * let inputs = + * Nj.build "phony" + * ~implicit_in:(List.map (fun m -> src /../ m ^ ".cmxs") modules) + * in *) + let ocamlopt = + let implicit_out = + [!Var.builddir / src -.- "cmi"; !Var.builddir / src -.- "cmx"; (* src -.- "cmt"; src -.- "o" *)] + in + let vars = [Var.include_flags, ["-I"; !Var.builddir / Filename.dirname src]] in + match item.module_def with + | Some modname -> + Nj.build "ocaml-module" + ~inputs:[!Var.builddir / src -.- "ml"] + ~implicit_in:(List.map (fun m -> src /../ m ^ ".cmi") modules) + ~outputs:[!Var.builddir / src /../ modname ^ ".cmxs"] + ~implicit_out + ~vars + | None -> + let inputs = + List.map (fun m -> !Var.builddir / src /../ m ^ ".cmx") modules @ + [ !Var.builddir / src -.- "ml" ] in - let rule_output = tested_file ^ ".out" in - { - ninja with - builds = - Nj.BuildMap.add rule_output - (Nj.Build.make_with_vars ~outputs:[Nj.Expr.Lit rule_output] - ~rule:rule_to_call ~vars) - ninja.builds; - } - in - let test_name = - tested_file - |> (if reset_test_outputs then Printf.sprintf "reset_file_%s" - else Printf.sprintf "test_file_%s") - |> Nj.Build.unpath - in - Some - ( test_name, - { - ninja with - builds = - Nj.BuildMap.add test_name - (Nj.Build.make_with_inputs ~outputs:[Nj.Expr.Lit test_name] - ~rule:"phony" - ~inputs:[Nj.Expr.Lit (tested_file ^ ".out")]) - ninja.builds; - } ) - -(** [collect_all_ninja_build ninja tested_file catala_exe catala_opts reset_test_outputs] - creates and returns all ninja build statements needed to test the - [tested_file]. *) -let collect_all_ninja_build - (ninja : ninja) - (tested_file : string) - lang - (reset_test_outputs : bool) : (string * ninja) option = - let expected_outputs = - (scan_catala_file tested_file lang).legacy_tests + Nj.build "ocaml-exec" + ~inputs + ~outputs:[!Var.builddir / src -.- "exe"] + ~implicit_out + ~vars in - if expected_outputs = [] then ( - Message.emit_debug "No expected outputs were found for test file %s" - tested_file; - None) - else - let ninja, test_names = - List.fold_left - (fun (ninja, test_names) expected_output -> - let expected_output_file = - expected_output.output_dir - ^ Filename.basename expected_output.tested_filename - ^ "." - ^ expected_output.id - in - let vars = - [ - Var.(name catala_cmd), Nj.Expr.Lit expected_output.cmd; - Var.(name tested_file), Nj.Expr.Lit tested_file; - Var.(name expected_output), Nj.Expr.Lit expected_output_file; - ] - and rule_to_call = - if reset_test_outputs then "reset_rule" else "test_rule" - in - let ninja_add_new_build - (rule_output : string) - (rule : string) - (vars : (string * Nj.Expr.t) list) - (ninja : ninja) : ninja = - { - ninja with - builds = - Nj.BuildMap.add rule_output - (Nj.Build.make_with_vars ~outputs:[Nj.Expr.Lit rule_output] - ~rule ~vars) - ninja.builds; - } - in - ( ninja_add_new_build - (expected_output_file ^ ".PHONY") - rule_to_call vars ninja, - test_names ^ " $\n " ^ expected_output_file ^ ".PHONY" )) - (ninja, "") expected_outputs + let tests = + let inputs = [src] in + let implicit_in = + List.map (fun m -> !(Var.module_dir m) / !(Var.module_src m)) modules @ + List.map (fun m -> !Var.builddir / !(Var.module_dir m) / m ^ ".cmxs") modules @ + item.included_files in - let test_name = - tested_file - |> (if reset_test_outputs then Printf.sprintf "reset_file_%s" - else Printf.sprintf "test_file_%s") - |> Nj.Build.unpath + let legacy_tests = + List.fold_left (fun acc test -> + let vars = [ + Var.test_id, [test.id]; + Var.test_command, test.cmd; + Var.test_out, [Filename.dirname src / Filename.basename src -.- "out" / !Var.test_id]; + ] in + Nj.build "out-test" ~inputs ~implicit_in ~outputs:["outtest@"^src^"@"^test.id] ~vars :: + Nj.build "out-reset" ~inputs ~implicit_in ~outputs:["outtest-reset@"^src^"@"^test.id] ~implicit_out:[!Var.test_out] ~vars :: + acc + ) + [] item.legacy_tests in - Some - ( test_name, - { - ninja with - builds = - Nj.BuildMap.add test_name - (Nj.Build.make_with_inputs ~outputs:[Nj.Expr.Lit test_name] - ~rule:"phony" ~inputs:[Nj.Expr.Lit test_names]) - ninja.builds; - } ) - -(** [add_root_test_build ninja all_file_names all_test_builds] add the 'test' - ninja build declaration calling the rule 'run_and_display_final_message' for - [all_test_builds] which correspond to [all_file_names]. *) -let add_root_test_build - (ninja : ninja) - (all_file_names : string list) - (all_test_builds : string) : ninja = - let file_names_str = - List.hd all_file_names - ^ "" - ^ List.fold_left - (fun acc name -> acc ^ "; " ^ name) - "" (List.tl all_file_names) + let inline_tests = + if not item.has_inline_tests then [] else + [ + Nj.build "inline-tests" ~inputs ~implicit_in ~outputs:["inline@" ^ src]; + Nj.build "inline-reset" ~inputs ~implicit_in ~outputs:["inline-reset@" ^ src]; + ] + in + let tests = + if item.legacy_tests = [] && not item.has_inline_tests then [] + else + [Nj.build "phony" + ~outputs:["test@" ^ src] + ~inputs: + ((if item.has_inline_tests then + ["inline@" ^ item.file_name] + else []) @ + List.map (fun test -> + "outtest@" ^ item.file_name ^"@"^ test.id) + item.legacy_tests); + Nj.build "phony" + ~outputs:["test-reset@" ^ src] + ~inputs: + ((if item.has_inline_tests then + ["inline-reset@" ^ item.file_name] + else []) @ + List.map (fun test -> + "outtest-reset@" ^ item.file_name ^"@"^ test.id) + item.legacy_tests)] + in + legacy_tests @ inline_tests @ tests in - { - ninja with - builds = - Nj.BuildMap.add "test" - (Nj.Build.make_with_vars_and_inputs ~outputs:[Nj.Expr.Lit "test"] - ~rule:"run_and_display_final_message" - ~inputs:[Nj.Expr.Lit all_test_builds] - ~vars: - [ - ( Var.(name test_file_or_folder), - Nj.Expr.Lit ("in [ " ^ file_names_str ^ " ]") ); - ]) - ninja.builds; - } + Seq.concat @@ List.to_seq [ + Seq.return header; + Seq.return ocaml; + Seq.return ocamlopt; + List.to_seq tests; + ] + +let build_statements dir = + (* Unfortunately we need to express the module name bindings first, so need to iterate twice using Seq.memoize *) + scan_tree dir |> Seq.memoize |> fun items -> + Seq.concat @@ List.to_seq [ + Seq.flat_map gen_module_def items; + Seq.flat_map gen_build_statements items; + Seq.return (Nj.comment "\n- Global targets - #\n"); + let items_with_tests = Seq.filter + (fun item -> item.legacy_tests <> [] || item.has_inline_tests) + items + in + if Seq.is_empty items_with_tests then Seq.empty + else + let get_targets prefix = Seq.map (fun item -> prefix ^ item.file_name) items_with_tests |> List.of_seq in + List.to_seq [ + Nj.build "phony" ~outputs:["test"] ~inputs:(get_targets "test@"); + Nj.build "phony" ~outputs:["test-reset"] ~inputs:(get_targets "test-reset@"); + ] + ] + +let global_build_targets = + Nj.build "phony" ~outputs:["test"] ~inputs:[Var.(!catala_tests)] + +let gen_ninja_file catala_exe catala_flags dir = + let ( @+ ) = Seq.append in + Seq.return (Nj.Comment (Printf.sprintf "File generated by Clerk v.%s\n" version)) @+ + Seq.return (Nj.Comment "- Global variables - #\n") @+ + List.to_seq (base_bindings catala_exe catala_flags) @+ + Seq.return (Nj.Comment "- Base rules - #\n") @+ + List.to_seq static_base_rules @+ + Seq.return (Nj.Comment "- Project-specific build statements - #") @+ + build_statements dir (**{1 Running}*) @@ -563,124 +646,6 @@ let run_file Message.emit_debug "Running: %s" command; Sys.command command -(** {1 Driver} *) - -type ninja_building_context = { - last_valid_ninja : ninja; - curr_ninja : ninja option; - all_file_names : string list; - all_test_builds : string; - all_failed_names : string list; -} -(** Record used to keep tracks of the current context while building the - [Ninja_utils.ninja].*) - -(** [ninja_building_context_init ninja_init] returns the empty context - corresponding to [ninja_init]. *) -let ninja_building_context_init (ninja_init : Nj.ninja) : ninja_building_context - = - { - last_valid_ninja = ninja_init; - curr_ninja = Some ninja_init; - all_file_names = []; - all_test_builds = ""; - all_failed_names = []; - } - -(** [collect_in_directory ctx file_or_folder ninja_start reset_test_outputs] - updates the building context [ctx] by adding new ninja build statements - needed to test files in [folder].*) -let collect_in_folder - (ctx : ninja_building_context) - (folder : string) - (ninja_start : Nj.ninja) - (reset_test_outputs : bool) : ninja_building_context = - let ninja, test_file_names = - let collect f (ninja, test_file_names) file lang = - match f ninja file lang reset_test_outputs with - | None -> - (* Skips none Catala file. *) - ninja, test_file_names - | Some (test_file_name, ninja) -> - ninja, test_file_names ^ " $\n " ^ test_file_name - in - Seq.fold_left - (fun acc (file, lang) -> - let acc = collect collect_all_ninja_build acc file lang in - collect collect_inline_ninja_builds acc file lang) - (ninja_start, "") - (File.scan_tree (fun f -> match get_lang f with Some l -> Some (f, l) | None -> None) folder) - in - let test_dir_name = - Printf.sprintf "test_dir_%s" (folder |> Nj.Build.unpath) - in - let curr_ninja = - if 0 = String.length test_file_names then None - else - Some - { - ninja with - builds = - Nj.BuildMap.add test_dir_name - (Nj.Build.make_with_vars_and_inputs - ~outputs:[Nj.Expr.Lit test_dir_name] - ~rule:"run_and_display_final_message" - ~inputs:[Nj.Expr.Lit test_file_names] - ~vars: - [ - ( Var.(name test_file_or_folder), - Nj.Expr.Lit ("in folder '" ^ folder ^ "'") ); - ]) - ninja.builds; - } - in - if Option.is_some curr_ninja then - { - ctx with - last_valid_ninja = ninja_start; - curr_ninja; - all_file_names = folder :: ctx.all_file_names; - all_test_builds = ctx.all_test_builds ^ " $\n " ^ test_dir_name; - } - else - { - ctx with - last_valid_ninja = ninja_start; - curr_ninja; - all_failed_names = folder :: ctx.all_failed_names; - } - -(** [collect_in_file ctx file_or_folder ninja_start reset_test_outputs] updates - the building context [ctx] by adding new ninja build statements needed to - test the [tested_file].*) -let collect_in_file - (ctx : ninja_building_context) - (tested_file : string) - lang - (ninja_start : Nj.ninja) - (reset_test_outputs : bool) : ninja_building_context = - let add ctx f ninja_start tested_file = - match f ninja_start tested_file lang reset_test_outputs with - | Some (test_file_name, ninja) -> - { - last_valid_ninja = ninja; - curr_ninja = Some ninja; - all_file_names = tested_file :: ctx.all_file_names; - all_test_builds = ctx.all_test_builds ^ " $\n " ^ test_file_name; - all_failed_names = List.filter (( <> ) tested_file) ctx.all_failed_names; - } - | None -> - { - ctx with - last_valid_ninja = ninja_start; - curr_ninja = None; - all_failed_names = tested_file :: ctx.all_failed_names; - } - in - let ctx = add ctx collect_all_ninja_build ninja_start tested_file in - let ninja = Option.value ~default:ninja_start ctx.curr_ninja in - add ctx collect_inline_ninja_builds ninja tested_file - (** {1 Return code values} *) let return_ok = 0 @@ -688,26 +653,6 @@ let return_err = 1 (** {1 Driver} *) -(** [add_root_test_build ctx files_or_folders reset_test_outputs] updates the - [ctx] by adding ninja build statements needed to test or - [reset_test_outputs] [files_or_folders]. *) -let add_test_builds - (ctx : ninja_building_context) - (files_or_folders : string list) - (reset_test_outputs : bool) : ninja_building_context = - files_or_folders - |> List.fold_left - (fun ctx file_or_folder -> - let curr_ninja = - match ctx.curr_ninja with - | Some ninja -> ninja - | None -> ctx.last_valid_ninja - in - if Sys.is_directory file_or_folder then - collect_in_folder ctx file_or_folder curr_ninja reset_test_outputs - else collect_in_file ctx file_or_folder (Option.get (get_lang file_or_folder)) curr_ninja reset_test_outputs) - ctx - let makeflags_to_ninja_flags (makeflags : string option) = match makeflags with | None -> "" @@ -723,85 +668,70 @@ let makeflags_to_ninja_flags (makeflags : string option) = let driver (files_or_folders : string list) (command : [> ]) + (chdir : string option) (catala_exe : string option) - (catala_opts : string option) + (catala_opts : string list) (makeflags : string option) (debug : bool) - (scope : string option) + (color : Cli.when_enum) + (_scope : string option) (reset_test_outputs : bool) (ninja_output : string option) : int = try - let _options = Cli.enforce_globals ~debug () in + Option.iter Sys.chdir chdir; + let _options = Cli.enforce_globals ~debug ~color () in let ninja_flags = makeflags_to_ninja_flags makeflags in - let files_or_folders = List.sort_uniq String.compare files_or_folders - and catala_exe = Option.fold ~none:"catala" ~some:Fun.id catala_exe - and catala_opts = Option.fold ~none:"" ~some:Fun.id catala_opts - and with_ninja_output k = + (* let ninja_flags = if debug then ninja_flags ^ "-d explain" else ninja_flags in *) + (* let files_or_folders = List.sort_uniq String.compare files_or_folders + * and catala_exe = Option.fold ~none:"catala" ~some:Fun.id catala_exe + * and catala_opts = Option.fold ~none:"" ~some:Fun.id catala_opts *) + (* let catala_opts = match color with + * | Cli.Always -> "--color=always"::catala_opts + * | Cli.Auto when Unix.(isatty stderr) -> "--color=always"::catala_opts + * | _ -> catala_opts + * in *) + let with_ninja_output k = match ninja_output with | Some f -> k f - | None -> ( - let f = Filename.temp_file "clerk_build_" ".ninja" in - match k f with - | exception e -> - if not debug then Sys.remove f; - Message.emit_debug "Ninja file left in @{%s@} for reference" f; - raise e - | r -> - (* Sys.remove f; *) - r) + | None -> File.with_temp_file "clerk_build_" ".ninja" k in match command with | `Test -> ( - Message.emit_debug "building ninja rules..."; - let ctx = - add_test_builds - (ninja_building_context_init (ninja_start catala_exe catala_opts)) - files_or_folders reset_test_outputs - in - let there_is_some_fails = 0 <> List.length ctx.all_failed_names in - let ninja = - match ctx.curr_ninja with - | Some ninja -> ninja - | None -> ctx.last_valid_ninja - in - if there_is_some_fails then - List.iter - (Message.emit_warning "No test case found for @{%s@}") - ctx.all_failed_names; - if 0 = List.compare_lengths ctx.all_failed_names files_or_folders then - return_ok - else + Message.emit_debug "building ninja rules..."; with_ninja_output - @@ fun nin -> - match - File.with_formatter_of_file nin (fun fmt -> - Nj.format fmt - (add_root_test_build ninja ctx.all_file_names - ctx.all_test_builds)) - with - | () -> - let ninja_cmd = - "ninja -k 0 -f " ^ nin ^ " " ^ ninja_flags ^ " test" - in - Message.emit_debug "executing '%s'..." ninja_cmd; - Sys.command ninja_cmd - | exception Sys_error e -> Message.raise_error "can not write in %s" e) - | `Run -> ( - match scope with - | Some scope -> - let res = - List.fold_left - (fun ret f -> ret + run_file f catala_exe catala_opts scope) - 0 files_or_folders + @@ fun nin_file -> + File.with_formatter_of_file nin_file (fun nin_ppf -> + Nj.format nin_ppf + (gen_ninja_file catala_exe catala_opts ".")); + let targets = + let target = if reset_test_outputs then "test-reset" else "test" in + match files_or_folders with + | [] -> [target] + | files -> List.map (fun f -> target ^ "@" ^ f) files in - if 0 <> res then return_err else return_ok - | None -> - Message.raise_error "Please provide a scope to run with the -s option") + let ninja_cmd = + String.concat " " ( + "ninja -k 0 -f" :: nin_file :: ninja_flags :: targets) + in + Message.emit_debug "executing '%s'..." ninja_cmd; + Sys.command ninja_cmd) + | `Run -> (assert false + (* match scope with + * | Some scope -> + * let res = + * List.fold_left + * (fun ret f -> ret + run_file f catala_exe catala_opts scope) + * 0 files_or_folders + * in + * if 0 <> res then return_err else return_ok + * | None -> + * Message.raise_error "Please provide a scope to run with the -s option" *)) | `Runtest -> ( match files_or_folders with | [f] -> - Clerk_runtest.run_inline_tests ~reset:reset_test_outputs f catala_exe - (List.filter (( <> ) "") (String.split_on_char ' ' catala_opts)); + Clerk_runtest.run_inline_tests ~reset:reset_test_outputs f + (Option.value ~default:"catala" catala_exe) + catala_opts; 0 | _ -> Message.raise_error "Please specify a single catala file to test") with Message.CompilerError content -> diff --git a/build_system/clerk_runtest.ml b/build_system/clerk_runtest.ml index 9c554113..ec583243 100644 --- a/build_system/clerk_runtest.ml +++ b/build_system/clerk_runtest.ml @@ -205,7 +205,9 @@ let run_inline_tests else catala_exe in let cmd = - Array.of_list ((catala_exe :: catala_opts) @ test.params @ [file]) + match test.params with + | cmd0 :: flags -> Array.of_list (catala_exe :: cmd0 :: catala_opts @ flags @ [file]) + | [] -> Array.of_list (catala_exe :: catala_opts @ [file]) in let env = Unix.environment () diff --git a/build_system/ninja_utils.ml b/build_system/ninja_utils.ml index 42c65973..c612304f 100644 --- a/build_system/ninja_utils.ml +++ b/build_system/ninja_utils.ml @@ -14,18 +14,41 @@ License for the specific language governing permissions and limitations under the License. *) +(** Ninja variable names *) +module Var = struct + type t = V of string + + let make s = V s + + let name (V v) = v + + let v (V v) = Printf.sprintf "${%s}" v + +end + module Expr = struct - type t = Lit of string | Var of string | Seq of t list + type t = string list - let rec format fmt = function - | Lit s -> Format.pp_print_string fmt s - | Var s -> Format.fprintf fmt "$%s" s - | Seq ls -> format_list fmt ls - - and format_list fmt ls = + let format = Format.pp_print_list ~pp_sep:(fun fmt () -> Format.pp_print_char fmt ' ') - format fmt ls + (fun fmt s -> + Format.pp_print_string fmt + (Re.replace_string Re.(compile space) ~by:"$ " s)) +end + +module Binding = struct + type t = Var.t * Expr.t + let make var e = var, e + let format ~global ppf (v, e) = + if not global then Format.pp_print_string ppf " "; + Format.fprintf ppf "%s = %a" + (Var.name v) + Expr.format e; + if global then Format.pp_print_newline ppf () + + let format_list ~global ppf l = + Format.pp_print_list ~pp_sep:Format.pp_print_newline (format ~global) ppf l end module Rule = struct @@ -35,62 +58,78 @@ module Rule = struct { name; command; description = Option.some description } let format fmt rule = - let format_description fmt = function - | Some e -> Format.fprintf fmt " description = %a\n" Expr.format e - | None -> Format.fprintf fmt "\n" + let bindings = + Binding.make (Var.make "command") rule.command :: + Option.(to_list (map (fun d -> Binding.make (Var.make "description") d) rule.description)) in - Format.fprintf fmt "rule %s\n command = %a\n%a" rule.name Expr.format - rule.command format_description rule.description + Format.fprintf fmt "rule %s\n%a" + rule.name (Binding.format_list ~global:false) bindings end module Build = struct type t = { - outputs : Expr.t list; rule : string; - inputs : Expr.t list option; - vars : (string * Expr.t) list; + inputs : Expr.t option; + implicit_in : Expr.t; + outputs : Expr.t; + implicit_out : Expr.t option; + vars : Binding.t list; } - let make ~outputs ~rule = { outputs; rule; inputs = Option.none; vars = [] } + let make ?inputs ?(implicit_in=[]) ~outputs ?implicit_out ?(vars=[]) rule = + { rule; inputs; implicit_in; outputs; implicit_out; vars } - let make_with_vars ~outputs ~rule ~vars = - { outputs; rule; inputs = Option.none; vars } - - let make_with_inputs ~outputs ~rule ~inputs = - { outputs; rule; inputs = Option.some inputs; vars = [] } - - let make_with_vars_and_inputs ~outputs ~rule ~inputs ~vars = - { outputs; rule; inputs = Option.some inputs; vars } - - let empty = make ~outputs:[Expr.Lit "empty"] ~rule:"phony" + let empty = make ~outputs:["empty"] "phony" let unpath ?(sep = "-") path = Re.Pcre.(substitute ~rex:(regexp "/") ~subst:(fun _ -> sep)) path - let format fmt build = - let format_inputs fmt = function - | Some exs -> Format.fprintf fmt " %a" Expr.format_list exs - | None -> () - and format_vars fmt vars = - List.iter - (fun (name, exp) -> - Format.fprintf fmt " %s = %a\n" name Expr.format exp) - vars - in - Format.fprintf fmt "build %a: %s%a\n%a" Expr.format_list build.outputs - build.rule format_inputs build.inputs format_vars build.vars + let format fmt t = + Format.fprintf fmt "build %a%a: %s%a%a%a%a" + Expr.format t.outputs + (Format.pp_print_option + (fun fmt i -> + Format.pp_print_string fmt " | "; + Expr.format fmt i)) + t.implicit_out + t.rule + (Format.pp_print_option + (fun ppf e -> Format.pp_print_char ppf ' '; Expr.format ppf e)) + t.inputs + (fun ppf -> function [] -> () | e -> Format.pp_print_string ppf " | "; Expr.format ppf e) + t.implicit_in + (if t.vars = [] then fun _ () -> () else Format.pp_print_newline) () + (Binding.format_list ~global:false) + t.vars end -module RuleMap : Map.S with type key = String.t = Map.Make (String) -module BuildMap : Map.S with type key = String.t = Map.Make (String) +type def = Comment of string | Binding of Binding.t | Rule of Rule.t | Build of Build.t -type ninja = { rules : Rule.t RuleMap.t; builds : Build.t BuildMap.t } +let comment s = Comment s +let binding v e = Binding (Binding.make v e) +let rule name ~command ~description = + Rule (Rule.make name ~command ~description) +let build ?inputs ?implicit_in ~outputs ?implicit_out ?vars rule = + Build (Build.make ?inputs ?implicit_in ~outputs ?implicit_out ?vars rule) -let empty = { rules = RuleMap.empty; builds = BuildMap.empty } -let format fmt ninja = - let format_for_all iter format = - iter (fun _name rule -> Format.fprintf fmt "%a\n" format rule) +let format_def ppf def = + let () = match def with + | Comment s -> + Format.pp_print_list ~pp_sep:Format.pp_print_newline + (fun ppf s -> + if s <> "" then Format.pp_print_string ppf "# "; + Format.pp_print_string ppf s) + ppf + (String.split_on_char '\n' s) + | Binding b -> Binding.format ~global:true ppf b + | Rule r -> Rule.format ppf r; Format.pp_print_newline ppf () + | Build b -> Build.format ppf b in - format_for_all RuleMap.iter Rule.format ninja.rules; - format_for_all BuildMap.iter Build.format ninja.builds + Format.pp_print_flush ppf () + +type ninja = def Seq.t + +let format ppf t = + Format.pp_print_seq ~pp_sep:Format.pp_print_newline format_def ppf t; + Format.pp_print_newline ppf () diff --git a/build_system/ninja_utils.mli b/build_system/ninja_utils.mli index af7781d0..0f54954e 100644 --- a/build_system/ninja_utils.mli +++ b/build_system/ninja_utils.mli @@ -34,31 +34,35 @@ (** {1 Ninja expressions} *) +(** Ninja variable names, distinguishing binding name ("x") from references in expressions ("$x") *) +module Var : sig + type t + + val make: string -> t + + (** Var base name, used when binding it *) + val name: t -> string + + (** Var reference with a preceding "$", for use in expressoins *) + val v: t -> string +end + + (** Helper module to build ninja expressions. *) module Expr : sig - (** Represents a ninja expression. Which could be either a literal, a - {{:https://ninja-build.org/manual.html#_variables} variable references} - ($_) or a sequence of sub-expressions. - - {b Note:} for now, there are no visible differences between an [Expr.Seq] - and a list of {!type: Expr.t}, indeed, in both cases, one space is added - between each expression -- resp. sub-expression. The difference only comes - from the semantic: an [Expr.Seq] is {b a unique} Ninja expression. *) - type t = - | Lit of string - (* Literal string. *) - | Var of string - (* Variable reference. *) - | Seq of t list - (* Sequence of sub-expressions. *) + (** Ninja expressions are represented as raw string lists, which may contain variables or "$-escapes" *) + type t = string list val format : Format.formatter -> t -> unit (** [format fmt exp] outputs in [fmt] the string representation of the ninja - expression [exp]. *) + expression [exp]. Spaces in individual elements are escaped (but no check is made for e.g. newlines) *) - val format_list : Format.formatter -> t list -> unit - (** [format fmt ls] outputs in [fmt] the string representation of a list [ls] - of ninja expressions [exp] by adding a space between each expression. *) +end + +module Binding : sig + type t = Var.t * Expr.t + val make: Var.t -> Expr.t -> t + val format: global:bool -> Format.formatter -> t -> unit end (** {1 Ninja rules} *) @@ -66,7 +70,7 @@ end (** Helper module to build {{:https://ninja-build.org/manual.html#_rules} ninja rules}. *) module Rule : sig - type t = { name : string; command : Expr.t; description : Expr.t option } + type t (** Represents the minimal ninja rule representation for Clerk: {[ @@ -89,12 +93,7 @@ end (** Helper module to build ninja {{:https://ninja-build.org/manual.html#_build_statements} build statements}. *) module Build : sig - type t = { - outputs : Expr.t list; - rule : string; - inputs : Expr.t list option; - vars : (string * Expr.t) list; - } + type t (** Represents the minimal ninja build statement representation for Clerk: {[ @@ -102,28 +101,15 @@ module Build : sig [] ]}*) - val make : outputs:Expr.t list -> rule:string -> t - (** [make ~outputs ~rule] returns the corresponding ninja {!type:Build.t} with - no {!field:inputs} or {!field:vars}. *) - - val make_with_vars : - outputs:Expr.t list -> rule:string -> vars:(string * Expr.t) list -> t - (** [make_with_vars ~outputs ~rule ~vars] returns the corresponding ninja - {!type:Build.t} with no {!field:inputs}. *) - - val make_with_inputs : - outputs:Expr.t list -> rule:string -> inputs:Expr.t list -> t - (** [make_with_vars ~outputs ~rule ~inputs] returns the corresponding ninja - {!type:Build.t} with no {!field:vars}. *) - - val make_with_vars_and_inputs : - outputs:Expr.t list -> - rule:string -> - inputs:Expr.t list -> - vars:(string * Expr.t) list -> + val make : + ?inputs:Expr.t -> + ?implicit_in:Expr.t -> + outputs:Expr.t -> + ?implicit_out:Expr.t -> + ?vars:(Var.t * Expr.t) list -> + string -> t - (** [make_with_vars ~outputs ~rule ~inputs ~vars] returns the corresponding - ninja {!type: Build.t}. *) + (** [make ~outputs rule] returns the corresponding ninja {!type:Build.t}. *) val empty : t (** [empty] is the minimal ninja {!type:Build.t} with ["empty"] as @@ -139,20 +125,22 @@ module Build : sig [build]. *) end -(** {1 Maps} *) +type def = Comment of string | Binding of Binding.t | Rule of Rule.t | Build of Build.t -module RuleMap : Map.S with type key = String.t -module BuildMap : Map.S with type key = String.t +val comment: string -> def +val binding: Var.t -> Expr.t -> def +val rule: string -> command:Expr.t -> description:Expr.t -> def +val build: + ?inputs:Expr.t -> + ?implicit_in:Expr.t -> + outputs:Expr.t -> + ?implicit_out:Expr.t -> + ?vars:(Var.t * Expr.t) list -> + string -> + def -(** {1 Ninja} *) +val format_def: Format.formatter -> def -> unit -type ninja = { rules : Rule.t RuleMap.t; builds : Build.t BuildMap.t } -(** Represents the minimal ninja architecture (list of rule and build - statements) needed for clerk. *) +type ninja = def Seq.t -val empty : ninja -(** [empty] returns the empty empty ninja structure. *) - -val format : Format.formatter -> ninja -> unit -(** [format fmt build] outputs in [fmt] the string representation of all - [ninja.rules] and [ninja.builds]. *) +val format: Format.formatter -> ninja -> unit diff --git a/compiler/catala_utils/cli.ml b/compiler/catala_utils/cli.ml index 5eb69487..18dc3913 100644 --- a/compiler/catala_utils/cli.ml +++ b/compiler/catala_utils/cli.ml @@ -39,6 +39,7 @@ type options = { mutable message_format : message_format_enum; mutable trace : bool; mutable plugins_dirs : string list; + mutable build_dir : string option; mutable disable_warnings : bool; mutable max_prec_digits : int; } @@ -57,6 +58,7 @@ let globals = message_format = Human; trace = false; plugins_dirs = []; + build_dir = None; disable_warnings = false; max_prec_digits = 20; } @@ -69,6 +71,7 @@ let enforce_globals ?message_format ?trace ?plugins_dirs + ?build_dir ?disable_warnings ?max_prec_digits () = @@ -79,6 +82,7 @@ let enforce_globals Option.iter (fun x -> globals.message_format <- x) message_format; Option.iter (fun x -> globals.trace <- x) trace; Option.iter (fun x -> globals.plugins_dirs <- x) plugins_dirs; + Option.iter (fun x -> globals.build_dir <- x) build_dir; Option.iter (fun x -> globals.disable_warnings <- x) disable_warnings; Option.iter (fun x -> globals.max_prec_digits <- x) max_prec_digits; globals @@ -187,6 +191,11 @@ module Flags = struct in value & opt_all string default & info ["plugin-dir"] ~docv:"DIR" ~env ~doc + let build_dir = + value + & opt (some string) None + & info ["build-dir"] ~docv:"DIR" ~doc:"Directory where build artefacts are expected to be found. This doesn't affect outptuts, but is used when looking up compiled modules." + let disable_warnings = value & flag @@ -210,13 +219,14 @@ module Flags = struct message_format trace plugins_dirs + build_dir disable_warnings max_prec_digits : options = if debug then Printexc.record_backtrace true; (* This sets some global refs for convenience, but most importantly returns the options record. *) enforce_globals ~language ~debug ~color ~message_format ~trace - ~plugins_dirs ~disable_warnings ~max_prec_digits () + ~plugins_dirs ~build_dir ~disable_warnings ~max_prec_digits () in Term.( const make @@ -226,6 +236,7 @@ module Flags = struct $ message_format $ trace $ plugins_dirs + $ build_dir $ disable_warnings $ max_prec_digits) @@ -319,6 +330,7 @@ module Flags = struct "Disables the search for counterexamples. Useful when you want a \ deterministic output from the Catala compiler, since provers can \ have some randomness in them." + end (* Retrieve current version from dune *) diff --git a/compiler/catala_utils/cli.mli b/compiler/catala_utils/cli.mli index 141bc8cf..ae5f34f5 100644 --- a/compiler/catala_utils/cli.mli +++ b/compiler/catala_utils/cli.mli @@ -20,6 +20,8 @@ type backend_lang = En | Fr | Pl (** The usual auto/always/never option argument *) type when_enum = Auto | Always | Never +val when_opt: when_enum Cmdliner.Arg.conv + type message_format_enum = | Human | GNU (** Format of error and warning messages output by the compiler. *) @@ -41,6 +43,7 @@ type options = private { mutable message_format : message_format_enum; mutable trace : bool; mutable plugins_dirs : string list; + mutable build_dir : string option; mutable disable_warnings : bool; mutable max_prec_digits : int; } @@ -60,6 +63,7 @@ val enforce_globals : ?message_format:message_format_enum -> ?trace:bool -> ?plugins_dirs:string list -> + ?build_dir:string option -> ?disable_warnings:bool -> ?max_prec_digits:int -> unit -> diff --git a/compiler/catala_utils/file.ml b/compiler/catala_utils/file.ml index b7743ee5..194a97db 100644 --- a/compiler/catala_utils/file.ml +++ b/compiler/catala_utils/file.ml @@ -120,10 +120,17 @@ let check_directory d = let ( / ) = Filename.concat let dirname = Filename.dirname let ( /../ ) a b = dirname a / b +let ( -.- ) file ext = Filename.chop_extension file ^ "." ^ ext + let equal = String.equal let compare = String.compare let format ppf t = Format.fprintf ppf "\"@{%s@}\"" t +module Set = Set.Make(struct + type nonrec t = t + let compare = compare +end) + let scan_tree f t = let is_dir t = try Sys.is_directory t @@ -136,7 +143,7 @@ let scan_tree f t = Sys.readdir d |> Array.to_list |> List.filter not_hidden - |> List.map (fun t -> d / t) + |> (if d = "." then fun t -> t else List.map (fun t -> d / t)) |> do_files and do_files flist = let dirs, files = @@ -148,4 +155,4 @@ let scan_tree f t = (Seq.concat (Seq.map do_dir (List.to_seq dirs))) (Seq.filter_map f (List.to_seq files)) in - do_dir t + do_files [t] diff --git a/compiler/catala_utils/file.mli b/compiler/catala_utils/file.mli index a1ddfd49..8c3bdf50 100644 --- a/compiler/catala_utils/file.mli +++ b/compiler/catala_utils/file.mli @@ -95,6 +95,9 @@ val dirname : t -> t val ( /../ ) : t -> t -> t (** Sugar for [Filename.dirname "a" / b] *) +val ( -.- ) : t -> string -> t +(** Extension replacement: chops the given filename extension, and replaces it with the given one (which shouldn't contain a dot) *) + val equal : t -> t -> bool (** String comparison no fancy file resolution *) @@ -105,7 +108,9 @@ val format : Format.formatter -> t -> unit (** Formats a filename in a consistent style, with double-quotes and color (when the output supports) *) +module Set: Set.S with type elt = t + val scan_tree : (t -> 'a option) -> t -> 'a Seq.t (** Recursively scans a directory for files. Directories or files matching ".*" or "_*" are ignored. Unreadable files or subdirectories are ignored with a - debug message. *) + debug message. If [t] is a plain file, scan just that non-recursively. *) diff --git a/compiler/shared_ast/interpreter.ml b/compiler/shared_ast/interpreter.ml index c29ef7e6..8deac5bb 100644 --- a/compiler/shared_ast/interpreter.ml +++ b/compiler/shared_ast/interpreter.ml @@ -947,7 +947,7 @@ let load_runtime_modules = function List.iter Dynlink.( fun m -> - try loadfile (adapt_filename (Filename.remove_extension m ^ ".cmo")) + try loadfile (adapt_filename File.(match Cli.globals.build_dir with None -> m -.- "cmo" | Some d -> d / m -.- "cmo")) with Dynlink.Error dl_err -> Message.raise_error "Could not load module %s, has it been suitably compiled?@;\ diff --git a/compiler/shared_ast/interpreter.mli b/compiler/shared_ast/interpreter.mli index 92b50551..639a3edb 100644 --- a/compiler/shared_ast/interpreter.mli +++ b/compiler/shared_ast/interpreter.mli @@ -74,4 +74,4 @@ val interpret_program_lcalc : val load_runtime_modules : string list -> unit (** Dynlink the given runtime modules, in order to make them callable by the - interpreter *) + interpreter. If Cli.globals.build_dir is specified, the runtime module names are assumed to be relative and looked up there. *) diff --git a/compiler/surface/lexer.cppo.ml b/compiler/surface/lexer.cppo.ml index 73c0923a..38e0f8a5 100644 --- a/compiler/surface/lexer.cppo.ml +++ b/compiler/surface/lexer.cppo.ml @@ -820,7 +820,7 @@ let line_dir_arg_re = let lex_line (lexbuf : lexbuf) : (string * L.line_token) option = match%sedlex lexbuf with | eof -> None - | "```catala-test", Star (Compl '\n'), ('\n' | eof) -> + | "```catala-test", hspace, Star (Compl '\n'), ('\n' | eof) -> let str = Utf8.lexeme lexbuf in (try let id = Re.Group.get (Re.exec line_test_id_re str) 1 in diff --git a/tests/test_modules/good/mod_def.catala_en b/tests/test_modules/good/mod_def.catala_en index 33e89205..68a1d46b 100644 --- a/tests/test_modules/good/mod_def.catala_en +++ b/tests/test_modules/good/mod_def.catala_en @@ -22,5 +22,6 @@ scope S: ``` ```catala-test-inline -$ catala module --compile --plugin-dir=../../../_build/default/compiler/plugins --disable_warnings +$ catala typecheck --disable_warnings +[RESULT] Typechecking successful! ```