From f5c03e19eff3d18425dc559ad4ba3a142c3dd69b Mon Sep 17 00:00:00 2001 From: Rijnard van Tonder Date: Tue, 9 Apr 2019 03:18:31 -0400 Subject: [PATCH] Initial code commit --- .gitignore | 3 + LICENSE | 2 +- Makefile | 27 ++ README.md | 172 ++++++- comby | 1 + comby.opam | 13 + dune | 3 + dune-project | 2 + lib/comby.ml | 5 + lib/comby.mli | 5 + lib/dune | 14 + lib/language/ast.ml | 30 ++ lib/language/dune | 5 + lib/language/parser.ml | 25 ++ lib/language/rule.ml | 233 ++++++++++ lib/language/rule.mli | 20 + lib/language/syntax.ml | 11 + lib/match/dune | 5 + lib/match/environment.ml | 74 +++ lib/match/environment.mli | 26 ++ lib/match/location.ml | 14 + lib/match/match.ml | 6 + lib/match/match.mli | 67 +++ lib/match/match_context.ml | 12 + lib/match/range.ml | 10 + lib/match/types.ml | 8 + lib/matchers/bash.ml | 28 ++ lib/matchers/c.ml | 22 + lib/matchers/c.mli | 4 + lib/matchers/configuration.ml | 13 + lib/matchers/dune | 5 + lib/matchers/generic.ml | 20 + lib/matchers/generic.mli | 4 + lib/matchers/go.ml | 10 + lib/matchers/go.mli | 4 + lib/matchers/html.ml | 17 + lib/matchers/matcher.ml | 401 +++++++++++++++++ lib/matchers/matcher.mli | 3 + lib/matchers/matchers.ml | 10 + lib/matchers/matchers.mli | 10 + lib/matchers/python.ml | 20 + lib/matchers/types.ml | 44 ++ lib/parsers/comments.ml | 69 +++ lib/parsers/comments.mli | 11 + lib/parsers/delimiters.ml | 18 + lib/parsers/dune | 5 + lib/parsers/string_literals.ml | 57 +++ lib/rewriter/dune | 5 + lib/rewriter/rewrite.ml | 112 +++++ lib/rewriter/rewrite.mli | 22 + lib/rewriter/rewrite_template.ml | 63 +++ lib/rewriter/rewrite_template.mli | 10 + lib/statistics/dune | 5 + lib/statistics/statistics.ml | 34 ++ lib/statistics/statistics.mli | 14 + lib/statistics/time.ml | 25 ++ src/dune | 12 + src/main.ml | 385 ++++++++++++++++ src/specification.ml | 32 ++ test/dune | 21 + test/test_bash.ml | 77 ++++ test/test_c.ml | 105 +++++ test/test_c_separators.ml | 36 ++ test/test_c_style_comments.ml | 225 ++++++++++ test/test_cli.ml | 178 ++++++++ test/test_generic.ml | 408 +++++++++++++++++ test/test_go.ml | 101 +++++ test/test_integration.ml | 295 ++++++++++++ test/test_match_rule.ml | 719 ++++++++++++++++++++++++++++++ test/test_rewrite_parts.ml | 223 +++++++++ test/test_rewrite_rule.ml | 169 +++++++ test/test_statistics.ml | 87 ++++ test/test_string_literals.ml | 412 +++++++++++++++++ 73 files changed, 5341 insertions(+), 2 deletions(-) create mode 100644 .gitignore create mode 100644 Makefile create mode 120000 comby create mode 100644 comby.opam create mode 100644 dune create mode 100644 dune-project create mode 100644 lib/comby.ml create mode 100644 lib/comby.mli create mode 100644 lib/dune create mode 100644 lib/language/ast.ml create mode 100644 lib/language/dune create mode 100644 lib/language/parser.ml create mode 100644 lib/language/rule.ml create mode 100644 lib/language/rule.mli create mode 100644 lib/language/syntax.ml create mode 100644 lib/match/dune create mode 100644 lib/match/environment.ml create mode 100644 lib/match/environment.mli create mode 100644 lib/match/location.ml create mode 100644 lib/match/match.ml create mode 100644 lib/match/match.mli create mode 100644 lib/match/match_context.ml create mode 100644 lib/match/range.ml create mode 100644 lib/match/types.ml create mode 100644 lib/matchers/bash.ml create mode 100644 lib/matchers/c.ml create mode 100644 lib/matchers/c.mli create mode 100644 lib/matchers/configuration.ml create mode 100644 lib/matchers/dune create mode 100644 lib/matchers/generic.ml create mode 100644 lib/matchers/generic.mli create mode 100644 lib/matchers/go.ml create mode 100644 lib/matchers/go.mli create mode 100644 lib/matchers/html.ml create mode 100644 lib/matchers/matcher.ml create mode 100644 lib/matchers/matcher.mli create mode 100644 lib/matchers/matchers.ml create mode 100644 lib/matchers/matchers.mli create mode 100644 lib/matchers/python.ml create mode 100644 lib/matchers/types.ml create mode 100644 lib/parsers/comments.ml create mode 100644 lib/parsers/comments.mli create mode 100644 lib/parsers/delimiters.ml create mode 100644 lib/parsers/dune create mode 100644 lib/parsers/string_literals.ml create mode 100644 lib/rewriter/dune create mode 100644 lib/rewriter/rewrite.ml create mode 100644 lib/rewriter/rewrite.mli create mode 100644 lib/rewriter/rewrite_template.ml create mode 100644 lib/rewriter/rewrite_template.mli create mode 100644 lib/statistics/dune create mode 100644 lib/statistics/statistics.ml create mode 100644 lib/statistics/statistics.mli create mode 100644 lib/statistics/time.ml create mode 100644 src/dune create mode 100644 src/main.ml create mode 100644 src/specification.ml create mode 100644 test/dune create mode 100644 test/test_bash.ml create mode 100644 test/test_c.ml create mode 100644 test/test_c_separators.ml create mode 100644 test/test_c_style_comments.ml create mode 100644 test/test_cli.ml create mode 100644 test/test_generic.ml create mode 100644 test/test_go.ml create mode 100644 test/test_integration.ml create mode 100644 test/test_match_rule.ml create mode 100644 test/test_rewrite_parts.ml create mode 100644 test/test_rewrite_rule.ml create mode 100644 test/test_statistics.ml create mode 100644 test/test_string_literals.ml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d76c253 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.merlin +_build +*.install diff --git a/LICENSE b/LICENSE index 94a8ce3..578824a 100644 --- a/LICENSE +++ b/LICENSE @@ -41,7 +41,7 @@ form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain + of this License, Derivative Works shall not include works that refoo separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ff40819 --- /dev/null +++ b/Makefile @@ -0,0 +1,27 @@ +all: build comby + +build: + @dune build + +comby comby-server: + @ln -s _build/install/default/bin/$@ ./$@ + +install: + @dune install + +doc: + @dune build @doc + +test: + @dune runtest + +clean: + @dune clean + +uninstall: + @dune uninstall + +promote: + @dune promote + +.PHONY: all build install test clean promote diff --git a/README.md b/README.md index f2833ce..fd03830 100644 --- a/README.md +++ b/README.md @@ -1 +1,171 @@ -# comby \ No newline at end of file +# comby + +## Build + +- Install [opam](https://opam.ocaml.org/doc/Install.html) + +- Create a new switch if you don't have OCaml installed + +``` +opam init +opam switch create 4.05.0 4.05.0 +``` + +- Install dependencies + +### Linux + +``` +sudo apt-get install pkg-config pcre +``` + +### Mac + +``` +brew install pkg-config pcre +``` + +- Install opam libraries + +``` +opam install ppx_deriving_yojson +opam install core +opam install ppxlib +opam install ppx_deriving +opam install angstrom +opam install hack_parallel +opam install opium +opam install pcre +opam install oasis +``` + +- Install mparser + +``` +git clone https://github.com/comby-tools/mparser +oasis setup +ocaml setup.ml -configure --enable-pcre --enable-re +ocaml setup.ml -build +ocaml setup.ml -install +``` + +- Build and test + +``` +make +make test +``` + +## Running + +`./comby MATCH_TEMPLATE REWRITE_TEMPLATE` by default rewrites all files +in-place in the current directory and all subdirectories. + +- Adding `-f .ml` will rewrite all files ending in `.ml`. Adding `-f README.md` will rewrite all files +ending in `README.md` + +- Input can also be fed via `-stdin` instead of rewriting files: + +``` +./comby -stdin 'printf(":[1] :[2]")' 'printf("comby, :[1]")' << EOF 2> /dev/null +int main(void) { + printf("hello world"); +} +EOF +``` + +Outputs: + +``` +int main(void) { + printf("comby, hello"); +} +``` +- Adding a `-json` flag will output JSON content of the rewrite: + +``` +[ + { + "range": { + "start": { "offset": 19, "line": -1, "column": -1 }, + "end": { "offset": 41, "line": -1, "column": -1 } + }, + "replacement_content": "printf(\"comby, hello\")", + "environment": [ + [ + "1", + { + "value": "hello", + "range": { + "start": { "offset": 15, "line": -1, "column": -1 }, + "end": { "offset": 20, "line": -1, "column": -1 } + } + } + ] + ] + } +] +``` + +- Adding a `-match-only` flag will output JSON content of all matches: + +``` +[ + { + "range": { + "start": { "offset": 19, "line": 1, "column": 20 }, + "end": { "offset": 40, "line": 1, "column": 41 } + }, + "environment": [ + [ + "1", + { + "value": "hello", + "range": { + "start": { "offset": 27, "line": 1, "column": 28 }, + "end": { "offset": 32, "line": 1, "column": 33 } + } + } + ], + [ + "2", + { + "value": "world", + "range": { + "start": { "offset": 33, "line": 1, "column": 34 }, + "end": { "offset": 38, "line": 1, "column": 39 } + } + } + ] + ], + "matched": "printf(\"hello world\")" + } +] +``` + +See other flags for more options with `./comby -h`: + +``` +Run a rewrite pass. + + comby [MATCH_TEMPLATE REWRITE_TEMPLATE] + +=== flags === + + [-directory path] Run on files in a directory. Default is current + directory: /Users/rvt/comby + [-filter extensions] CSV of extensions to include + [-jobs n] Number of worker processes + [-json] Output JSON format for matches or rewrite text to stdout + [-match-only] Only perform matching (ignore rewrite templates) + [-rule rule] Apply rules to matches. Respects -f + [-sequential] Run sequentially + [-stdin] Read source from stdin + [-templates path] CSV of directories containing templates + [-timeout seconds] Set match timeout on a source. Default: 3 + [-verbose] Log to /tmp/comby.out + [-build-info] print info about this build and exit + [-version] print the version of this build and exit + [-help] print this help text and exit + (alias: -?) +``` diff --git a/comby b/comby new file mode 120000 index 0000000..fe72cd4 --- /dev/null +++ b/comby @@ -0,0 +1 @@ +_build/install/default/bin/comby \ No newline at end of file diff --git a/comby.opam b/comby.opam new file mode 100644 index 0000000..43671d1 --- /dev/null +++ b/comby.opam @@ -0,0 +1,13 @@ +opam-version: "2.0" +version: "0.1.0" +maintainer: "rvantonder@gmail.com" +authors: ["Rijnard van Tonder"] +homepage: "https://github.com/comby-tools/comby" +bug-reports: "https://github.com/comby-tools/comby/issues" +dev-repo: "git+https://github.com/comby-tools/comby.git" +license: "Apache-2.0" +build: [["jbuilder" "build" "-p" name "-j" jobs "@install"]] +depends: [ + "mparser" + "core" +] diff --git a/dune b/dune new file mode 100644 index 0000000..5dbe9e4 --- /dev/null +++ b/dune @@ -0,0 +1,3 @@ +(env + (dev + (flags (:standard -w A-3-4-32-34-39-40-41-42-44-45-48-49-50-57)))) diff --git a/dune-project b/dune-project new file mode 100644 index 0000000..fd99759 --- /dev/null +++ b/dune-project @@ -0,0 +1,2 @@ +(lang dune 1.8) +(name comby) diff --git a/lib/comby.ml b/lib/comby.ml new file mode 100644 index 0000000..b5732e0 --- /dev/null +++ b/lib/comby.ml @@ -0,0 +1,5 @@ +module Language = Language +module Matchers = Matchers +module Match = Match +module Rewriter = Rewriter +module Statistics = Statistics diff --git a/lib/comby.mli b/lib/comby.mli new file mode 100644 index 0000000..b5732e0 --- /dev/null +++ b/lib/comby.mli @@ -0,0 +1,5 @@ +module Language = Language +module Matchers = Matchers +module Match = Match +module Rewriter = Rewriter +module Statistics = Statistics diff --git a/lib/dune b/lib/dune new file mode 100644 index 0000000..2591600 --- /dev/null +++ b/lib/dune @@ -0,0 +1,14 @@ +(library + (name comby) + (public_name comby) + (preprocess (pps ppx_deriving.show ppx_deriving.eq ppx_sexp_conv)) + (libraries + ppxlib + core + mparser + mparser.pcre + comby.matchers + comby.rewriter + comby.match + comby.language + comby.statistics)) diff --git a/lib/language/ast.ml b/lib/language/ast.ml new file mode 100644 index 0000000..4ba322c --- /dev/null +++ b/lib/language/ast.ml @@ -0,0 +1,30 @@ +open Core + +type atom = + | Variable of string + | String of string +[@@deriving sexp] + +type antecedent = atom +[@@deriving sexp] + +type expression = + | True + | False + | Equal of atom * atom + | Not_equal of atom * atom + | Match of atom * (antecedent * consequent) list + | RewriteTemplate of string + | Rewrite of atom * (antecedent * consequent) list + | Rewrite_old of atom * ((atom * atom * expression list) list) +and consequent = expression list +[@@deriving sexp] + +let (=) left right = Equal (left, right) + +let (<>) left right = Not_equal (left, right) + +(* Semantics for rewrite_rule with a list more than one expression is + undefined *) +type t = expression list +[@@deriving sexp] diff --git a/lib/language/dune b/lib/language/dune new file mode 100644 index 0000000..d0bef78 --- /dev/null +++ b/lib/language/dune @@ -0,0 +1,5 @@ +(library + (name language) + (public_name comby.language) + (preprocess (pps ppx_deriving.show ppx_sexp_conv ppx_sexp_message ppx_deriving_yojson)) + (libraries comby.parsers comby.match comby.rewriter comby.matchers ppxlib core mparser mparser.pcre yojson ppx_deriving_yojson ppx_deriving_yojson.runtime)) diff --git a/lib/language/parser.ml b/lib/language/parser.ml new file mode 100644 index 0000000..1765981 --- /dev/null +++ b/lib/language/parser.ml @@ -0,0 +1,25 @@ +open Core + +open MParser +open MParser_PCRE.Tokens + +open Ast + +let variable_parser s = + (string Syntax.variable_left_delimiter + >> (many (alphanum <|> char '_') |>> String.of_char_list) + << string Syntax.variable_right_delimiter) s + +let value_parser s = + string_literal s + +let operator_parser s = + ((string Syntax.equal) + <|> (string Syntax.not_equal)) s + +let atom_parser s = + ((variable_parser >>= fun variable -> return (Variable variable)) + <|> (value_parser >>= fun value -> return (String value))) s + +let rewrite_template_parser s = + (value_parser >>= fun value -> return (RewriteTemplate value)) s diff --git a/lib/language/rule.ml b/lib/language/rule.ml new file mode 100644 index 0000000..4ecb4e9 --- /dev/null +++ b/lib/language/rule.ml @@ -0,0 +1,233 @@ +open Core +open MParser +open MParser_PCRE.Tokens + +open Match +open Matchers + +open Rewriter + +open Ast +open Parser + +type result = bool * environment option + +let sat = fst + +let result_env = snd + +let match_configuration_of_syntax template = + (* decide match configuration based on whether there are holes *) + let antecedent_contains_hole_syntax case = + String.is_substring case ~substring:Syntax.variable_left_delimiter + in + if antecedent_contains_hole_syntax template then + Configuration.create ~match_kind:Fuzzy () + else + Configuration.create ~match_kind:Exact () + +let merge_match_environments matches environment' = + List.map matches ~f:(fun { environment; _ } -> + Environment.merge environment environment') + +type rewrite_context = + { variable : string } + +let rec apply ?(matcher = (module Matchers.Generic : Matchers.Matcher)) predicates env = + let open Option in + let module Matcher = (val matcher : Matchers.Matcher) in + + let equal_in_environment var value env = + match Environment.lookup env var with + | None -> false, Some env + | Some var_value -> String.equal var_value value, Some env + in + (* accepts only one expression *) + let rec rule_match ?(rewrite_context : rewrite_context option) env = + function + | True -> true, Some env + | False -> false, Some env + | Equal (Variable var, String value) + | Equal (String value, Variable var) -> + equal_in_environment var value env + | Equal (String left, String right) -> + String.equal left right, Some env + | Equal (Variable left, Variable right) -> + let result = + Environment.lookup env left >>= fun left -> + Environment.lookup env right >>= fun right -> + return (String.equal left right) + in + Option.value result ~default:false, Some env + | Not_equal (left, right) -> + let sat, env = rule_match env (Equal (left, right)) in + not sat, env + | Match (Variable variable, cases) -> + let result = + Environment.lookup env variable >>= fun source -> + List.find_map cases ~f:(fun (template, case_expression) -> + match template with + | String template -> + begin + let configuration = match_configuration_of_syntax template in + Matcher.all ~configuration ~template ~source |> function + | [] -> None + | matches -> + (* merge environments. overwrite behavior is undefined *) + let fold_matches (sat, out) { environment; _ } = + let fold_cases (sat, out) predicate = + if sat then + let env' = Environment.merge env environment in + rule_match ?rewrite_context env' predicate + else + (sat, out) + in + List.fold case_expression ~init:(sat, out) ~f:fold_cases + in + List.fold matches ~init:(true, None) ~f:fold_matches + |> Option.some + end + | Variable _ -> + failwith "| :[hole] is invalid. Maybe you meant to put quotes") + in + Option.value_map result ~f:ident ~default:(false, Some env) + | Match (String template, cases) -> + let source, _ = Rewriter.Rewrite_template.substitute template env in + let fresh_var = Uuid.(Fn.compose to_string create ()) in + let env = Environment.add env fresh_var source in + rule_match env (Match (Variable fresh_var, cases)) + | RewriteTemplate rewrite_template -> + begin + match rewrite_context with + | None -> false, None + | Some { variable; _ } -> + (* FIXME(RVT) assumes only contextual rewrite for now. *) + let env = + Rewrite_template.substitute rewrite_template env + |> fst + |> fun replacement' -> + Environment.update env variable replacement' + |> Option.some + in + true, env + end + | Rewrite (Variable variable, cases) -> + let result = + Environment.lookup env variable >>= fun source -> + List.find_map cases ~f:(fun (template, case_expression) -> + match template with + | String template -> + begin + let configuration = match_configuration_of_syntax template in + let matches = Matcher.all ~configuration ~template ~source in + if List.is_empty matches then + None + else + let fold_cases (sat, out) predicate = + if sat then + let env = + match out with + | Some out -> Environment.merge out env + | None -> env + in + match matches with + | { environment; _ } :: _ -> + let env = Environment.merge env environment in + rule_match ~rewrite_context:{ variable } env predicate + | _ -> + sat, out + else + (sat, out) + in + List.fold case_expression ~init:(true, None) ~f:fold_cases + |> Option.some + end + | Variable _ -> + failwith "| :[hole] is invalid. Maybe you meant to put quotes") + in + Option.value_map result ~f:ident ~default:(false, Some env) + | Rewrite _ -> failwith "TODO" + | Rewrite_old _ -> failwith "Deprecated" + in + List.fold predicates ~init:(true, None) ~f:(fun (sat, out) predicate -> + if sat then + let env = + Option.value_map out + ~f:(fun out -> Environment.merge out env) + ~default:env + in + rule_match env predicate + else + (sat, out)) + + +let make_equality_expression operator left right = + if String.equal operator Syntax.equal then + return (Equal (left, right)) + else if + String.equal operator Syntax.not_equal then + return (Not_equal (left, right)) + else + let message = + Format.sprintf + "Unhandled operator %s. Did you mean %s or %s?" + operator + Syntax.equal + Syntax.not_equal in + fail message + +let create rule = + let operator_parser = + spaces >> atom_parser >>= fun left -> + spaces >> operator_parser >>= fun operator -> + spaces >> atom_parser << spaces >>= fun right -> + make_equality_expression operator left right << spaces + in + let true' = spaces >> string Syntax.true' << spaces |>> fun _ -> True in + let false' = spaces >> string Syntax.false' << spaces |>> fun _ -> False in + let rec expression_parser s = + choice + [ pattern_parser + (* string literals are ambiguous, so attempt to parse operator first *) + ; attempt operator_parser + ; rewrite_template_parser + ; true' + ; false' + ] + s + and pattern_parser s = + let case_parser : (atom * expression list, unit) parser = + spaces >> string Syntax.pipe_operator >> + spaces >> atom_parser << spaces << string Syntax.arrow << spaces >>= fun antecedent -> + spaces >> comma_sep expression_parser << spaces |>> fun consequent -> + antecedent, consequent + in + let pattern keyword = + string keyword << spaces >> atom_parser << spaces << char '{' << spaces + >>= fun atom -> + many1 case_parser + << char '}' << spaces + >>= fun cases -> return (atom, cases) + in + let match_pattern = + pattern Syntax.start_match_pattern |>> fun (atom, cases) -> + (Match (atom, cases)) + in + let rewrite_pattern = + pattern Syntax.start_rewrite_pattern |>> fun (atom, cases) -> + (Rewrite (atom, cases)) + in + choice [ match_pattern; rewrite_pattern ] + s + in + let rule_parser s = + (spaces + >> string Syntax.rule_prefix + >> spaces + >> comma_sep expression_parser + << eof) + s + in + match parse_string rule_parser rule () with + | Success rule -> Ok rule + | Failed (msg, _) -> Or_error.error_string msg diff --git a/lib/language/rule.mli b/lib/language/rule.mli new file mode 100644 index 0000000..de27632 --- /dev/null +++ b/lib/language/rule.mli @@ -0,0 +1,20 @@ +open Core + +open Matchers +open Match + +open Ast + +type result = bool * environment option + +val sat : result -> bool + +val result_env : result -> environment option + +val create : string -> expression list Or_error.t + +val apply + : ?matcher:(module Matcher) + -> t + -> environment + -> result diff --git a/lib/language/syntax.ml b/lib/language/syntax.ml new file mode 100644 index 0000000..3117c03 --- /dev/null +++ b/lib/language/syntax.ml @@ -0,0 +1,11 @@ +let rule_prefix = "where" +let start_match_pattern = "match" +let start_rewrite_pattern = "rewrite" +let equal = "==" +let not_equal = "!=" +let variable_left_delimiter = ":[" +let variable_right_delimiter = "]" +let true' = "true" +let false' = "false" +let pipe_operator = "|" +let arrow = "->" diff --git a/lib/match/dune b/lib/match/dune new file mode 100644 index 0000000..96a3e00 --- /dev/null +++ b/lib/match/dune @@ -0,0 +1,5 @@ +(library + (name match) + (public_name comby.match) + (preprocess (pps ppx_deriving.show ppx_deriving.eq ppx_sexp_conv ppx_sexp_message ppx_deriving_yojson)) + (libraries comby.parsers ppxlib core mparser mparser.pcre yojson ppx_deriving_yojson ppx_deriving_yojson.runtime)) diff --git a/lib/match/environment.ml b/lib/match/environment.ml new file mode 100644 index 0000000..b8e71d0 --- /dev/null +++ b/lib/match/environment.ml @@ -0,0 +1,74 @@ +open Core + +module Data = struct + type t = + { value : string + ; range : Range.t + } + [@@deriving yojson, eq, sexp] +end + +open Data +type data = Data.t +[@@deriving yojson, eq, sexp] + +type t = data Core.String.Map.t + +type t_alist = (string * data) list +[@@deriving yojson, eq] + +let create () : t = + String.Map.empty + +let vars (env : t) : string list = + Map.keys env + +let add ?(range = Range.default) (env : t) (var : string) (value : string) : t = + Map.add env ~key:var ~data:{ value; range } + |> function + | `Duplicate -> env + | `Ok env -> env + +let lookup (env : t) (var : string) : string option = + Map.find env var + |> Option.map ~f:(fun { value; _ } -> value) + +let lookup_range (env : t) (var : string) : Range.t option = + Map.find env var + |> Option.map ~f:(fun { range; _ } -> range) + +let fold (env : t) = + Map.fold env + +let update env var value = + Map.change env var ~f:(Option.map ~f:(fun result -> { result with value })) + +let update_range env var range = + Map.change env var ~f:(Option.map ~f:(fun result -> { result with range })) + +let to_string env = + Map.fold env ~init:"" ~f:(fun ~key:variable ~data:{ value; _ } acc -> + Format.sprintf "%s |-> %s\n%s" variable value acc) + +let furthest_match env = + Map.fold + env + ~init:0 + ~f:(fun ~key:_ ~data:{ range = { match_start = { offset; _ }; _ }; _ } max -> + Int.max offset max) + +let equal env1 env2 = + Map.equal Data.equal env1 env2 + +let merge env1 env2 = + Map.merge_skewed env1 env2 ~combine:(fun ~key:_ v1 _ -> v1) + +let copy env = + fold env ~init:(create ()) ~f:(fun ~key ~data:{ value; range } env' -> + add ~range env' key value) + +let to_yojson env = + Map.to_alist env + |> t_alist_to_yojson + +let of_yojson _ = assert false diff --git a/lib/match/environment.mli b/lib/match/environment.mli new file mode 100644 index 0000000..6a312f3 --- /dev/null +++ b/lib/match/environment.mli @@ -0,0 +1,26 @@ +type t +[@@deriving yojson] + +val create : unit -> t + +val vars : t -> string list + +val add : ?range:Range.t -> t -> string -> string -> t + +val lookup : t -> string -> string option + +val update : t -> string -> string -> t + +val lookup_range : t -> string -> Range.t option + +val update_range : t -> string -> Range.t -> t + +val furthest_match : t -> int + +val equal : t -> t -> bool + +val merge : t -> t -> t + +val copy : t -> t + +val to_string : t -> string diff --git a/lib/match/location.ml b/lib/match/location.ml new file mode 100644 index 0000000..a82d380 --- /dev/null +++ b/lib/match/location.ml @@ -0,0 +1,14 @@ +open Core + +type t = + { offset : int + ; line : int + ; column : int + } +[@@deriving yojson, eq, sexp] + +let default = + { offset = -1 + ; line = -1 + ; column = -1 + } diff --git a/lib/match/match.ml b/lib/match/match.ml new file mode 100644 index 0000000..a5a460c --- /dev/null +++ b/lib/match/match.ml @@ -0,0 +1,6 @@ +module Location = Location +module Range = Range +module Environment = Environment + +include Types +include Match_context diff --git a/lib/match/match.mli b/lib/match/match.mli new file mode 100644 index 0000000..4483a4c --- /dev/null +++ b/lib/match/match.mli @@ -0,0 +1,67 @@ +module Location : sig + type t = + { offset : int + ; line : int + ; column : int + } + [@@deriving yojson, eq, sexp] + + val default : t +end + +type location = Location.t +[@@deriving yojson, eq, sexp] + +module Range : sig + type t = + { match_start : location [@key "start"] + ; match_end : location [@key "end"] + } + [@@deriving yojson, eq, sexp] + + val default : t +end + +type range = Range.t +[@@deriving yojson, eq, sexp] + +module Environment : sig + type t + [@@deriving yojson, eq] + + val create : unit -> t + + val vars : t -> string list + + val add : ?range:range -> t -> string -> string -> t + + val lookup : t -> string -> string option + + val update : t -> string -> string -> t + + val lookup_range : t -> string -> range option + + val update_range : t -> string -> range -> t + + val furthest_match : t -> int + + val equal : t -> t -> bool + + val copy : t -> t + + val merge : t -> t -> t + + val to_string : t -> string +end + +type environment = Environment.t +[@@deriving yojson] + +type t = + { range : range + ; environment : environment + ; matched : string + } +[@@deriving yojson] + +val create : unit -> t diff --git a/lib/match/match_context.ml b/lib/match/match_context.ml new file mode 100644 index 0000000..7cd10b6 --- /dev/null +++ b/lib/match/match_context.ml @@ -0,0 +1,12 @@ +type t = + { range : Range.t + ; environment : Environment.t + ; matched : string + } +[@@deriving yojson] + +let create () = + { range = Range.default + ; environment = Environment.create () + ; matched = "" + } diff --git a/lib/match/range.ml b/lib/match/range.ml new file mode 100644 index 0000000..9cf945e --- /dev/null +++ b/lib/match/range.ml @@ -0,0 +1,10 @@ +type t = + { match_start : Location.t [@key "start"] + ; match_end : Location.t [@key "end"] + } +[@@deriving yojson, eq, sexp] + +let default = + { match_start = Location.default + ; match_end = Location.default + } diff --git a/lib/match/types.ml b/lib/match/types.ml new file mode 100644 index 0000000..4081413 --- /dev/null +++ b/lib/match/types.ml @@ -0,0 +1,8 @@ +type location = Location.t +[@@deriving yojson, eq, sexp] + +type range = Range.t +[@@deriving yojson, eq, sexp] + +type environment = Environment.t +[@@deriving yojson] diff --git a/lib/matchers/bash.ml b/lib/matchers/bash.ml new file mode 100644 index 0000000..7ccdbef --- /dev/null +++ b/lib/matchers/bash.ml @@ -0,0 +1,28 @@ +open MParser + +module Syntax = struct + include Generic.Syntax + + let user_defined_delimiters = + [ ("if", "fi") + ; ("case", "esac") + ] + @ Generic.Syntax.user_defined_delimiters + + let escapable_string_literals = + [ {|"|} + ; {|'|} + ] + + let escape_char = + '\\' + + let raw_string_literals = + [] + + let comment_parser s = + (Parsers.Comments.c_multiline + <|> Parsers.Comments.c_newline) s +end + +include Matcher.Make(Syntax) diff --git a/lib/matchers/c.ml b/lib/matchers/c.ml new file mode 100644 index 0000000..43b4e3b --- /dev/null +++ b/lib/matchers/c.ml @@ -0,0 +1,22 @@ +open MParser + +module Syntax = struct + include Generic.Syntax + + let escapable_string_literals = + [ {|"|} + ; {|'|} + ] + + let escape_char = + '\\' + + let raw_string_literals = + [] + + let comment_parser s = + (Parsers.Comments.c_multiline + <|> Parsers.Comments.c_newline) s +end + +include Matcher.Make(Syntax) diff --git a/lib/matchers/c.mli b/lib/matchers/c.mli new file mode 100644 index 0000000..80e9438 --- /dev/null +++ b/lib/matchers/c.mli @@ -0,0 +1,4 @@ +open Types + +module Syntax : Syntax.S +include Matcher.S diff --git a/lib/matchers/configuration.ml b/lib/matchers/configuration.ml new file mode 100644 index 0000000..4a0f7b0 --- /dev/null +++ b/lib/matchers/configuration.ml @@ -0,0 +1,13 @@ +type match_kind = + | Exact + | Fuzzy + +type t = + { match_kind : match_kind + ; significant_whitespace: bool + } + +let create ?(match_kind = Fuzzy) ?(significant_whitespace = false) () = + { match_kind + ; significant_whitespace + } diff --git a/lib/matchers/dune b/lib/matchers/dune new file mode 100644 index 0000000..c4f388a --- /dev/null +++ b/lib/matchers/dune @@ -0,0 +1,5 @@ +(library + (name matchers) + (public_name comby.matchers) + (preprocess (pps ppx_deriving.show ppx_sexp_conv ppx_sexp_message)) + (libraries comby.parsers comby.match ppxlib core mparser mparser.pcre)) diff --git a/lib/matchers/generic.ml b/lib/matchers/generic.ml new file mode 100644 index 0000000..ebe51ff --- /dev/null +++ b/lib/matchers/generic.ml @@ -0,0 +1,20 @@ +module Syntax = struct + (** these are nestable. strings, on the other hand, are not + nestable without escapes *) + let user_defined_delimiters = + [ ("(", ")") + ; ("{", "}") + ; ("[", "]") + ] + + let escapable_string_literals = [] + + let escape_char = + '\\' + + let raw_string_literals = [] + + let comment_parser = MParser.zero +end + +include Matcher.Make(Syntax) diff --git a/lib/matchers/generic.mli b/lib/matchers/generic.mli new file mode 100644 index 0000000..80e9438 --- /dev/null +++ b/lib/matchers/generic.mli @@ -0,0 +1,4 @@ +open Types + +module Syntax : Syntax.S +include Matcher.S diff --git a/lib/matchers/go.ml b/lib/matchers/go.ml new file mode 100644 index 0000000..cf9a13c --- /dev/null +++ b/lib/matchers/go.ml @@ -0,0 +1,10 @@ +module Syntax = struct + include C.Syntax + + let raw_string_literals = + [ ({|`|}, {|`|}) + ] + +end + +include Matcher.Make(Syntax) diff --git a/lib/matchers/go.mli b/lib/matchers/go.mli new file mode 100644 index 0000000..80e9438 --- /dev/null +++ b/lib/matchers/go.mli @@ -0,0 +1,4 @@ +open Types + +module Syntax : Syntax.S +include Matcher.S diff --git a/lib/matchers/html.ml b/lib/matchers/html.ml new file mode 100644 index 0000000..d1c1da9 --- /dev/null +++ b/lib/matchers/html.ml @@ -0,0 +1,17 @@ +module Syntax = struct + include Generic.Syntax + + let user_defined_delimiters = + Generic.Syntax.user_defined_delimiters @ + [ ("<", ">") + ] + + + let escapable_string_literals = + [ {|"|} + ; {|'|} + ] + +end + +include Matcher.Make(Syntax) diff --git a/lib/matchers/matcher.ml b/lib/matchers/matcher.ml new file mode 100644 index 0000000..8250582 --- /dev/null +++ b/lib/matchers/matcher.ml @@ -0,0 +1,401 @@ +open Core +open MParser + +open Configuration +open Match +open Range +open Location +open Types + +let configuration_ref = ref (Configuration.create ()) + +let debug = false + +let f _ = return Unit + +let extract_matched_text source { offset = match_start; _ } { offset = match_end; _ } = + String.slice source match_start match_end + +let is_not p s = + if is_ok (p s) then + Empty_failed (unknown_error s) + else + match read_char s with + | Some c -> Consumed_ok (c, advance_state s 1, No_error) + | None -> Empty_failed (unknown_error s) + +type 'a literal_parser_callback = contents:string -> left_delimiter:string -> right_delimiter:string -> 'a +type 'a nested_delimiter_callback = left_delimiter:string -> right_delimiter:string -> 'a + +module Make (Syntax : Syntax.S) = struct + let escapable_string_literal_parser (f : 'a literal_parser_callback) = + List.map Syntax.escapable_string_literals ~f:(fun delimiter -> + let module M = + Parsers.String_literals.Escapable.Make(struct + let delimiter = delimiter + let escape = Syntax.escape_char + end) + in + M.base_string_literal >>= fun contents -> + return (f ~contents ~left_delimiter:delimiter ~right_delimiter:delimiter)) + |> choice + + + let raw_string_literal_parser (f : 'a literal_parser_callback) = + List.map Syntax.raw_string_literals ~f:(fun (left_delimiter, right_delimiter) -> + let module M = + Parsers.String_literals.Raw.Make(struct + let left_delimiter = left_delimiter + let right_delimiter = right_delimiter + end) + in + M.base_string_literal >>= fun contents -> + return (f ~contents ~left_delimiter ~right_delimiter)) + |> choice + + let escapable_literal_grammar ~right_delimiter = + (attempt + (char Syntax.escape_char + >> string right_delimiter + >>= fun s -> return (Format.sprintf "%c%s" Syntax.escape_char s)) + ) + <|> + (attempt + (char Syntax.escape_char + >> char Syntax.escape_char + >> return (Format.sprintf "%c%c" Syntax.escape_char Syntax.escape_char)) + ) + <|> (is_not (string right_delimiter) |>> String.of_char) + + let raw_literal_grammar ~right_delimiter = + is_not (string right_delimiter) |>> String.of_char + + (** a parser that understands the single hole matching is alphanum and _ *) + let generate_single_hole_parser () = + (alphanum <|> char '_') |>> String.of_char + + let generate_spaces_parser () = + (* at least a space followed by comments and spaces *) + (spaces1 + >> many Syntax.comment_parser << spaces + >>= fun result -> f result) + <|> + (* This case not covered by tests, may not be needed *) + (many1 Syntax.comment_parser << spaces >>= fun result -> f result) + + let sequence_chain (plist : ('c, Match.t) parser sexp_list) : ('c, Match.t) parser = + List.fold plist ~init:(return Unit) ~f:(>>) + + let nested_delimiters_parser (f : 'a nested_delimiter_callback) = + Syntax.user_defined_delimiters + |> List.map ~f:(fun (left_delimiter, right_delimiter) -> + Parsers.Delimiters.between + (f ~left_delimiter ~right_delimiter) + left_delimiter right_delimiter) + |> choice + + (** All code can have comments interpolated *) + let generate_string_token_parser str : ('c, _) parser = + many Syntax.comment_parser + >> string str + >> many Syntax.comment_parser + >>= fun result -> f result + + let reserved_delimiters = + List.concat_map Syntax.user_defined_delimiters ~f:(fun (from, until) -> [from; until]) + |> List.append [":["; "]"] (* lazy hole *) + |> List.append [":[["; "]]"] (* single token hole *) + |> List.map ~f:string + |> choice + + let reserved = + reserved_delimiters + <|> (space |>> Char.to_string) + + let greedy_hole_parser _s = + string ":[" >> (many (alphanum <|> char '_') |>> String.of_char_list) << string "]" + + let single_hole_parser _s = + string ":[[" >> (many (alphanum <|> char '_') |>> String.of_char_list) << string "]]" + + let until_of_from from = + Syntax.user_defined_delimiters + |> List.find_map ~f:(fun (from', until) -> if from = from' then Some until else None) + |> function + | Some until -> until + | None -> assert false + + let record_matches identifier p : ('c, Match.t) parser = + get_pos >>= fun (pre_index, pre_line, pre_column) -> + p >>= fun matched -> + get_pos >>= fun (post_index, post_line, post_column) -> + update_user_state + (fun ({ Match.environment; _ } as result) -> + if debug then begin + Format.printf "Updating user state:@."; + Format.printf "%s |-> %s@." identifier (String.concat matched); + Format.printf "ID %s: %d:%d:%d - %d:%d:%d@." + identifier + pre_index pre_line pre_column + post_index post_line post_column; + end; + let pre_location : Location.t = + Location. + { offset = pre_index + ; line = pre_line + ; column = pre_column + } + in + let post_location : Location.t = + Location. + { offset = post_index + ; line = post_line + ; column = post_column + } + in + let range = { match_start = pre_location; match_end = post_location } in + let environment = Environment.add ~range environment identifier (String.concat matched) in + { result with environment }) + >>= fun () -> f matched + + let generate_hole_parser ?priority_left_delimiter:left_delimiter ?priority_right_delimiter:right_delimiter = + let between_nested_delims p from = + let until = until_of_from from in + between (string from) (string until) p + |>> fun result -> (String.concat @@ [from] @ result @ [until]) + in + let between_nested_delims p = + (match left_delimiter, right_delimiter with + | Some left_delimiter, Some right_delimiter -> [ (left_delimiter, right_delimiter) ] + | _ -> Syntax.user_defined_delimiters) + |> List.map ~f:fst + |> List.map ~f:(between_nested_delims p) + |> choice + in + (* applies looser delimiter constraints for matching *) + let reserved = + (match left_delimiter, right_delimiter with + | Some left_delimiter, Some right_delimiter -> [ (left_delimiter, right_delimiter) ] + | _ -> Syntax.user_defined_delimiters) + |> List.concat_map ~f:(fun (from, until) -> [from; until]) + |> List.map ~f:string + |> choice + in + (* a parser that understands the hole matching cut off points happen at + delimiters *) + let rec nested_grammar s = + (Syntax.comment_parser + <|> escapable_string_literal_parser (fun ~contents ~left_delimiter:_ ~right_delimiter:_ -> contents) + <|> raw_string_literal_parser (fun ~contents ~left_delimiter:_ ~right_delimiter:_ -> contents) + <|> delimsx + <|> (is_not reserved |>> String.of_char)) + s + and delimsx s = (between_nested_delims (many nested_grammar)) s + in + nested_grammar + + let turn_holes_into_matchers_for_this_level ?left_delimiter ?right_delimiter p_list = + List.fold_right p_list ~init:[] ~f:(fun p acc -> + let process_hole = + match parse_string p "_signal_hole" (Match.create ()) with + | Failed _ -> p + | Success result -> + match result with + | Hole (Lazy (identifier, dimension)) -> + let matcher = + match dimension with + | Code -> + generate_hole_parser + ?priority_left_delimiter:left_delimiter + ?priority_right_delimiter:right_delimiter + | Escapable_string_literal -> + let right_delimiter = Option.value_exn right_delimiter in + escapable_literal_grammar ~right_delimiter + | Raw_string_literal -> + let right_delimiter = Option.value_exn right_delimiter in + raw_literal_grammar ~right_delimiter + | Comment -> failwith "Unimplemented" + in + let rest = + match acc with + | [] -> eof >>= fun () -> f [""] + | _ -> sequence_chain acc + in + (* continue until rest, but don't consume rest. *) + let hole_semantics = many (not_followed_by rest "" >> matcher) in + record_matches identifier hole_semantics + + | Hole (Single (identifier, _)) -> + let hole_semantics = many1 (generate_single_hole_parser ()) in + record_matches identifier hole_semantics + + | _ -> failwith "Hole expected" + in + process_hole::acc) + + let hole_parser sort dimension = + let skip_signal result = + skip (string "_signal_hole") |>> fun () -> result + in + match sort with + | `Single -> + single_hole_parser () |>> fun id -> + skip_signal (Hole (Single (id, dimension))) + | `Lazy -> + greedy_hole_parser () |>> fun id -> + skip_signal (Hole (Lazy (id, dimension))) + + let generate_hole_for_literal sort ~contents ~left_delimiter ~right_delimiter s = + let p = + many (hole_parser `Single sort + <|> (hole_parser `Lazy sort) + <|> ((many1 (is_not (string ":[" <|> string ":[[")) + |>> String.of_char_list) |>> generate_string_token_parser)) + in + match parse_string p contents "" with + | Success p -> + begin + match sort with + | Escapable_string_literal + | Raw_string_literal -> + (turn_holes_into_matchers_for_this_level ~left_delimiter ~right_delimiter p + |> sequence_chain) s + | _ -> assert false + end + | Failed (_msg, _) -> + failwith "literal parser did not succeed" + + let rec generate_parsers s = + many (common s) + + and common _s = + (hole_parser `Single Code) + <|> (hole_parser `Lazy Code) + (* string literals are handled specially because match semantics change inside string delimiters *) + <|> (escapable_string_literal_parser (generate_hole_for_literal Escapable_string_literal)) + <|> (raw_string_literal_parser (generate_hole_for_literal Raw_string_literal)) + (* whitespace is handled specially because we may change whether they are significant for matching *) + <|> (spaces1 |>> generate_spaces_parser) + (* nested delimiters are handled specially for nestedness *) + <|> (nested_delimiters_parser generate_outer_delimiter_parsers) + (* everything else *) + <|> ((many1 (is_not reserved) |>> String.of_char_list) |>> generate_string_token_parser) + + and generate_outer_delimiter_parsers ~left_delimiter ~right_delimiter s = + (generate_parsers s >>= fun p_list -> + (turn_holes_into_matchers_for_this_level ~left_delimiter ~right_delimiter + ([ string left_delimiter + >>= fun _ -> f [left_delimiter]] + @ p_list + @ [ string right_delimiter + >>= fun _ -> f [right_delimiter]]) + |> sequence_chain) + |> return + ) s + + let general_parser_generator s = + let outer_p = + generate_parsers s >>= fun p_list -> + (* eof of template is here *) + eof >> (* result is unit so ignore *) + (* customize the inner parser *) + let inner_p = + let matcher : ('a, Match.t) parser = + turn_holes_into_matchers_for_this_level p_list + |> sequence_chain + in + let matcher : ('a, Match.t) parser = + let with_positions (matcher : ('a, Match.t) parser) : ('a, Match.t) parser = + get_pos >>= fun (pre_offset, pre_line, pre_column) -> + matcher >>= fun _last_production -> + get_pos >>= fun (post_offset, post_line, post_column) -> + let match_start = + { offset = pre_offset + ; line = pre_line + ; column = pre_column + } in + let match_end = + { offset = post_offset + ; line = post_line + ; column = post_column + } + in + let range = { match_start; match_end } in + update_user_state (fun result -> { result with range }) + >> return Unit + in + with_positions matcher + in + match !configuration_ref.match_kind with + | Exact -> matcher << eof + | Fuzzy -> + many + (not_followed_by matcher "" >> + ( + (* respect grammar but ignore contents up to a match *) + skip Syntax.comment_parser + <|> skip (escapable_string_literal_parser (fun ~contents:_ ~left_delimiter:_ ~right_delimiter:_ -> ())) + <|> skip (raw_string_literal_parser (fun ~contents:_ ~left_delimiter:_ ~right_delimiter:_ -> ())) + <|> skip any_char) + ) + >> matcher + in + return inner_p + in + outer_p s + + let to_template template : ('a, Match.t) MParser.t Or_error.t = + match parse_string general_parser_generator template 0 with + | Success p -> Ok p + | Failed (msg, _) -> Or_error.error_string msg + + (** shift: start the scan in the source at an offset *) + let first' shift p source : Match.t Or_error.t = + let set_start_pos p = fun s -> p (advance_state s shift) in + let p = set_start_pos p in + match parse_string' p source (Match.create ()) with + | Success (_, result) -> Ok result + | Failed (msg, _) -> Or_error.error_string msg + + let first ?configuration ?shift template source = + let open Or_error in + configuration_ref := Option.value configuration ~default:!configuration_ref; + to_template template >>= fun p -> + let shift = + match shift with + | Some s -> s + | None -> 0 + in + first' shift p source + + let all ?configuration ~template ~source:original_source : Match.t list = + let open Or_error in + configuration_ref := Option.value configuration ~default:!configuration_ref; + let make_result = function + | Ok ok -> ok + | Error _ -> [] + in + make_result @@ begin + to_template template >>= fun p -> + if original_source = "" || template = "" then + return [] + else + let rec aux acc shift = + match first' shift p original_source with + | Ok ({range = { match_start; match_end; _ }; _} as result) -> + let shift = match_end.offset in + let matched = extract_matched_text original_source match_start match_end in + let result = { result with matched } in + if shift >= String.length original_source then + result :: acc + else + aux (result :: acc) shift + | Error _ -> acc + in + let matches = aux [] 0 |> List.rev in + (* TODO(RVT): reintroduce nested matches *) + let compute_nested_matches matches = matches in + let matches = compute_nested_matches matches in + return matches + end +end diff --git a/lib/matchers/matcher.mli b/lib/matchers/matcher.mli new file mode 100644 index 0000000..6de850f --- /dev/null +++ b/lib/matchers/matcher.mli @@ -0,0 +1,3 @@ +open Types + +module Make (Syntax: Syntax.S) : Matcher.S diff --git a/lib/matchers/matchers.ml b/lib/matchers/matchers.ml new file mode 100644 index 0000000..b735a3d --- /dev/null +++ b/lib/matchers/matchers.ml @@ -0,0 +1,10 @@ +module Generic = Generic +module C = C +module Go = Go +module Python = Python +module Bash = Bash +module Html = Html + +module Configuration = Configuration + +module type Matcher = Types.Matcher.S diff --git a/lib/matchers/matchers.mli b/lib/matchers/matchers.mli new file mode 100644 index 0000000..b735a3d --- /dev/null +++ b/lib/matchers/matchers.mli @@ -0,0 +1,10 @@ +module Generic = Generic +module C = C +module Go = Go +module Python = Python +module Bash = Bash +module Html = Html + +module Configuration = Configuration + +module type Matcher = Types.Matcher.S diff --git a/lib/matchers/python.ml b/lib/matchers/python.ml new file mode 100644 index 0000000..4dd4518 --- /dev/null +++ b/lib/matchers/python.ml @@ -0,0 +1,20 @@ +module Python = struct + include Generic.Syntax + + let escapable_string_literals = + [ {|"|} + ; {|'|} + ] + + let escape_char = + '\\' + + let raw_string_literals = + [ ({|"""|}, {|"""|}) + ] + + let comment_parser s = + Parsers.Comments.python_newline s +end + +include Matcher.Make(Python) diff --git a/lib/matchers/types.ml b/lib/matchers/types.ml new file mode 100644 index 0000000..80662bb --- /dev/null +++ b/lib/matchers/types.ml @@ -0,0 +1,44 @@ +open Core + +module Syntax = struct + module type S = sig + val user_defined_delimiters : (string * string) list + val escapable_string_literals : string list + val escape_char : char + val raw_string_literals : (string * string) list + val comment_parser : (string, _) MParser.t + end +end + +type dimension = + | Code + | Escapable_string_literal + | Raw_string_literal + | Comment + +type hole = + | Lazy of (string * dimension) + | Single of (string * dimension) + +type production = + | Unit + | String of string + | Hole of hole + | Match of (int * string * string) + +module Matcher = struct + module type S = sig + val first + : ?configuration:Configuration.t + -> ?shift:int + -> string + -> string + -> Match.t Or_error.t + + val all + : ?configuration:Configuration.t + -> template:string + -> source:string + -> Match.t list + end +end diff --git a/lib/parsers/comments.ml b/lib/parsers/comments.ml new file mode 100644 index 0000000..90c4931 --- /dev/null +++ b/lib/parsers/comments.ml @@ -0,0 +1,69 @@ +open Core +open MParser + +let to_string from until between : string = + from ^ (String.of_char_list between) ^ until + +let anything_including_newlines ~until = + (many + (not_followed_by (string until) "" + >>= fun () -> any_char_or_nl)) + +let anything_excluding_newlines ~until = + (many + (not_followed_by (string until) "" + >>= fun () -> any_char)) + +(** a parser for comments with delimiters [from] and [until] that do not nest *) +let non_nested_comment_delimiters from until s = + (between + (string from) + (string until) + (anything_including_newlines ~until) + |>> to_string from until + ) s + +(** a parser for /* ... */ style block comments. *) +let c_multiline s = + non_nested_comment_delimiters "/*" "*/" s + +let c_newline s = + (string "//" >> anything_excluding_newlines ~until:"\n" + |>> fun l -> "//"^(String.of_char_list l)) s + +let python_newline s = + (string "#" >> anything_excluding_newlines ~until:"\n" + |>> fun l -> ("#"^String.of_char_list l)) s + +let any_newline comment_string s = + (string comment_string >> anything_excluding_newlines ~until:"\n" |>> fun l -> (comment_string^String.of_char_list l)) s + +let is_not p s = + if is_ok (p s) then + Empty_failed (unknown_error s) + else + match read_char s with + | Some c -> + Consumed_ok (c, advance_state s 1, No_error) + | None -> + Empty_failed (unknown_error s) + +(** A nested comment parser *) +let skip_nested_comments_inner from until s = + let reserved = skip ((string from) <|> (string until)) in + let rec grammar s = + ((comment_delimiters >>= fun string -> return string) + <|> + (is_not reserved >>= fun c -> return (Char.to_string c))) + s + + and comment_delimiters s = + (between + (string from) + (string until) + ((many grammar) >>= fun result -> + return (String.concat result))) + s + in + (comment_delimiters >>= fun _ -> + return ()) s diff --git a/lib/parsers/comments.mli b/lib/parsers/comments.mli new file mode 100644 index 0000000..62ea5dd --- /dev/null +++ b/lib/parsers/comments.mli @@ -0,0 +1,11 @@ +(** C-style /* */ block comment parser *) +val c_multiline : (string, _) MParser.t + +(** C++-style // line comment parser *) +val c_newline : (string, _) MParser.t + +(** Python-style # line comment parser *) +val python_newline : (string, _) MParser.t + +(** Anything until newline *) +val any_newline : string -> (string, _) MParser.t diff --git a/lib/parsers/delimiters.ml b/lib/parsers/delimiters.ml new file mode 100644 index 0000000..edde7b2 --- /dev/null +++ b/lib/parsers/delimiters.ml @@ -0,0 +1,18 @@ +open MParser + +(** Significant, potentially nested delimiters. *) + +let parens' p = + char '(' >> p << char ')' + +let braces' p = + char '{' >> p << char '}' + +let brackets' p = + char '<' >> p << char '>' + +let squares' p = + char '[' >> p << char ']' + +let between p from until = + string from >> p << string until diff --git a/lib/parsers/dune b/lib/parsers/dune new file mode 100644 index 0000000..4e55e15 --- /dev/null +++ b/lib/parsers/dune @@ -0,0 +1,5 @@ +(library + (name parsers) + (public_name comby.parsers) + (preprocess (pps ppx_deriving.show ppx_sexp_conv)) + (libraries ppxlib core mparser mparser.pcre)) diff --git a/lib/parsers/string_literals.ml b/lib/parsers/string_literals.ml new file mode 100644 index 0000000..da07e3d --- /dev/null +++ b/lib/parsers/string_literals.ml @@ -0,0 +1,57 @@ +open Core +open MParser + +(** Assumes the left and right delimiter are the same, and that these can be + escaped. Does not parse a string body containing newlines (as usual when + escaping with \n) *) +module Escapable = struct + module type S = sig + val delimiter : string + val escape : char + end + + module Make (M : S) = struct + (* delimiters can be escaped and parsing continues within the string body *) + let escaped_char_s s = + any_char s + + let char_token_s s = + ((char M.escape >> escaped_char_s >>= fun c -> return (Format.sprintf {|%c%c|} M.escape c)) + <|> (any_char |>> String.of_char) + ) + s + + let base_string_literal s = + ((string M.delimiter >> (many_until char_token_s (string M.delimiter)) + |>> String.concat) + >>= fun result -> + return (Format.sprintf {|%s%s%s|} M.delimiter result M.delimiter) + ) + s + end +end + +(** Quoted or raw strings. Allows different left and right delimiters, and + disallows any sort of escaping. Does not support raw strings with identifiers + yet, e.g., {blah||blah} (OCaml) or delim``delim + syntax (Go) *) +module Raw = struct + module type S = sig + val left_delimiter : string + val right_delimiter : string + end + + module Make (M : S) = struct + let char_token_s s = + (any_char_or_nl |>> String.of_char) s + + let base_string_literal s = + (( + string M.left_delimiter >> (many_until char_token_s (string M.right_delimiter)) + |>> String.concat "raw string literal body") + >>= fun result -> + return (Format.sprintf {|%s%s%s|} M.left_delimiter result M.right_delimiter) + ) + s + end +end diff --git a/lib/rewriter/dune b/lib/rewriter/dune new file mode 100644 index 0000000..a45a7c2 --- /dev/null +++ b/lib/rewriter/dune @@ -0,0 +1,5 @@ +(library + (name rewriter) + (public_name comby.rewriter) + (preprocess (pps ppx_deriving.show ppx_sexp_conv ppx_deriving_yojson)) + (libraries comby.matchers ppxlib core)) diff --git a/lib/rewriter/rewrite.ml b/lib/rewriter/rewrite.ml new file mode 100644 index 0000000..9eee55f --- /dev/null +++ b/lib/rewriter/rewrite.ml @@ -0,0 +1,112 @@ +open Core + +open Match + +type match_context_replacement = + { range : range + ; replacement_content : string + ; environment : environment + } +[@@deriving yojson] + +type result = + { rewritten_source : string + ; contextual_substitutions : match_context_replacement list + } +[@@deriving yojson] + +let empty_result = + { rewritten_source = "" + ; contextual_substitutions = [] + } +[@@deriving yojson] + +let substitute_match_contexts (matches: Match.t list) source replacements = + let rewrite_template, environment = + List.fold2_exn + matches replacements + ~init:(source, Environment.create ()) + ~f:(fun + (rewrite_template, accumulator_environment) + ({ environment = _match_environment; _ } as match_) + { replacement_content; _ } -> + (* create a hole in the rewrite template based on this match context *) + let hole_id, rewrite_template = Rewrite_template.of_match_context match_ ~source:rewrite_template in + (* add this match context replacement to the environment *) + let accumulator_environment = Environment.add accumulator_environment hole_id replacement_content in + (* update match context replacements offset *) + rewrite_template, accumulator_environment) + in + let rewritten_source = Rewrite_template.substitute rewrite_template environment |> fst in + let offsets = Rewrite_template.get_offsets_for_holes rewrite_template (Environment.vars environment) in + let offsets = Rewrite_template.get_offsets_after_substitution offsets environment in + let contextual_substitutions = + List.map2_exn replacements offsets ~f:(fun replacement (_uid, offset) -> + let match_start = { Location.default with offset } in + let offset = offset + String.length replacement.replacement_content in + let match_end = { Location.default with offset } in + let range = Range.{ match_start; match_end } in + { replacement with range }) + in + { rewritten_source + ; contextual_substitutions + } + + (* + store range information for this match_context replacement: + (a) its offset in the original source + (b) its replacement context (to calculate the range) + (c) an environment of values that are updated to reflect their relative offset in the rewrite template + *) +let substitute_in_rewrite_template rewrite_template ({ environment; _ } : Match.t) = + let replacement_content, vars_substituted_for = Rewrite_template.substitute rewrite_template environment in + let offsets = Rewrite_template.get_offsets_for_holes rewrite_template (Environment.vars environment) in + let offsets = Rewrite_template.get_offsets_after_substitution offsets environment in + let environment = + List.fold offsets ~init:(Environment.create ()) ~f:(fun acc (var, relative_offset) -> + if List.mem vars_substituted_for var ~equal:String.equal then + let value = Option.value_exn (Environment.lookup environment var) in + (* FIXME(RVT): Location does not update row/column here *) + let start_location = + Location.{ default with offset = relative_offset } + in + let end_location = + let offset = relative_offset + String.length value in + Location.{ default with offset } + in + let range = + Range. + { match_start = start_location + ; match_end = end_location + } + in + Environment.add ~range acc var value + else + acc) + in + { replacement_content + ; environment + ; range = + { match_start = { Location.default with offset = 0 } + ; match_end = Location.default + } + } + +let all ?source ~rewrite_template matches : result option = + if matches = [] then None else + match source with + (* in-place substitution *) + | Some source -> + let matches : Match.t list = List.rev matches in + matches + |> List.map ~f:(substitute_in_rewrite_template rewrite_template) + |> substitute_match_contexts matches source + |> Option.some + (* no-inplace substitution, emit result separated by newlines *) + | None -> + matches + |> List.map ~f:(substitute_in_rewrite_template rewrite_template) + |> List.map ~f:(fun { replacement_content; _ } -> replacement_content) + |> String.concat ~sep:"\n" + |> (fun rewritten_source -> { rewritten_source; contextual_substitutions = [] }) + |> Option.some diff --git a/lib/rewriter/rewrite.mli b/lib/rewriter/rewrite.mli new file mode 100644 index 0000000..bf7f0cd --- /dev/null +++ b/lib/rewriter/rewrite.mli @@ -0,0 +1,22 @@ +open Match + +type match_context_replacement = + { range : range + ; replacement_content : string + ; environment : environment + } +[@@deriving yojson] + +type result = + { rewritten_source : string + ; contextual_substitutions : match_context_replacement list + } +[@@deriving yojson] + +(** if [source] is given, substitute in-place. If not, + emit result separated by newlines *) +val all + : ?source:string + -> rewrite_template:string + -> Match.t list + -> result option diff --git a/lib/rewriter/rewrite_template.ml b/lib/rewriter/rewrite_template.ml new file mode 100644 index 0000000..1a708e8 --- /dev/null +++ b/lib/rewriter/rewrite_template.ml @@ -0,0 +1,63 @@ +open Core + +open Match + +let substitute template env = + Environment.vars env + |> List.fold ~init:(template, []) ~f:(fun (acc, vars) variable -> + match Environment.lookup env variable with + | Some value -> + if Option.is_some (String.substr_index template ~pattern:(":["^variable^"]")) then + (String.substr_replace_all acc ~pattern:(":["^variable^"]") ~with_:value, variable::vars) + else + acc, vars + | None -> acc, vars) + +let of_match_context + { range = + { match_start = { offset = start_index; _ } + ; match_end = { offset = end_index; _ } } + ; _ + } + ~source = + let before_part = + if start_index = 0 then + "" + else + String.slice source 0 start_index + in + let after_part = String.slice source end_index (String.length source) in + let hole_id = Uuid.(Fn.compose to_string create ()) in + let rewrite_template = String.concat [before_part; ":["; hole_id; "]"; after_part] in + hole_id, rewrite_template + +(* return the offset for holes (specified by variables) in a given match template *) +let get_offsets_for_holes rewrite_template variables = + let sorted_variables = + List.fold variables ~init:[] ~f:(fun acc variable -> + match String.substr_index rewrite_template ~pattern:(":["^variable^"]") with + | Some index -> + (variable, index)::acc + | None -> acc) + |> List.sort ~compare:(fun (_, i1) (_, i2) -> i1 - i2) + |> List.map ~f:fst + in + List.fold sorted_variables ~init:(rewrite_template, []) ~f:(fun (rewrite_template, acc) variable -> + match String.substr_index rewrite_template ~pattern:(":["^variable^"]") with + | Some index -> + let rewrite_template = + String.substr_replace_all rewrite_template ~pattern:(":["^variable^"]") ~with_:"" in + rewrite_template, (variable, index)::acc + | None -> rewrite_template, acc) + |> snd + +(* pretend we substituted vars in offsets with environment. return what the offsets are after *) +let get_offsets_after_substitution offsets environment = + List.fold_right offsets ~init:([],0) ~f:(fun (var, offset) (acc, shift) -> + match Environment.lookup environment var with + | None -> failwith "Expected var" + | Some s -> + let offset' = offset + shift in + let shift = shift + String.length s in + ((var, offset')::acc), shift) + |> fst diff --git a/lib/rewriter/rewrite_template.mli b/lib/rewriter/rewrite_template.mli new file mode 100644 index 0000000..d482a74 --- /dev/null +++ b/lib/rewriter/rewrite_template.mli @@ -0,0 +1,10 @@ +open Match + +(** substitute returns the result and variables substituted for *) +val substitute : string -> Environment.t -> (string * string list) + +val of_match_context : Match.t -> source:string -> (string * string) + +val get_offsets_for_holes : string -> string list -> (string * int) list + +val get_offsets_after_substitution : (string * int) list -> Environment.t -> (string * int) list diff --git a/lib/statistics/dune b/lib/statistics/dune new file mode 100644 index 0000000..fcfbcf8 --- /dev/null +++ b/lib/statistics/dune @@ -0,0 +1,5 @@ +(library + (name statistics) + (public_name comby.statistics) + (preprocess (pps ppx_deriving.show ppx_sexp_conv ppx_sexp_message ppx_deriving_yojson)) + (libraries ppxlib core yojson ppx_deriving_yojson ppx_deriving_yojson.runtime)) diff --git a/lib/statistics/statistics.ml b/lib/statistics/statistics.ml new file mode 100644 index 0000000..0431ac4 --- /dev/null +++ b/lib/statistics/statistics.ml @@ -0,0 +1,34 @@ +module Time = Time +module Timer = Timer + +type t = + { number_of_files : int + ; lines_of_code : int + ; number_of_matches : int + ; total_time : float + } +[@@deriving yojson] + +let empty = + { number_of_files = 0 + ; lines_of_code = 0 + ; number_of_matches = 0 + ; total_time = 0.0 + } + +let merge + { number_of_files + ; lines_of_code + ; number_of_matches + ; total_time + } + { number_of_files = number_of_files' + ; lines_of_code = lines_of_code' + ; number_of_matches = number_of_matches' + ; total_time = total_time' + } = + { number_of_files = number_of_files + number_of_files' + ; lines_of_code = lines_of_code + lines_of_code' + ; number_of_matches = number_of_matches + number_of_matches' + ; total_time = total_time +. total_time' + } diff --git a/lib/statistics/statistics.mli b/lib/statistics/statistics.mli new file mode 100644 index 0000000..3f4ffa3 --- /dev/null +++ b/lib/statistics/statistics.mli @@ -0,0 +1,14 @@ +module Time = Time +module Timer = Timer + +type t = + { number_of_files : int + ; lines_of_code : int + ; number_of_matches : int + ; total_time : float + } +[@@deriving yojson] + +val empty : t + +val merge : t -> t -> t diff --git a/lib/statistics/time.ml b/lib/statistics/time.ml new file mode 100644 index 0000000..35fdcc9 --- /dev/null +++ b/lib/statistics/time.ml @@ -0,0 +1,25 @@ +open Core + +let start () = Unix.gettimeofday () + +let stop start = + (Unix.gettimeofday () -. start) *. 1000.0 + +exception Time_out + +let time_out ~after f args = + let behavior = + Signal.(Expert.signal alrm (`Handle (fun _ -> raise Time_out))) + in + let cancel_alarm () = + Unix.alarm 0 |> ignore; + Signal.(Expert.set alrm behavior) + in + Unix.alarm after |> ignore; + match f args with + | result -> + cancel_alarm (); + result + | exception exc -> + cancel_alarm (); + raise exc diff --git a/src/dune b/src/dune new file mode 100644 index 0000000..c812054 --- /dev/null +++ b/src/dune @@ -0,0 +1,12 @@ +(executables + (libraries comby core ppx_deriving_yojson ppx_deriving_yojson.runtime hack_parallel) + (preprocess (pps ppx_deriving_yojson ppx_let ppx_deriving.show)) + (names main)) + +(alias + (name DEFAULT) + (deps main.exe)) + +(install + (section bin) + (files (main.exe as comby))) diff --git a/src/main.ml b/src/main.ml new file mode 100644 index 0000000..b3143de --- /dev/null +++ b/src/main.ml @@ -0,0 +1,385 @@ +open Core +open Command.Let_syntax + +open Hack_parallel + +open Matchers +open Match +open Language +open Rewriter +open Statistics + +type json_result = + { matches : Match.t list + ; source : string + } +[@@deriving yojson] + +type input_kind = + | Paths of string list + | Path of string + | String of string +[@@deriving show] + +type processed_source_result = + | Matches of (Match.t list * int) + | Rewritten of (Rewrite.match_context_replacement list * string * int) + | Nothing + +let read = Fn.compose String.rstrip In_channel.read_all + +let read_template = + Fn.compose + String.chop_suffix_exn ~suffix:"\n" + In_channel.read_all + +let verbose_out_file = "/tmp/comby.out" + +let get_matches (module Matcher : Matchers.Matcher) configuration match_template match_rule source = + let rule = Rule.create match_rule |> Or_error.ok_exn in + Matcher.all ~configuration ~template:match_template ~source + |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule ~matcher:(module Matcher) environment)) + +let apply_rewrite_rule matcher rewrite_rule matches = + let open Option in + match rewrite_rule with + | "" -> matches + | rewrite_rule -> + begin + match Rule.create rewrite_rule with + | Ok rule -> + List.filter_map matches ~f:(fun ({ environment; _ } as match_) -> + let sat, env = Rule.apply rule ~matcher environment in + (if sat then env else None) + >>| fun environment -> { match_ with environment }) + | Error _ -> [] + end + +let rewrite rewrite_template _rewrite_rule source matches = + Rewrite.all ~source ~rewrite_template matches + +let process_single_source matcher verbose configuration source specification match_timeout = + let open Specification in + try + let input_text = + match source with + | String input_text -> input_text + | Path path -> + if verbose then + Out_channel.with_file ~append:true verbose_out_file ~f:(fun out_channel -> + Out_channel.output_lines out_channel [Format.sprintf "Processing %s%!" path]); + In_channel.read_all path + | _ -> failwith "Don't send multiple paths to process_single_source" + in + match specification with + | { match_specification = { match_template; match_rule } + ; rewrite_specification = None + } -> + let matches = + try + let f () = get_matches matcher configuration match_template match_rule input_text in + Statistics.Time.time_out ~after:match_timeout f (); + with Statistics.Time.Time_out -> + Out_channel.with_file ~append:true verbose_out_file ~f:(fun out_channel -> + Out_channel.output_lines out_channel [Format.sprintf "TIMEOUT: %s@." (show_input_kind source) ]); + [] + in + Matches (matches, List.length matches) + | { match_specification = { match_template; match_rule } + ; rewrite_specification = Some { rewrite_template; rewrite_rule } + } -> + let result = + try + let f () = + get_matches matcher configuration match_template match_rule input_text + |> fun matches -> + (* TODO(RVT): merge match and rewrite rule application. *) + apply_rewrite_rule matcher rewrite_rule matches + |> fun matches -> + if matches = [] then + (* If there are no matches, return the original source (for editor support). *) + Some (Some (Rewrite.{ rewritten_source = input_text; contextual_substitutions = [] }), []) + else + Some (rewrite rewrite_template rewrite_rule input_text matches, matches) + in + Statistics.Time.time_out ~after:match_timeout f (); + with Statistics.Time.Time_out -> + Out_channel.with_file ~append:true verbose_out_file ~f:(fun out_channel -> + Out_channel.output_lines out_channel [Format.sprintf "TIMEOUT: FOR %s@." (show_input_kind source) ]); + None + in + result + |> function + | Some (Some { rewritten_source; contextual_substitutions }, matches) -> + Rewritten (contextual_substitutions, rewritten_source, List.length matches) + | Some (None, _) + | None -> Nothing + with + | _ -> Nothing + +let output_result stdin spec_number json source_path result = + match result with + | Nothing -> () + | Matches (matches, _) -> + if json then + let json_matches = `List (List.map ~f:Match.to_yojson matches) in + Format.printf "%s%!" @@ Yojson.Safe.pretty_to_string json_matches + else + let with_file = + match source_path with + | Some path -> Format.sprintf " in %s " path + | None -> " " + in + Format.printf + "%d matches%sfor spec %d (add -json for json format)@." + (List.length matches) + with_file + (spec_number + 1) + | Rewritten (replacements, result, _) -> + match source_path, json, stdin with + (* default: rewrite in place *) + | Some path, false, false -> Out_channel.write_all path ~data:result + (* stdin, not JSON *) + | _, false, true -> Format.printf "%s%!" result + (* stdin, JSON with path *) + | Some path, true, _ -> + let json_rewrites = + let value = `List (List.map ~f:Rewrite.match_context_replacement_to_yojson replacements) in + `Assoc [(path, value)] + in + Format.printf "%s%!" @@ Yojson.Safe.pretty_to_string json_rewrites + (* JSON, no path *) + | None, true, _ -> + let json_rewrites = + `List (List.map ~f:Rewrite.match_context_replacement_to_yojson replacements) in + Format.printf "%s%!" @@ Yojson.Safe.pretty_to_string json_rewrites + | _ -> Format.printf "%s%!" result + +let write_statistics number_of_matches paths total_time = + let total_time = Statistics.Time.stop total_time in + let lines_of_code = + List.fold paths ~init:0 ~f:(fun acc paths -> + In_channel.read_lines paths + |> List.length + |> (+) acc) + in + let statistics = + { number_of_files = List.length paths + ; lines_of_code + ; number_of_matches + ; total_time = total_time + } + in + Format.eprintf "%s%!" + @@ Yojson.Safe.pretty_to_string + @@ Statistics.to_yojson statistics + +let paths_with_file_size paths = + List.map paths ~f:(fun path -> + let length = + In_channel.create path + |> fun channel -> + In_channel.length channel + |> Int64.to_int + |> (fun value -> Option.value_exn value) + |> (fun value -> In_channel.close channel; value) + in + (path, length)) + +let run + matcher + (sources : input_kind) + (specifications : Specification.t list) + sequential + number_of_workers + stdin + json + verbose + match_timeout = + let number_of_workers = if sequential then 0 else number_of_workers in + let scheduler = Scheduler.create ~number_of_workers () in + let configuration = Configuration.create ~match_kind:Fuzzy () in + let total_time = Statistics.Time.start () in + + let run_on_specifications input output_file = + let result, count = + List.fold specifications ~init:(Nothing,0) ~f:(fun (result, count) specification -> + let input = + match result with + | Nothing | Matches _ -> input + | Rewritten (_, content, _) -> String content + in + process_single_source matcher verbose configuration input specification match_timeout + |> function + | Nothing -> Nothing, count + | Matches (x, number_of_matches) -> + Matches (x, number_of_matches), count + number_of_matches + | Rewritten (x, content, number_of_matches) -> + Rewritten (x, content, number_of_matches), + count + number_of_matches) + in + output_result stdin 0 json output_file result; + count + in + + match sources with + | String source -> + let number_of_matches = run_on_specifications (String source) None in + (* FIXME(RVT): statistics for single source text doesn't output LOC *) + write_statistics number_of_matches [] total_time + | Paths paths -> + if sequential then + let number_of_matches = + List.fold ~init:0 paths ~f:(fun acc path -> + let matches = run_on_specifications (Path path) (Some path) in + acc + matches) + in + write_statistics number_of_matches paths total_time + else + let map init paths = + List.fold + paths + ~init + ~f:(fun count path -> count + run_on_specifications (Path path) (Some path)) + in + let number_of_matches = + try Scheduler.map_reduce scheduler ~init:0 ~map ~reduce:(+) paths + with End_of_file -> 0 + in + begin + try Scheduler.destroy scheduler + with Unix.Unix_error (_,"kill",_) -> + (* No kill command on Mac OS X *) + () + end; + write_statistics number_of_matches paths total_time + | _ -> failwith "No single path handled here" + +let parse_source_directories ?(file_extensions = []) target_directory = + let rec ls_rec path = + if Sys.is_file path = `Yes then + match file_extensions with + | [] -> [path] + | suffixes when List.exists suffixes ~f:(fun suffix -> String.is_suffix ~suffix path) -> + [path] + | _ -> [] + else + try + Sys.ls_dir path + |> List.map ~f:(fun sub -> ls_rec (Filename.concat path sub)) + |> List.concat + with + | _ -> [] + in + ls_rec target_directory + +let parse_specification_directories match_only specification_directory_paths = + let parse_directory path = + let match_template = + let filename = path ^/ "match" in + try read_template filename + with _ -> failwith (Format.sprintf "Could not read required match file %s" filename) + in + let match_rule = + let filename = path ^/ "match_rule" in + try Some (read filename) + with _ -> None + in + let rewrite_template = + let filename = path ^/ "rewrite" in + if match_only then + None + else + try Some (read_template filename) + with _ -> None + in + let rewrite_rule = + let filename = path ^/ "rewrite_rule" in + if match_only then + None + else + try Some (read filename) + with _ -> None + in + Specification.create ~match_template ?match_rule ?rewrite_template ?rewrite_rule () + in + List.map specification_directory_paths ~f:parse_directory + +let base_command_parameters : (unit -> 'result) Command.Param.t = + [%map_open + (* flags. *) + let sequential = flag "sequential" no_arg ~doc:"Run sequentially" + and match_only = flag "match-only" no_arg ~doc:"Only perform matching (ignore rewrite templates)" + and verbose = flag "verbose" no_arg ~doc:(Format.sprintf "Log to %s" verbose_out_file) + and rule = flag "rule" (optional_with_default "where true" string) ~doc:"rule Apply rules to matches. Respects -f" + and match_timeout = flag "timeout" (optional_with_default 3 int) ~doc:"seconds Set match timeout on a source. Default: 3" + and target_directory = flag "directory" (optional_with_default "." string) ~doc:(Format.sprintf "path Run on files in a directory. Default is current directory: %s" @@ Sys.getcwd ()) + and specification_directories = flag "templates" (optional (Arg_type.comma_separated string)) ~doc:"path CSV of directories containing templates" + and file_extensions = flag "filter" (optional (Arg_type.comma_separated string)) ~doc:"extensions CSV of extensions to include" + and json = flag "json" no_arg ~doc:"Output JSON format for matches or rewrite text to stdout" + and number_of_workers = flag "jobs" (optional_with_default 4 int) ~doc:"n Number of worker processes. Default: 4" + and stdin = flag "stdin" no_arg ~doc:"Read source from stdin" + and anonymous_arguments = + anon (maybe (t2 + ("MATCH_TEMPLATE" %: string) + ("REWRITE_TEMPLATE" %: string))) + in + fun () -> + let () = + match Rule.create rule with + | Ok _ -> () + | Error error -> + let message = Error.to_string_hum error in + Format.printf "Match rule parse error: %s@." message; + exit 1 + in + let specifications = + match specification_directories, anonymous_arguments with + | None, None + | Some [], None -> + Format.eprintf + "Please either specify templates on the command line or using \ + -templates [dir] for templates in directory [dir].@."; + exit 1 + | None, Some (match_template, rewrite_template) -> + if match_only then + [Specification.create ~match_template ~match_rule:rule ()] + else + [Specification.create ~match_template ~rewrite_template ~match_rule:rule ~rewrite_rule:rule ()] + | Some specification_directories, None -> + parse_specification_directories match_only specification_directories + | Some specification_directories, Some _ -> + Format.eprintf + "Warning: ignoring match and rewrite templates and rules on \ + commandline and using those in directories instead@."; + parse_specification_directories match_only specification_directories + in + let sources = + match stdin with + | false -> Paths (parse_source_directories ?file_extensions target_directory) + | true -> String (In_channel.input_all In_channel.stdin) + in + + let matcher = + let default = (module Matchers.C : Matchers.Matcher) in + match file_extensions with + | None | Some [] -> default + | Some (hd::_) -> + match hd with + | ".c" | ".h" | ".cc" | ".cpp" | ".hpp" -> (module Matchers.C : Matchers.Matcher) + | ".py" -> (module Matchers.Python : Matchers.Matcher) + | ".go" -> (module Matchers.Go : Matchers.Matcher) + | ".sh" -> (module Matchers.Bash : Matchers.Matcher) + | ".html" -> (module Matchers.Html : Matchers.Matcher) + | _ -> default + in + run matcher sources specifications sequential number_of_workers stdin json verbose match_timeout + ] + +let default_command = + Command.basic ~summary:"Run a rewrite pass." base_command_parameters + +let () = + Scheduler.Daemon.check_entry_point (); + default_command + |> Command.run diff --git a/src/specification.ml b/src/specification.ml new file mode 100644 index 0000000..7df1adf --- /dev/null +++ b/src/specification.ml @@ -0,0 +1,32 @@ +open Core + +type match_specification = + { match_template : string + ; match_rule : string + } +[@@deriving show] + +type rewrite_specification = + { rewrite_template : string + ; rewrite_rule : string + } +[@@deriving show] + +type t = + { match_specification : match_specification + ; rewrite_specification : rewrite_specification option + } +[@@deriving show] + +let create + ?rewrite_template + ?(match_rule = "where true") + ?(rewrite_rule = "") + ~match_template + () = + let match_specification = { match_template; match_rule } in + let rewrite_specification = + Option.map rewrite_template ~f:(fun rewrite_template -> + { rewrite_template; rewrite_rule}) + in + { match_specification; rewrite_specification } diff --git a/test/dune b/test/dune new file mode 100644 index 0000000..3a5f835 --- /dev/null +++ b/test/dune @@ -0,0 +1,21 @@ +(library + (name test_integration) + (modules + test_match_rule + test_integration + test_statistics + test_c + test_cli + test_bash + test_go + test_c_style_comments + test_c_separators + test_string_literals + test_generic + test_rewrite_parts + test_rewrite_rule) + (inline_tests) + (preprocess (pps ppx_expect ppx_sexp_message)) + (libraries + comby + core)) diff --git a/test/test_bash.ml b/test/test_bash.ml new file mode 100644 index 0000000..2c26244 --- /dev/null +++ b/test/test_bash.ml @@ -0,0 +1,77 @@ +open Core + +open Matchers +open Rewriter + +let configuration = Configuration.create ~match_kind:Fuzzy () + +let run_bash source match_template rewrite_template = + Bash.first ~configuration match_template source + |> function + | Ok result -> + Rewrite.all ~source ~rewrite_template [result] + |> (fun x -> Option.value_exn x) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string + | Error _ -> + print_string rewrite_template + +let run_go source match_template rewrite_template = + Go.first ~configuration match_template source + |> function + | Ok result -> + Rewrite.all ~source ~rewrite_template [result] + |> (fun x -> Option.value_exn x) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string + | Error _ -> + print_string rewrite_template + +let%expect_test "custom_long_delimiters" = + let source = + {| + case + case + block 1 + esac + + case + block 2 + esac + esac + |} + in + let match_template = {|case :[1] esac|} in + let rewrite_template = {|case nuked blocks esac|} in + + run_bash source match_template rewrite_template; + [%expect_exact {| + case nuked blocks esac + |}] + +let%expect_test "custom_long_delimiters_doesn't_work_in_go" = + let source = + {| + case + case + block 1 + esac + + case + block 2 + esac + esac + |} + in + let match_template = {|case :[1] esac|} in + let rewrite_template = {|case nuked blocks esac|} in + + run_go source match_template rewrite_template; + [%expect_exact {| + case nuked blocks esac + + case + block 2 + esac + esac + |}] diff --git a/test/test_c.ml b/test/test_c.ml new file mode 100644 index 0000000..cb83860 --- /dev/null +++ b/test/test_c.ml @@ -0,0 +1,105 @@ +open Core + +open Matchers +open Rewriter + +let configuration = Configuration.create ~match_kind:Fuzzy () + +let run source match_template rewrite_template = + C.first ~configuration match_template source + |> function + | Ok result -> + Rewrite.all ~source ~rewrite_template [result] + |> (fun x -> Option.value_exn x) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string + | Error _ -> + print_string rewrite_template + +let%expect_test "comments_1" = + let source = {|match this /**/ expect end|} in + let match_template = {|match this :[1] end|} in + let rewrite_template = {|:[1]|} in + + run source match_template rewrite_template; + [%expect_exact {|expect|}] + +let%expect_test "comments_2" = + let source = {|match this /* */ expect end|} in + let match_template = {|match this :[1] end|} in + let rewrite_template = {|:[1]|} in + + run source match_template rewrite_template; + [%expect_exact {|expect|}] + +let%expect_test "comments_3" = + let source = {|match this /* blah blah */ expect /**/ end|} in + let match_template = {|match this :[1] end|} in + let rewrite_template = {|:[1]|} in + + run source match_template rewrite_template; + [%expect_exact {|expect|}] + +let%expect_test "comments_4" = + let source = {|match this expect/**/end|} in + let match_template = {|match this :[1]end|} in + let rewrite_template = {|:[1]|} in + + run source match_template rewrite_template; + [%expect_exact {|expect|}] + +let%expect_test "comments_5" = + let source = {|match this expect /**/end|} in + let match_template = {|match this :[1] end|} in + let rewrite_template = {|:[1]|} in + + run source match_template rewrite_template; + [%expect_exact {|expect|}] + +let%expect_test "comments_6" = + let source = {|/* don't match this (a) end */|} in + let match_template = {|match this :[1] end|} in + let rewrite_template = {|nothing matches|} in + + run source match_template rewrite_template; + [%expect_exact {|nothing matches|}] + +let%expect_test "comments_7" = + let source = {|/* don't match /**/ this (a) end */|} in + let match_template = {|match this :[1] end|} in + let rewrite_template = {|nothing matches|} in + + run source match_template rewrite_template; + [%expect_exact {|nothing matches|}] + +let%expect_test "comments_8" = + let source = {|(/* don't match this (a) end */)|} in + let match_template = {|match this :[1] end|} in + let rewrite_template = {|nothing matches|} in + + run source match_template rewrite_template; + [%expect_exact {|nothing matches|}] + +let%expect_test "comments_9" = + let source = {|/* don't match this (a) end */ do match this (b) end|} in + let match_template = {|match this :[1] end|} in + let rewrite_template = {|:[1]|} in + + run source match_template rewrite_template; + [%expect_exact {|/* don't match this (a) end */ do (b)|}] + +let%expect_test "comments_10" = + let source = {|/* don't match this (a) end */ do match this () end|} in + let match_template = {|match this :[1] end|} in + let rewrite_template = {|:[1]|} in + + run source match_template rewrite_template; + [%expect_exact {|/* don't match this (a) end */ do ()|}] + +let%expect_test "comments_11" = + let source = {|do match this (b) end /* don't match this (a) end */|} in + let match_template = {|match this :[1] end|} in + let rewrite_template = {|:[1]|} in + + run source match_template rewrite_template; + [%expect_exact {|do (b) /* don't match this (a) end */|}] diff --git a/test/test_c_separators.ml b/test/test_c_separators.ml new file mode 100644 index 0000000..8253374 --- /dev/null +++ b/test/test_c_separators.ml @@ -0,0 +1,36 @@ +open Core + +open Matchers +open Rewriter + +let configuration = Configuration.create ~match_kind:Fuzzy () + +let run source match_template rewrite_template = + C.first ~configuration match_template source + |> function + | Ok result -> + Rewrite.all ~source ~rewrite_template [result] + |> (fun x -> Option.value_exn x) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string + | Error _ -> + print_string rewrite_template + +let%expect_test "whitespace_should_not_matter_between_separators" = + let source = {|*p|} in + let match_template = {|*:[1]|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|p|}]; + + let source = {|* p|} in + let match_template = {|*:[1]|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {| p|}]; + + let source = {|* p|} in + let match_template = {|* :[1]|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|p|}] diff --git a/test/test_c_style_comments.ml b/test/test_c_style_comments.ml new file mode 100644 index 0000000..14a10e3 --- /dev/null +++ b/test/test_c_style_comments.ml @@ -0,0 +1,225 @@ +open Core + +open Matchers +open Rewriter + +let configuration = Configuration.create ~match_kind:Fuzzy () + +let all ?(configuration = configuration) template source = + C.all ~configuration ~template ~source + +let print_matches matches = + List.map matches ~f:Match.to_yojson + |> (fun matches -> `List matches) + |> Yojson.Safe.pretty_to_string + |> print_string + +let%expect_test "rewrite_comments_1" = + let template = "replace this :[1] end" in + let source = "/* don't replace this () end */ do replace this () end" in + let rewrite_template = "X" in + + all template source + |> (fun matches -> + Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact "/* don't replace this () end */ do X"] + +let%expect_test "rewrite_comments_2" = + let template = + {| + if (:[1]) { :[2] } + |} + in + + let source = + {| + /* if (fake_condition_body_must_be_non_empty) { fake_body; } */ + // if (fake_condition_body_must_be_non_empty) { fake_body; } + if (real_condition_body_must_be_empty) { + int i; + int j; + } + |} + in + + let rewrite_template = + {| + if (:[1]) {} + |} + in + + all template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact + {| + /* if (fake_condition_body_must_be_non_empty) { fake_body; } */ + if (real_condition_body_must_be_empty) {} + |}] + +let%expect_test "capture_comments" = + let template = {|if (:[1]) { :[2] }|} in + let source = {|if (true) { /* some comment */ console.log(z); }|} in + let matches = all template source in + print_matches matches; + [%expect_exact {|[ + { + "range": { + "start": { "offset": 0, "line": 1, "column": 1 }, + "end": { "offset": 48, "line": 1, "column": 49 } + }, + "environment": [ + [ + "1", + { + "value": "true", + "range": { + "start": { "offset": 4, "line": 1, "column": 5 }, + "end": { "offset": 8, "line": 1, "column": 9 } + } + } + ], + [ + "2", + { + "value": "console.log(z);", + "range": { + "start": { "offset": 31, "line": 1, "column": 32 }, + "end": { "offset": 46, "line": 1, "column": 47 } + } + } + ] + ], + "matched": "if (true) { /* some comment */ console.log(z); }" + } +]|}] + +let%expect_test "single_quote_in_comment" = + let template = + {| {:[1]} |} + in + + let source = + {| + /*'*/ + {test} + |} + in + + let rewrite_template = + {| + {:[1]} + |} + in + + all template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact + {| + {test} + |}] + +let%expect_test "single_quote_in_comment" = + let template = + {| {:[1]} |} + in + + let source = + {| + { + a = 1; + /* Events with mask == AE_NONE are not set. So let's initiaize the + * vector with it. */ + for (i = 0; i < setsize; i++) + } + |} + in + + let rewrite_template = + {| + {:[1]} + |} + in + + all template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact + {| + { + a = 1; + /* Events with mask == AE_NONE are not set. So let's initiaize the + * vector with it. */ + for (i = 0; i < setsize; i++) + } + |}] + +let%expect_test "single_quote_in_comment" = + let template = + {| {:[1]} |} + in + + let source = + {| + { + a = 1; + /* ' */ + for (i = 0; i < setsize; i++) + } + |} + in + + let rewrite_template = + {| + {:[1]} + |} + in + + all template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact + {| + { + a = 1; + /* ' */ + for (i = 0; i < setsize; i++) + } + |}] + +let%expect_test "give_back_the_comment_characters_for_newline_comments_too" = + let template = + {| {:[1]} |} + in + + let source = + {| + { + // a comment + } + |} + in + + let rewrite_template = + {| + {:[1]} + |} + in + + all template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact + {| + { + // a comment + } + |}] diff --git a/test/test_cli.ml b/test/test_cli.ml new file mode 100644 index 0000000..bdcf245 --- /dev/null +++ b/test/test_cli.ml @@ -0,0 +1,178 @@ +open Core + +module Time = Core_kernel.Time_ns.Span + +let binary_path = "../../../comby" + +let read_with_timeout read_from_channel = + let read_from_fd = Unix.descr_of_in_channel read_from_channel in + let read_from_channel = + Unix.select ~read:[read_from_fd] ~write:[] ~except:[] ~timeout:(`After (Time.of_int_sec 5)) () + |> (fun { Unix.Select_fds.read; _ } -> List.hd_exn read) + |> Unix.in_channel_of_descr + in + In_channel.input_all read_from_channel + +let read_source_from_stdin command source = + let open Unix.Process_channels in + let { stdin; stdout ; stderr = _ } = Unix.open_process_full ~env:[||] command in + Out_channel.output_string stdin source; + Out_channel.flush stdin; + Out_channel.close stdin; + read_with_timeout stdout + +let%expect_test "stdin_command" = + let source = "hello world" in + let match_template = "hello :[1]" in + let rewrite_template = ":[1]" in + let command_args = + Format.sprintf "-stdin '%s' '%s' -f .c" match_template rewrite_template + in + let command = Format.sprintf "%s %s" binary_path command_args in + read_source_from_stdin command source + |> print_string; + [%expect_exact {|world|}] + +let%expect_test "with_match_rule" = + let source = "hello world" in + let match_template = "hello :[1]" in + let rewrite_template = ":[1]" in + let rule = {|where :[1] == "world"|} in + let command_args = + Format.sprintf "-stdin '%s' '%s' -rule '%s' -f .c " + match_template rewrite_template rule + in + let command = Format.sprintf "%s %s" binary_path command_args in + read_source_from_stdin command source + |> print_string; + [%expect_exact {|world|}]; + + let source = "hello world" in + let match_template = "hello :[1]" in + let rewrite_template = ":[1]" in + let rule = {|where :[1] != "world"|} in + let command_args = + Format.sprintf "-stdin '%s' '%s' -rule '%s' -f .c " + match_template rewrite_template rule + in + let command = Format.sprintf "%s %s" binary_path command_args in + read_source_from_stdin command source + |> print_string; + [%expect_exact {|hello world|}] + +let%expect_test "with_rewrite_rule" = + let source = "hello world" in + let match_template = ":[2] :[1]" in + let rewrite_template = ":[1]" in + let rule = {|where rewrite :[1] { | ":[_]" -> ":[2]" }|} in + let command_args = + Format.sprintf "-stdin '%s' '%s' -rule '%s' -f .c " + match_template rewrite_template rule + in + let command = Format.sprintf "%s %s" binary_path command_args in + read_source_from_stdin command source + |> print_string; + [%expect_exact {|hello|}] + +let%expect_test "json_output_option" = + let source = "a X c a Y c" in + let match_template = "a :[1] c" in + let rewrite_template = "c :[1] a" in + let command_args = + Format.sprintf "-stdin -json '%s' '%s' -f .c " + match_template rewrite_template + in + let command = Format.sprintf "%s %s" binary_path command_args in + read_source_from_stdin command source + |> print_string; + [%expect_exact {|[ + { + "range": { + "start": { "offset": 6, "line": -1, "column": -1 }, + "end": { "offset": 11, "line": -1, "column": -1 } + }, + "replacement_content": "c Y a", + "environment": [ + [ + "1", + { + "value": "Y", + "range": { + "start": { "offset": 2, "line": -1, "column": -1 }, + "end": { "offset": 3, "line": -1, "column": -1 } + } + } + ] + ] + }, + { + "range": { + "start": { "offset": 0, "line": -1, "column": -1 }, + "end": { "offset": 5, "line": -1, "column": -1 } + }, + "replacement_content": "c X a", + "environment": [ + [ + "1", + { + "value": "X", + "range": { + "start": { "offset": 2, "line": -1, "column": -1 }, + "end": { "offset": 3, "line": -1, "column": -1 } + } + } + ] + ] + } +]|}]; + + let source = "a X c a Y c" in + let match_template = "a :[1] c" in + let rewrite_template = "c :[1] a" in + let command_args = + Format.sprintf "-stdin -json -match-only '%s' '%s' -f .c " + match_template rewrite_template + in + let command = Format.sprintf "%s %s" binary_path command_args in + read_source_from_stdin command source + |> print_string; + [%expect_exact {|[ + { + "range": { + "start": { "offset": 0, "line": 1, "column": 1 }, + "end": { "offset": 5, "line": 1, "column": 6 } + }, + "environment": [ + [ + "1", + { + "value": "X", + "range": { + "start": { "offset": 2, "line": 1, "column": 3 }, + "end": { "offset": 3, "line": 1, "column": 4 } + } + } + ] + ], + "matched": "a X c" + }, + { + "range": { + "start": { "offset": 6, "line": 1, "column": 7 }, + "end": { "offset": 11, "line": 1, "column": 12 } + }, + "environment": [ + [ + "1", + { + "value": "Y", + "range": { + "start": { "offset": 8, "line": 1, "column": 9 }, + "end": { "offset": 9, "line": 1, "column": 10 } + } + } + ] + ], + "matched": "a Y c" + } +]|}] diff --git a/test/test_generic.ml b/test/test_generic.ml new file mode 100644 index 0000000..d3e15a0 --- /dev/null +++ b/test/test_generic.ml @@ -0,0 +1,408 @@ +open Core + +open Matchers +open Rewriter + +let configuration = Configuration.create ~match_kind:Fuzzy () + +let format s = + let s = String.chop_prefix_exn ~prefix:"\n" s in + let leading_indentation = Option.value_exn (String.lfindi s ~f:(fun _ c -> c <> ' ')) in + s + |> String.split ~on:'\n' + |> List.map ~f:(Fn.flip String.drop_prefix leading_indentation) + |> String.concat ~sep:"\n" + |> String.chop_suffix_exn ~suffix:"\n" + +let run ?(configuration = configuration) source match_template rewrite_template = + Generic.first ~configuration match_template source + |> function + | Ok result -> + Rewrite.all ~source ~rewrite_template [result] + |> (fun x -> Option.value_exn x) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string + | Error _ -> + (* this is too annoying to fix everytime the grammar changes. *) + print_string "" + +let run_all ?(configuration = configuration) source match_template rewrite_template = + Generic.all ~configuration ~template:match_template ~source + |> (fun results -> Option.value_exn (Rewrite.all ~source ~rewrite_template results)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string + + +let%expect_test "basic" = + let source = {|a b c d|} in + let match_template = {|:[1]|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|a b c d|}]; + + let source = {|a b c d|} in + let match_template = {|a :[1] c d|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|b|}]; + + let source = {|a b c d|} in + let match_template = {|a :[1] d|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|b c|}]; + + let source = {|a b c d|} in + let match_template = {|a :[1]|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|b c d|}]; + + let source = {|a b c d|} in + let match_template = {|:[1] c d|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|a b|}]; + + let source = {|a b c d|} in + let match_template = {|:[1] :[2]|} in + let rewrite_template = {|(:[1]) (:[2])|} in + run source match_template rewrite_template; + [%expect_exact {|(a) (b c d)|}]; + + let source = {|a b c d|} in + let match_template = {|:[2] :[1]|} in + let rewrite_template = {|(:[2]) (:[1])|} in + run source match_template rewrite_template; + [%expect_exact {|(a) (b c d)|}]; + + let source = {|a b c d|} in + let match_template = {|a :[2] :[1] d|} in + let rewrite_template = {|(:[2]) (:[1])|} in + run source match_template rewrite_template; + [%expect_exact {|(b) (c)|}]; + + let source = {|a b c d|} in + let match_template = {|a :[2] :[1]|} in + let rewrite_template = {|(:[2]) (:[1])|} in + run source match_template rewrite_template; + [%expect_exact {|(b) (c d)|}]; + + let source = {|a b c d|} in + let match_template = {|a :[2] c :[1]|} in + let rewrite_template = {|(:[2]) (:[1])|} in + run source match_template rewrite_template; + [%expect_exact {|(b) (d)|}]; + + let source = {|x:|} in + let match_template = {|:[1]:|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|x|}] + +let%expect_test "basic_failures" = + let source = {|a x b bbq|} in + let match_template = {|a :[1] b c|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {||}]; + + let source = {|a b c d|} in + let match_template = {|a :[2] d :[1]|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact + {||}]; + + let source = {|a b c d|} in + let match_template = {|a :[2] b :[1]|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {||}] + +let%expect_test "delimiter_matching" = + let source = {|(a b c) d|} in + let match_template = {|(:[1]) d|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|a b c|}]; + + let source = {|(a b c) d|} in + let match_template = {|(:[1] b :[2]) d|} in + let rewrite_template = {|(:[1]) (:[2])|} in + run source match_template rewrite_template; + [%expect_exact {|(a) (c)|}]; + + let source = {|q(a b c) d|} in + let match_template = {|q(:[1] b :[2]) d|} in + let rewrite_template = {|(:[1]) (:[2])|} in + run source match_template rewrite_template; + [%expect_exact {|(a) (c)|}]; + + let source = {|((a) b)|} in + let match_template = {|(:[1] b)|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|(a)|}]; + + let source = {|((a b c)) d|} in + let match_template = {|(:[1]) d|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|(a b c)|}]; + + let source = {|((a b c)) d|} in + let match_template = {|(:[1]) d|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|(a b c)|}]; + + let source = {|((a b c) q) d|} in + let match_template = {|((:[1]) q) d|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|a b c|}]; + + let source = {|((a b c) q) d|} in + let match_template = {|((:[1] c) q) d|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|a b|}]; + + let source = {|((a b () c) q) d|} in + let match_template = {|((:[1] () c) q) d|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|a b|}]; + + let source = {|((a ((x) d) b c)) d|} in + let match_template = {|((a :[1] :[2] c)) d|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|((x) d)|}]; + + let source = {|((a ((x) d) b c)) d|} in + let match_template = {|((a (:[1]) :[2] c)) d|} in + let rewrite_template = {|:[1] :[2]|} in + run source match_template rewrite_template; + [%expect_exact {|(x) d b|}]; + + let source = {|(b (c) d)|} in + let match_template = {|(:[1])|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|b (c) d|}]; + + let source = {|(b (c) d.)|} in + let match_template = {|(:[1].)|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|b (c) d|}]; + + let source = {|(b (c.) d.)|} in + let match_template = {|(:[1].)|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|b (c.) d|}]; + + let source = {|(b. (c) d.)|} in + let match_template = {|(:[1].)|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|b. (c) d|}]; + + let source = {|(b (c) d.)|} in + let match_template = {|(b :[1] d.)|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|(c)|}]; + + let source = {|outer(inner(dst,src),src)|} in + let match_template = {|outer(:[1],src)|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|inner(dst,src)|}]; + + let source = {|(b ((c)) d.)|} in + let match_template = {|(b :[1] d.)|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|((c))|}]; + + let source = {|a b c|} in + let match_template = {|a :[1] c|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|b|}]; + + let source = {|x = foo;|} in + let match_template = {|x = :[1];|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|foo|}]; + + let source = {|((a {{x} d} b c)) d|} in + let match_template = {|((a {:[1] d} :[2] c)) d|} in + let rewrite_template = {|:[1] :[2]|} in + run source match_template rewrite_template; + [%expect_exact {|{x} b|}]; + + let source = {|((a {([{x}]) d} b c)) d|} in + let match_template = {|((a {:[1] d} :[2] c)) d|} in + let rewrite_template = {|:[1] :[2]|} in + run source match_template rewrite_template; + [%expect_exact {|([{x}]) b|}]; + + let source = {|(((((x)))))|} in + let match_template = {|(((:[1])))|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|((x))|}]; + + let source = {|((((y(x)z))))|} in + let match_template = {|(((:[1])))|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|(y(x)z)|}]; + + let source = {|((((y(x)z))))|} in + let match_template = {|(((:[1]):[2]))|} in + let rewrite_template = {|:[1] :[2]|} in + run source match_template rewrite_template; + [%expect_exact {|(y(x)z) |}]; + + let source = {|(((x)z))|} in + let match_template = {|(((:[1]):[2]))|} in + let rewrite_template = {|:[1] :[2]|} in + run source match_template rewrite_template; + [%expect_exact {|x z|}]; + + let source = {|((((x))z))|} in + let match_template = {|(((:[1]):[2]))|} in + let rewrite_template = {|:[1] :[2]|} in + run source match_template rewrite_template; + [%expect_exact {|(x) z|}]; + + let source = {|lolwtfbbq|} in + let match_template = {|lol:[1]bbq|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|wtf|}]; + + let source = {|x = foo; x = bar;|} in + let match_template = {|x = :[1];|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|foo x = bar;|}]; + + let source = {|[ no match prefix ] x = foo; [ no match suffix ]|} in + let match_template = {|x = :[1];|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|[ no match prefix ] foo [ no match suffix ]|}]; + + let source = {|x = a; x = b; x = c|} in + let match_template = {|x = :[1];|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|a x = b; x = c|}]; + + let source = {|x = ( x = x; );|} in + let match_template = {|x = :[1];|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|( x = x; )|}]; + + let source = {|( x = x = x; )|} in + let match_template = {|x = :[1];|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|( x = x )|}]; + + let source = {|xxx a b d c 1 2 3 b d d blah|} in + let match_template = {|a :[1] c :[2] d|} in + let rewrite_template = {|:[1] :[2]|} in + run source match_template rewrite_template; + [%expect_exact {|xxx b d 1 2 3 b d blah|}]; + + let source = {|howevenlolwtfbbqispossible|} in + let match_template = {|lol:[1]bbq|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|howevenwtfispossible|}]; + + let source = {|lolhowevenlolwtfbbqispossiblebbq|} in + let match_template = {|lol:[1]bbq|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|howevenlolwtfispossiblebbq|}]; + + let source = {|hello my name is bob the builder|} in + let match_template = {|:[alongidentifiername] :[2] :[3] :[xyz] :[5] :[6]|} in + let rewrite_template = {|:[alongidentifiername] :[2] :[3] :[xyz] :[5] :[6]|} in + run source match_template rewrite_template; + [%expect_exact {|hello my name is bob the builder|}]; + + let source = {|www.testdofooname.com/picsinsideit/stunningpictureofkays1381737242g8k4n-280x428.jpg|} in + let match_template = {|www.:[1]-:[2].jpg|} in + let rewrite_template = {|:[1] :[2]|} in + run source match_template rewrite_template; + [%expect_exact {|testdofooname.com/picsinsideit/stunningpictureofkays1381737242g8k4n 280x428|}]; + + let source = {|https://api.github.com/repos/dmjacobsen/slurm/commits/716c1499695c68afcab848a1b49653574b4fc167|} in + let match_template = {|:[1]api.:[2]/repos/:[3]s/:[4]|} in + let rewrite_template = {|:[1] :[2] :[3] :[4]|} in + run source match_template rewrite_template; + [%expect_exact {|https:// github.com dmjacobsen/slurm/commit 716c1499695c68afcab848a1b49653574b4fc167|}]; + + let source = + {| + assert(stream->md_len + md_len - + si.foo_data_begin <= MAD_BUFFER_MDLEN); + memcpy(*stream->foo_data + stream->md_len, + mad_bit_nextbyte(&stream->ptr), + frame_used = md_len - si.foo_data_begin); + stream->md_len += frame_used; + |} |> format + in + let match_template = {|memcpy(:[1], :[2], :[3]);|} in + let rewrite_template = {|:[1], :[2], :[3]|} in + run source match_template rewrite_template; + [%expect_exact {|assert(stream->md_len + md_len - + si.foo_data_begin <= MAD_BUFFER_MDLEN); +*stream->foo_data + stream->md_len, mad_bit_nextbyte(&stream->ptr), frame_used = md_len - si.foo_data_begin +stream->md_len += frame_used;|}] + +let%expect_test "significant_whitespace" = + let configuration = Configuration.create ~match_kind:Fuzzy ~significant_whitespace:true () in + let run = run ~configuration in + + let source = {|two spaces|} in + let match_template = {|:[1] :[2]|} in + let rewrite_template = {|:[1] :[2]|} in + run source match_template rewrite_template; + [%expect_exact {|two spaces|}]; + + (* FIXME: this should fail. also test case where separators do or do not need + whitespace. e.g., strict about strcpy(src,dst) matching a template + strcpy(:[1],:[2]) versus strcpy(:[1], :[2]) *) + let source = {|two spaces|} in + let match_template = {|:[1] :[2]|} in + let rewrite_template = {|:[1] :[2]|} in + run source match_template rewrite_template; + [%expect_exact {|two spaces|}] + +let%expect_test "contextual_matching" = + let run = run_all in + + let source = {|memcpy(dst1, src1, 1); memcpy(dst2, src2, 2);|} in + let match_template = {|memcpy(:[1], :[2], :[3])|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|dst1; dst2;|}]; + + let source = {|memcpy(dst1, src1, 1); memcpy(dst2, src2, 2);|} in + let match_template = {|memcpy(:[1], :[2], :[3])|} in + let rewrite_template = {|:[1]|} in + run source match_template rewrite_template; + [%expect_exact {|dst1; dst2;|}]; diff --git a/test/test_go.ml b/test/test_go.ml new file mode 100644 index 0000000..a4393ca --- /dev/null +++ b/test/test_go.ml @@ -0,0 +1,101 @@ +open Core + +open Language +open Matchers +open Match +open Rewriter + +let configuration = Configuration.create ~match_kind:Fuzzy () + +let run ?(rule = "where true") source match_template rewrite_template = + let rule = Rule.create rule |> Or_error.ok_exn in + Go.first ~configuration match_template source + |> function + | Ok ({environment; _ } as result) -> + if Rule.(sat @@ apply rule environment) then + Rewrite.all ~source ~rewrite_template [result] + |> (fun x -> Option.value_exn x) + |> (fun { rewritten_source; _ } -> rewritten_source) |> print_string + else + assert false + | Error _ -> + print_string rewrite_template + +let%expect_test "gosimple_s1000" = + let source = + {| + select { + case x := <-ch: + fmt.Println(x) + } + |} + in + + let match_template = + {| + select { + case :[1] := :[2]: + :[3] + } + |} + in + + let rewrite_template = + {| + :[1] := :[2] + :[3] + |} + in + run source match_template rewrite_template; + [%expect_exact {| + x := <-ch + fmt.Println(x) + |}] + +let%expect_test "gosimple_s1001" = + let source = + {| + for i, x := range src { + dst[i] = x + } + |} + in + + let match_template = + {| + for :[index_define], :[src_element_define] := range :[src_array] { + :[dst_array][:[index_use]] = :[src_element_use] + } + |} + in + + let rewrite_template = + {| + copy(:[dst_array], :[src_array]) + |} + in + + let rule = {|where :[index_define] == :[index_use], :[src_element_define] == :[src_element_use]|} in + + run ~rule source match_template rewrite_template; + [%expect_exact {| + copy(dst, src) + |}] + +let%expect_test "gosimple_s1003" = + let source = + {| + if strings.Index(x, y) != -1 { ignore } + |} + in + + let match_template = + {| + if strings.:[1](x, y) != -1 { :[_] } + |} + in + + let rewrite_template = {|:[1]|} in + + run source match_template rewrite_template; + [%expect_exact {|Index|}] diff --git a/test/test_integration.ml b/test/test_integration.ml new file mode 100644 index 0000000..7d07c71 --- /dev/null +++ b/test/test_integration.ml @@ -0,0 +1,295 @@ +open Core + +open Matchers +open Rewriter + +let configuration = Configuration.create ~match_kind:Fuzzy () + +let all ?(configuration = configuration) template source = + Generic.all ~configuration ~template ~source + +let print_matches matches = + List.map matches ~f:Match.to_yojson + |> (fun matches -> `List matches) + |> Yojson.Safe.pretty_to_string + |> print_string + +let%expect_test "dont_get_stuck" = + let template = "" in + let source = "a" in + let matches = all ~configuration template source in + print_matches matches; + [%expect_exact {|[]|}] + +let%expect_test "dont_get_stuck" = + let template = "a" in + let source = "a" in + let matches = all template source in + print_matches matches; + [%expect_exact {|[ + { + "range": { + "start": { "offset": 0, "line": 1, "column": 1 }, + "end": { "offset": 1, "line": 1, "column": 2 } + }, + "environment": [], + "matched": "a" + } +]|}] + +let%expect_test "dont_get_stuck" = + let template = "a" in + let source = "aaa" in + let matches = all template source in + print_matches matches; + [%expect_exact {|[ + { + "range": { + "start": { "offset": 0, "line": 1, "column": 1 }, + "end": { "offset": 1, "line": 1, "column": 2 } + }, + "environment": [], + "matched": "a" + }, + { + "range": { + "start": { "offset": 1, "line": 1, "column": 2 }, + "end": { "offset": 2, "line": 1, "column": 3 } + }, + "environment": [], + "matched": "a" + }, + { + "range": { + "start": { "offset": 2, "line": 1, "column": 3 }, + "end": { "offset": 3, "line": 1, "column": 4 } + }, + "environment": [], + "matched": "a" + } +]|}] + +let%expect_test "rewrite_awesome_1" = + let template = "replace this :[1] end" in + let source = "xreplace this () end" in + let rewrite_template = "X" in + + all template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact "xX"] + +let%expect_test "rewrite_whole_template_matches" = + let template = {|rewrite :[1] <- this string|} in + let source = {|rewrite hello world <- this string|} in + let rewrite_template = "?" in + + all template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact "?"] + +let%expect_test "single_token" = + let template = {|:[[1]] this|} in + let source = {|the problem is this|} in + let matches = all template source in + print_matches matches; + [%expect_exact {|[ + { + "range": { + "start": { "offset": 12, "line": 1, "column": 13 }, + "end": { "offset": 19, "line": 1, "column": 20 } + }, + "environment": [ + [ + "1", + { + "value": "is", + "range": { + "start": { "offset": 12, "line": 1, "column": 13 }, + "end": { "offset": 14, "line": 1, "column": 15 } + } + } + ] + ], + "matched": "is this" + } +]|}] + + +let%expect_test "single_token_with_preceding_whitespace" = + let template = {| :[[1]] this|} in + let source = {|the problem is this|} in + let matches = all template source in + print_matches matches; + [%expect_exact {|[ + { + "range": { + "start": { "offset": 11, "line": 1, "column": 12 }, + "end": { "offset": 19, "line": 1, "column": 20 } + }, + "environment": [ + [ + "1", + { + "value": "is", + "range": { + "start": { "offset": 12, "line": 1, "column": 13 }, + "end": { "offset": 14, "line": 1, "column": 15 } + } + } + ] + ], + "matched": " is this" + } +]|}] + +let%expect_test "single_token_rewrite" = + let template = {| :[[1]] this|} in + let source = {|the problem is this|} in + let rewrite_template = ":[1]" in + all template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact "the problemis"] + +let%expect_test "single_token_match_inside_paren_no_succeeding_whitespace" = + let template = {|:[[1]](:[[2]])|} in + let source = {|foo(bar)|} in + let rewrite_template = ":[1] : :[2]" in + all template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact "foo : bar"] + + +let%expect_test "shift_or_at_least_dont_get_stuck" = + let template = ":[1]" in + let source = "a" in + let matches = all template source in + print_matches matches; + [%expect_exact {|[ + { + "range": { + "start": { "offset": 0, "line": 1, "column": 1 }, + "end": { "offset": 1, "line": 1, "column": 2 } + }, + "environment": [ + [ + "1", + { + "value": "a", + "range": { + "start": { "offset": 0, "line": 1, "column": 1 }, + "end": { "offset": 1, "line": 1, "column": 2 } + } + } + ] + ], + "matched": "a" + } +]|}] + +let%expect_test "shift_or_at_least_dont_get_stuck" = + let template = ":[1]" in + let source = "aa" in + let matches = all template source in + print_matches matches; + [%expect_exact {|[ + { + "range": { + "start": { "offset": 0, "line": 1, "column": 1 }, + "end": { "offset": 2, "line": 1, "column": 3 } + }, + "environment": [ + [ + "1", + { + "value": "aa", + "range": { + "start": { "offset": 0, "line": 1, "column": 1 }, + "end": { "offset": 2, "line": 1, "column": 3 } + } + } + ] + ], + "matched": "aa" + } +]|}] + +let%expect_test "nested_rewrite1" = + let source = + {| + x x y strcpy(strcpy(dst1, src1), src2); blah blah XXX + |} + in + + let template = + {| + strcpy(:[1], :[2]) + |} + in + + let matches = all template source in + print_matches matches; + [%expect_exact "[]"] + +(* FIXME(RVT) nested rewrites *) +let%expect_test "nested_rewrite2" = + let template = + {| + if :[var_check] != nil { + for :[defines] := range :[var_use] {:[inner_body]} + } + |} + in + let source = + {| + if fields.List != nil { + for _, field := range fields.List { + if field.Names != nil { + for _, fieldName := range field.Names { + stuff with fields and things + } + } + } + } + |} + in + let rewrite_template = "for :[defines] := range :[var_use] {:[inner_body]}" in + all template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact {|for _, field := range fields.List { + if field.Names != nil { + for _, fieldName := range field.Names { + stuff with fields and things + } + } + }|}] + +let%expect_test "match_:[[1]]" = + let template = + {| + :[[1]].next() + |} + in + let source = + {| + col_names = reader.next() + } + |} + in + let rewrite_template = "next(:[1])" in + all template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact {| + col_names =next(reader)} + |}] diff --git a/test/test_match_rule.ml b/test/test_match_rule.ml new file mode 100644 index 0000000..4a15510 --- /dev/null +++ b/test/test_match_rule.ml @@ -0,0 +1,719 @@ +open Core + +open Language +open Matchers +open Match + +let rule_parses rule = + match Rule.create rule with + | Ok _ -> "true" + | Error _ -> "false" + +let print_matches matches = + List.map matches ~f:Match.to_yojson + |> (fun matches -> `List matches) + |> Yojson.Safe.pretty_to_string + |> print_string + +let%expect_test "parse_rule" = + let rule = {| where :[1] == :[2], :[3] == "y" |} in + rule_parses rule |> print_string; + [%expect_exact {|true|}]; + + let rule = {| where :[1] == :[2], :[3] != "x" |} in + rule_parses rule |> print_string; + [%expect_exact {|true|}]; + + let rule = {| where :[1] != :[3] |} in + rule_parses rule |> print_string; + [%expect_exact {|true|}] + +let%expect_test "parse_basic" = + Rule.create {|where "a" == "a"|} + |> Or_error.ok_exn + |> fun rule -> print_s [%message (rule : Ast.expression list)]; + [%expect_exact {|(rule ((Equal (String a) (String a)))) +|}] + +let%expect_test "parse_match_one_case" = + Rule.create {|where match "match_me" { | "case_one" -> true }|} + |> Or_error.ok_exn + |> fun rule -> print_s [%message (rule : Ast.expression list)]; + [%expect_exact "(rule ((Match (String match_me) (((String case_one) (True)))))) +"] + +let%expect_test "parse_match_multi_case" = + Rule.create + {| where + match "match_me" { + | "case_one" -> true + | "case_two" -> false + } + |} + |> Or_error.ok_exn + |> fun rule -> print_s [%message (rule : Ast.expression list)]; + [%expect_exact "(rule + ((Match (String match_me) + (((String case_one) (True)) ((String case_two) (False)))))) +"] + +let sat ?(env = Environment.create ()) rule = + let rule = Rule.create rule |> Or_error.ok_exn in + Format.sprintf "%b" (Rule.(sat @@ apply rule env)) + +let make_env bindings = + List.fold bindings + ~init:(Environment.create ()) + ~f:(fun env (var, value) -> Environment.add env var value) + +let%expect_test "rule_sat" = + let rule = {| where "x" != "y" |} in + sat rule |> print_string; + [%expect_exact {|true|}]; + + let rule = {| where "x" != "x" |} in + sat rule |> print_string; + [%expect_exact {|false|}]; + + let rule = {| where "x" == "x" |} in + sat rule |> print_string; + [%expect_exact {|true|}]; + + let rule = {| where "x" == "y" |} in + sat rule |> print_string; + [%expect_exact {|false|}]; + + let rule = {| where :[x] == "y" |} in + sat rule |> print_string; + [%expect_exact {|false|}]; + + let rule = {| where :[x] == :[x] |} in + sat rule |> print_string; + [%expect_exact {|false|}] + +let%expect_test "rule_sat_with_env" = + let env = make_env ["1", "x"; "2", "y"; "3", "x"] in + + let rule = {| where :[1] == :[3], :[1] != :[2] |} in + sat ~env rule |> print_string; + [%expect_exact {|true|}]; + + let rule = {| where :[1] == :[3], :[1] != "y" |} in + sat ~env rule |> print_string; + [%expect_exact {|true|}]; + + let rule = {| where :[1] == :[3], :[1] == "x" |} in + sat ~env rule |> print_string; + [%expect_exact {|true|}]; + + let rule = {| where :[1] == :[2], :[1] != :[2] |} in + sat ~env rule |> print_string; + [%expect_exact {|false|}] + +let configuration = Configuration.create ~match_kind:Fuzzy () + +let format s = + let s = s |> String.chop_prefix_exn ~prefix:"\n" in + let leading_indentation = + Option.value_exn (String.lfindi s ~f:(fun _ c -> c <> ' ')) in + s + |> String.split ~on:'\n' + |> List.map ~f:(Fn.flip String.drop_prefix leading_indentation) + |> String.concat ~sep:"\n" + |> String.chop_suffix_exn ~suffix:"\n" + + +let%expect_test "where_true" = + let template = + {| + (:[1]) => {} + |} + |> format + in + + let source = + {| + (b,c) => {} + |} + |> format + in + + let rule = + {| where true + |} + |> Rule.create + |> Or_error.ok_exn + in + + Generic.all ~configuration ~template ~source + |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment)) + |> print_matches; + [%expect {| + [ + { + "range": { + "start": { "offset": 0, "line": 1, "column": 1 }, + "end": { "offset": 11, "line": 1, "column": 12 } + }, + "environment": [ + [ + "1", + { + "value": "b,c", + "range": { + "start": { "offset": 1, "line": 1, "column": 2 }, + "end": { "offset": 4, "line": 1, "column": 5 } + } + } + ] + ], + "matched": "(b,c) => {}" + } + ] |}] + +let%expect_test "match_sat" = + let template = + {| + (:[1]) => {} + |} + |> format + in + + let source = + {| + (b,c) => {} + |} + |> format + in + + let rule = + {| where + match :[1] { + | ":[_],:[_]" -> false + } + |} + |> Rule.create + |> Or_error.ok_exn + in + + Generic.all ~configuration ~template ~source + |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment)) + |> print_matches; + [%expect {| + [] |}]; + + let rule = + {| where + match :[1] { + | ":[_],:[_]" -> true + } + |} + |> Rule.create + |> Or_error.ok_exn + in + + Generic.all ~configuration ~template ~source + |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment)) + |> print_matches; + [%expect {| + [ + { + "range": { + "start": { "offset": 0, "line": 1, "column": 1 }, + "end": { "offset": 11, "line": 1, "column": 12 } + }, + "environment": [ + [ + "1", + { + "value": "b,c", + "range": { + "start": { "offset": 1, "line": 1, "column": 2 }, + "end": { "offset": 4, "line": 1, "column": 5 } + } + } + ] + ], + "matched": "(b,c) => {}" + } + ] |}]; + + let source = + {| + (a) => {} + (b,c) => {} + |} + |> format + in + + let rule = + {| where + match :[1] { + | ":[_],:[_]" -> false + | ":[_]" -> true + } + |} + |> Rule.create + |> Or_error.ok_exn + in + + Generic.all ~configuration ~template ~source + |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment)) + |> print_matches; + [%expect {| + [ + { + "range": { + "start": { "offset": 0, "line": 1, "column": 1 }, + "end": { "offset": 9, "line": 1, "column": 10 } + }, + "environment": [ + [ + "1", + { + "value": "a", + "range": { + "start": { "offset": 1, "line": 1, "column": 2 }, + "end": { "offset": 2, "line": 1, "column": 3 } + } + } + ] + ], + "matched": "(a) => {}" + } + ] |}]; + + let rule = + {| + where + match :[1] { + | ":[_],:[_]" -> false + | ":[_]" -> :[1] == "a" + } + |} + |> Rule.create + |> Or_error.ok_exn + in + + Generic.all ~configuration ~template ~source + |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment)) + |> print_matches; + [%expect {| + [ + { + "range": { + "start": { "offset": 0, "line": 1, "column": 1 }, + "end": { "offset": 9, "line": 1, "column": 10 } + }, + "environment": [ + [ + "1", + { + "value": "a", + "range": { + "start": { "offset": 1, "line": 1, "column": 2 }, + "end": { "offset": 2, "line": 1, "column": 3 } + } + } + ] + ], + "matched": "(a) => {}" + } + ] |}]; + + let rule = + {| where + match :[1] { + | ":[_],:[_]" -> false + | ":[_]" -> :[1] == "b" + } + |} + |> Rule.create + |> Or_error.ok_exn + in + + Generic.all ~configuration ~template ~source + |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment)) + |> print_matches; + [%expect {| + [] |}] + + +let%expect_test "match_s_suffix" = + let template = ":[1]s" in + + let source = "names" in + + let rule = + {| where true + |} + |> Rule.create + |> Or_error.ok_exn + in + + Generic.all ~configuration ~template ~source + |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment)) + |> print_matches; + [%expect {| + [ + { + "range": { + "start": { "offset": 0, "line": 1, "column": 1 }, + "end": { "offset": 5, "line": 1, "column": 6 } + }, + "environment": [ + [ + "1", + { + "value": "name", + "range": { + "start": { "offset": 0, "line": 1, "column": 1 }, + "end": { "offset": 4, "line": 1, "column": 5 } + } + } + ] + ], + "matched": "names" + } + ] |}] + +let%expect_test "match_s_suffix" = + let template = ":[1]" in + + let source = "names" in + + let rule = + {| where true + |} + |> Rule.create + |> Or_error.ok_exn + in + + Generic.all ~configuration ~template ~source + |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment)) + |> print_matches; + [%expect {| + [ + { + "range": { + "start": { "offset": 0, "line": 1, "column": 1 }, + "end": { "offset": 5, "line": 1, "column": 6 } + }, + "environment": [ + [ + "1", + { + "value": "names", + "range": { + "start": { "offset": 0, "line": 1, "column": 1 }, + "end": { "offset": 5, "line": 1, "column": 6 } + } + } + ] + ], + "matched": "names" + } + ] |}] + +let%expect_test "configuration_choice_based_on_case" = + let template = ":[1]" in + + let source = "names" in + + let rule = + {| where match :[1] { + | "ame" -> true + } + |} + |> Rule.create + |> Or_error.ok_exn + in + + Generic.all ~configuration ~template ~source + |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment)) + |> print_matches; + [%expect {| + [] |}]; + + let template = ":[1]" in + + let source = "names" in + + let rule = + {| where match :[1] { + | "names" -> true + } + |} + |> Rule.create + |> Or_error.ok_exn + in + + Generic.all ~configuration ~template ~source + |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment)) + |> print_matches; + [%expect {| + [ + { + "range": { + "start": { "offset": 0, "line": 1, "column": 1 }, + "end": { "offset": 5, "line": 1, "column": 6 } + }, + "environment": [ + [ + "1", + { + "value": "names", + "range": { + "start": { "offset": 0, "line": 1, "column": 1 }, + "end": { "offset": 5, "line": 1, "column": 6 } + } + } + ] + ], + "matched": "names" + } + ] |}]; + + let template = ":[1]" in + + let source = "namesXXXXX" in + + let rule = + {| where match :[1] { + | "names" -> true + } + |} + |> Rule.create + |> Or_error.ok_exn + in + + Generic.all ~configuration ~template ~source + |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment)) + |> print_matches; + [%expect {| + [] |}] + + + + +let%expect_test "match_using_environment_merge" = + let template = "{:[1]}" in + + let source = "{{ a : a } { a : a }}" in + + let rule = + {| where match :[1] { | "{ :[x] : :[y] }" -> :[x] == :[y] } + |} + |> Rule.create + |> Or_error.ok_exn + in + + Generic.all ~configuration ~template ~source + |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment)) + |> print_matches; + [%expect {| + [ + { + "range": { + "start": { "offset": 0, "line": 1, "column": 1 }, + "end": { "offset": 21, "line": 1, "column": 22 } + }, + "environment": [ + [ + "1", + { + "value": "{ a : a } { a : a }", + "range": { + "start": { "offset": 1, "line": 1, "column": 2 }, + "end": { "offset": 20, "line": 1, "column": 21 } + } + } + ] + ], + "matched": "{{ a : a } { a : a }}" + } + ] |}]; + + let template = "{:[1]}" in + + let source = "{{ a : a } { a : b }}" in + + let rule = + {| where match :[1] { | "{ :[x] : :[y] }" -> :[x] == :[y] } + |} + |> Rule.create + |> Or_error.ok_exn + in + + Generic.all ~configuration ~template ~source + |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment)) + |> print_matches; + [%expect {| + [] |}] + + +let%expect_test "nested_matches" = + let template = ":[1]" in + + let source = "{ { foo : { bar : { baz : qux } } } }" in + + let rule = + {| where match :[1] { + | "{ :[foo] : :[tail1] }" -> match :[tail1] { + | "{ :[bar] : :[tail2] }" -> match :[tail2] { + | "{ baz : :[qux] }" -> :[qux] == "qux", :[bar] == "bar" + } + } + } + |} + |> Rule.create + |> Or_error.ok_exn + in + + Generic.all ~configuration ~template ~source + |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment)) + |> print_matches; + [%expect {| + [ + { + "range": { + "start": { "offset": 0, "line": 1, "column": 1 }, + "end": { "offset": 37, "line": 1, "column": 38 } + }, + "environment": [ + [ + "1", + { + "value": "{ { foo : { bar : { baz : qux } } } }", + "range": { + "start": { "offset": 0, "line": 1, "column": 1 }, + "end": { "offset": 37, "line": 1, "column": 38 } + } + } + ] + ], + "matched": "{ { foo : { bar : { baz : qux } } } }" + } + ] |}]; + + let template = ":[1]" in + + let source = "{ { foo : { bar : { baz : qux } } } }" in + + let rule = + {| where match :[1] { + | "{ :[foo] : :[tail1] }" -> match :[tail1] { + | "{ :[bar] : :[tail2] }" -> match :[tail2] { + | "{ baz : :[qux] }" -> :[qux] == "fail" + } + } + } + |} + |> Rule.create + |> Or_error.ok_exn + in + + Generic.all ~configuration ~template ~source + |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment)) + |> print_matches; + [%expect {| + [] |}] + + +let%expect_test "match_on_template" = + let template = ":[1]" in + + let source = "oodles" in + + let rule = + {| where match "p:[1]" { + | "poodles" -> true + } + |} + |> Rule.create + |> Or_error.ok_exn + in + + Generic.all ~configuration ~template ~source + |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment)) + |> print_matches; + [%expect {| + [ + { + "range": { + "start": { "offset": 0, "line": 1, "column": 1 }, + "end": { "offset": 6, "line": 1, "column": 7 } + }, + "environment": [ + [ + "1", + { + "value": "oodles", + "range": { + "start": { "offset": 0, "line": 1, "column": 1 }, + "end": { "offset": 6, "line": 1, "column": 7 } + } + } + ] + ], + "matched": "oodles" + } + ] |}]; + + let template = ":[1]" in + + let source = "poodle" in + + let rule = + {| where match ":[1]s" { + | "poodles" -> true + } + |} + |> Rule.create + |> Or_error.ok_exn + in + + Generic.all ~configuration ~template ~source + |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment)) + |> print_matches; + [%expect {| + [ + { + "range": { + "start": { "offset": 0, "line": 1, "column": 1 }, + "end": { "offset": 6, "line": 1, "column": 7 } + }, + "environment": [ + [ + "1", + { + "value": "poodle", + "range": { + "start": { "offset": 0, "line": 1, "column": 1 }, + "end": { "offset": 6, "line": 1, "column": 7 } + } + } + ] + ], + "matched": "poodle" + } + ] |}]; + + let template = ":[1]" in + + let source = "poodle" in + + let rule = + {| where match ":[1]," { + | "poodle" -> true + } + |} + |> Rule.create + |> Or_error.ok_exn + in + + Generic.all ~configuration ~template ~source + |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment)) + |> print_matches; + [%expect {| + [] |}]; diff --git a/test/test_rewrite_parts.ml b/test/test_rewrite_parts.ml new file mode 100644 index 0000000..c6bb47b --- /dev/null +++ b/test/test_rewrite_parts.ml @@ -0,0 +1,223 @@ +open Core + +open Match +open Matchers +open Rewriter + +let%expect_test "get_offsets_for_holes" = + let rewrite_template = {|1234:[1]1234:[2]|} in + let result = Rewrite_template.get_offsets_for_holes rewrite_template ["1"; "2"] in + print_s [%message (result : (string * int) list)]; + [%expect_exact {|(result ((2 8) (1 4))) +|}] + +let%expect_test "get_offsets_for_holes_after_substitution_1" = + let rewrite_template = {|1234:[1]1234:[2]|} in + let offsets = Rewrite_template.get_offsets_for_holes rewrite_template ["1"; "2"] in + let environment = + Environment.create () + |> (fun environment -> Environment.add environment "1" "333") + |> (fun environment -> Environment.add environment "2" "22") + in + let result = Rewrite_template.get_offsets_after_substitution offsets environment in + print_s [%message (result : (string * int) list)]; + [%expect_exact {|(result ((2 11) (1 4))) +|}] + +let%expect_test "get_offsets_for_holes_after_substitution_1" = + let rewrite_template = {|1234:[1]1234:[3]11:[2]|} in + let offsets = Rewrite_template.get_offsets_for_holes rewrite_template ["1"; "3"; "2"] in + let environment = + Environment.create () + |> (fun environment -> Environment.add environment "1" "333") + |> (fun environment -> Environment.add environment "3" "333") + |> (fun environment -> Environment.add environment "2" "22") + in + let result = Rewrite_template.get_offsets_after_substitution offsets environment in + print_s [%message (result : (string * int) list)]; + [%expect_exact {|(result ((2 16) (3 11) (1 4))) +|}] + + +let configuration = Configuration.create ~match_kind:Fuzzy () + +let all ?(configuration = configuration) template source = + C.all ~configuration ~template ~source + +let%expect_test "comments_in_string_literals_should_not_be_treated_as_comments_by_fuzzy" = + let source = {|123433312343331122|} in + let match_template = {|1234:[1]1234:[3]11:[2]|} in + let rewrite_template = {|1234:[1]1234:[3]11:[2]|} in + all match_template source + |> Rewrite.all ~source ~rewrite_template + |> (function + | Some rewrite_result -> print_string (Yojson.Safe.pretty_to_string (Rewrite.result_to_yojson rewrite_result)) + | None -> print_string "BROKEN EXPECT"); + [%expect_exact {|{ + "rewritten_source": "123433312343331122", + "contextual_substitutions": [ + { + "range": { + "start": { "offset": 0, "line": -1, "column": -1 }, + "end": { "offset": 18, "line": -1, "column": -1 } + }, + "replacement_content": "123433312343331122", + "environment": [ + [ + "1", + { + "value": "333", + "range": { + "start": { "offset": 4, "line": -1, "column": -1 }, + "end": { "offset": 7, "line": -1, "column": -1 } + } + } + ], + [ + "2", + { + "value": "22", + "range": { + "start": { "offset": 16, "line": -1, "column": -1 }, + "end": { "offset": 18, "line": -1, "column": -1 } + } + } + ], + [ + "3", + { + "value": "333", + "range": { + "start": { "offset": 11, "line": -1, "column": -1 }, + "end": { "offset": 14, "line": -1, "column": -1 } + } + } + ] + ] + } + ] +}|}] + +let%expect_test "comments_in_string_literals_should_not_be_treated_as_comments_by_fuzzy" = + let source = {|123433312343331122;123433312343331122;|} in + let match_template = {|1234:[1]1234:[3]11:[2];|} in + let rewrite_template = {|1234:[1]1234:[3]11:[2];|} in + all match_template source + |> Rewrite.all ~source ~rewrite_template + |> (function + | Some rewrite_result -> print_string (Yojson.Safe.pretty_to_string (Rewrite.result_to_yojson rewrite_result)) + | None -> print_string "BROKEN EXPECT"); + [%expect_exact {|{ + "rewritten_source": "123433312343331122;123433312343331122;", + "contextual_substitutions": [ + { + "range": { + "start": { "offset": 19, "line": -1, "column": -1 }, + "end": { "offset": 38, "line": -1, "column": -1 } + }, + "replacement_content": "123433312343331122;", + "environment": [ + [ + "1", + { + "value": "333", + "range": { + "start": { "offset": 4, "line": -1, "column": -1 }, + "end": { "offset": 7, "line": -1, "column": -1 } + } + } + ], + [ + "2", + { + "value": "22", + "range": { + "start": { "offset": 16, "line": -1, "column": -1 }, + "end": { "offset": 18, "line": -1, "column": -1 } + } + } + ], + [ + "3", + { + "value": "333", + "range": { + "start": { "offset": 11, "line": -1, "column": -1 }, + "end": { "offset": 14, "line": -1, "column": -1 } + } + } + ] + ] + }, + { + "range": { + "start": { "offset": 0, "line": -1, "column": -1 }, + "end": { "offset": 19, "line": -1, "column": -1 } + }, + "replacement_content": "123433312343331122;", + "environment": [ + [ + "1", + { + "value": "333", + "range": { + "start": { "offset": 4, "line": -1, "column": -1 }, + "end": { "offset": 7, "line": -1, "column": -1 } + } + } + ], + [ + "2", + { + "value": "22", + "range": { + "start": { "offset": 16, "line": -1, "column": -1 }, + "end": { "offset": 18, "line": -1, "column": -1 } + } + } + ], + [ + "3", + { + "value": "333", + "range": { + "start": { "offset": 11, "line": -1, "column": -1 }, + "end": { "offset": 14, "line": -1, "column": -1 } + } + } + ] + ] + } + ] +}|}] + +let%expect_test "multiple_contextual_substitutions" = + let source = {|foo bar foo|} in + let match_template = {|foo|} in + let rewrite_template = {|xxxx|} in + all match_template source + |> Rewrite.all ~source ~rewrite_template + |> (function + | Some rewrite_result -> print_string (Yojson.Safe.pretty_to_string (Rewrite.result_to_yojson rewrite_result)) + | None -> print_string "BROKEN EXPECT"); + [%expect_exact {|{ + "rewritten_source": "xxxx bar xxxx", + "contextual_substitutions": [ + { + "range": { + "start": { "offset": 9, "line": -1, "column": -1 }, + "end": { "offset": 13, "line": -1, "column": -1 } + }, + "replacement_content": "xxxx", + "environment": [] + }, + { + "range": { + "start": { "offset": 0, "line": -1, "column": -1 }, + "end": { "offset": 4, "line": -1, "column": -1 } + }, + "replacement_content": "xxxx", + "environment": [] + } + ] +}|}] diff --git a/test/test_rewrite_rule.ml b/test/test_rewrite_rule.ml new file mode 100644 index 0000000..e4e90b8 --- /dev/null +++ b/test/test_rewrite_rule.ml @@ -0,0 +1,169 @@ +open Core + +open Language +open Matchers +open Match +open Rewriter + +let configuration = Configuration.create ~match_kind:Fuzzy () + +let format s = + let s = String.chop_prefix_exn ~prefix:"\n" s in + let leading_indentation = Option.value_exn (String.lfindi s ~f:(fun _ c -> c <> ' ')) in + s + |> String.split ~on:'\n' + |> List.map ~f:(Fn.flip String.drop_prefix leading_indentation) + |> String.concat ~sep:"\n" + |> String.chop_suffix_exn ~suffix:"\n" + +let run_rule source match_template rewrite_template rule = + Generic.first ~configuration match_template source + |> function + | Error _ -> print_string "bad" + | Ok result -> + match result with + | ({ environment; _ } as m) -> + let e = Rule.(result_env @@ apply rule environment) in + match e with + | None -> print_string "bad bad" + | Some e -> + { m with environment = e } + |> List.return + |> Rewrite.all ~source ~rewrite_template + |> (fun x -> Option.value_exn x) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string + +let%expect_test "rewrite_rule" = + let source = {|int|} in + let match_template = {|:[1]|} in + let rewrite_template = {|:[1]|} in + + let rule = + {| + where rewrite :[1] { + | "int" -> "expect" + } + |} + |> Rule.create + |> Or_error.ok_exn + in + + run_rule source match_template rewrite_template rule; + [%expect_exact {|expect|}] + +let%expect_test "rewrite_rule" = + let source = {|string expect|} in + let match_template = {|:[1] :[2]|} in + let rewrite_template = {|:[2]|} in + + let rule = + {| + where rewrite :[1] { + | "int" -> "5" + | "string" -> ":[2]" + } + |} + |> Rule.create + |> Or_error.ok_exn + in + + run_rule source match_template rewrite_template rule; + [%expect_exact {|expect|}] + +let%expect_test "conditional_rewrite_rule" = + let source = {|{ { a : { b : { c : d } } } }|} in + let match_template = {|:[1]|} in + let rewrite_template = {|:[1]|} in + + let rule = + {| + where rewrite :[1] { + | "{ :[a] : :[rest] }" -> "a" == :[a], "doot" + } + |} + |> Rule.create + |> Or_error.ok_exn + in + + run_rule source match_template rewrite_template rule; + [%expect_exact {|doot|}] + +let%expect_test "rewrite_rule_using_match_result" = + let source = {|{ { a : { b : { c : d } } } }|} in + let match_template = {|:[1]|} in + let rewrite_template = {|:[1]|} in + + let rule = + {| + where rewrite :[1] { + | "{ :[a] : :[rest] }" -> "a" == :[a], ":[rest]" + } + |} + |> Rule.create + |> Or_error.ok_exn + in + + run_rule source match_template rewrite_template rule; + [%expect_exact {|{ b : { c : d } }|}]; + + let source = {|{ { a : { b : { c : d } } } }|} in + let match_template = {|:[1]|} in + let rewrite_template = {|:[1]|} in + let rule = + {| + where rewrite :[1] { + | "{ :[a] : :[rest] }" -> "b" == :[a], ":[rest]" + } + |} + |> Rule.create + |> Or_error.ok_exn + in + + run_rule source match_template rewrite_template rule; + [%expect_exact {|{ { a : { b : { c : d } } } }|}] + + +let%expect_test "nested_rewrite_rule" = + let source = {|{ { a : { b : { c : d } } } }|} in + let match_template = {|:[1]|} in + let rewrite_template = {|:[1]|} in + + let rule = + {| + where + rewrite :[1] { + | "{ :[a] : :[rest] }" -> + rewrite :[a] { + | "a" -> "b" + }, "{ :[a] : :[rest] }" + } + |} + |> Rule.create + |> Or_error.ok_exn + in + + run_rule source match_template rewrite_template rule; + [%expect_exact {|{ b : { b : { c : d } } }|}] + +let%expect_test "sequenced_rewrite_rule" = + let source = {|{ { a : { b : { c : d } } } }|} in + let match_template = {|{ :[a] : :[rest] }|} in + let rewrite_template = {|{ :[a] : :[rest] }|} in + + let rule = + {| + where + rewrite :[a] { + | "a" -> "qqq" + }, + rewrite :[rest] { + | "{ b : { :[other] } }" -> "{ :[other] }" + } + |} + |> Rule.create + |> Or_error.ok_exn + in + + run_rule source match_template rewrite_template rule; + [%expect_exact {|{ { qqq : { c : d } } }|}] diff --git a/test/test_statistics.ml b/test/test_statistics.ml new file mode 100644 index 0000000..7b1daae --- /dev/null +++ b/test/test_statistics.ml @@ -0,0 +1,87 @@ +open Core + +open Language +open Matchers +open Match + +let configuration = Configuration.create ~match_kind:Fuzzy () + +let format s = + let s = s |> String.chop_prefix_exn ~prefix:"\n" in + let leading_indentation = + Option.value_exn (String.lfindi s ~f:(fun _ c -> c <> ' ')) in + s + |> String.split ~on:'\n' + |> List.map ~f:(Fn.flip String.drop_prefix leading_indentation) + |> String.concat ~sep:"\n" + |> String.chop_suffix_exn ~suffix:"\n" + +let %expect_test "statistics" = + let template = + {| + def :[fn_name](:[fn_params]) + |} + |> format + in + + let source = + {| + def foo(bar): + pass + + def bar(bazz): + pass + |} + |> format + in + + let rule = + {| where true + |} + |> Rule.create + |> Or_error.ok_exn + in + Go.all ~configuration ~template ~source + |> List.filter ~f:(fun { environment; _ } -> + Rule.(sat @@ apply rule environment)) + |> fun matches -> + let statistics = + Statistics. + { number_of_files = 1 + ; lines_of_code = 5 + ; number_of_matches = List.length matches + ; total_time = 0.0 + } + in + statistics + |> Statistics.to_yojson + |> Yojson.Safe.pretty_to_string + |> print_string; + [%expect {| + { + "number_of_files": 1, + "lines_of_code": 5, + "number_of_matches": 2, + "total_time": 0.0 + } |}]; + + let statistics' = + Statistics.merge + { number_of_files = 1 + ; lines_of_code = 10 + ; number_of_matches = 1 + ; total_time = 1.5 + } + statistics + in + statistics' + |> Statistics.to_yojson + |> Yojson.Safe.pretty_to_string + |> print_string; + [%expect {| + { + "number_of_files": 2, + "lines_of_code": 15, + "number_of_matches": 3, + "total_time": 1.5 + } |}] diff --git a/test/test_string_literals.ml b/test/test_string_literals.ml new file mode 100644 index 0000000..d79f434 --- /dev/null +++ b/test/test_string_literals.ml @@ -0,0 +1,412 @@ +open Core + +open Matchers +open Rewriter + +let format s = + let s = String.chop_prefix_exn ~prefix:"\n" s in + let leading_indentation = Option.value_exn (String.lfindi s ~f:(fun _ c -> c <> ' ')) in + s + |> String.split ~on:'\n' + |> List.map ~f:(Fn.flip String.drop_prefix leading_indentation) + |> String.concat ~sep:"\n" + |> String.chop_suffix_exn ~suffix:"\n" + +let configuration = Configuration.create ~match_kind:Fuzzy () + +let all ?(configuration = configuration) template source = + C.all ~configuration ~template ~source + +let print_matches matches = + List.map matches ~f:Match.to_yojson + |> (fun matches -> `List matches) + |> Yojson.Safe.pretty_to_string + |> print_string + +let%expect_test "comments_in_string_literals_should_not_be_treated_as_comments_by_fuzzy" = + let source = {|"/*"(x)|} in + let template = {|(:[1])|} in + let rewrite_template = {|:[1]|} in + all template source + |> Rewrite.all ~source ~rewrite_template + |> (function + | Some { rewritten_source; _ } -> print_string rewritten_source + | None -> print_string "BROKEN EXPECT"); + [%expect_exact {|"/*"x|}] + +let%expect_test "comments_in_string_literals_should_not_be_treated_as_comments_by_fuzzy_go_raw" = + let source = {|`//`(x)|} in + let template = {|(:[1])|} in + let rewrite_template = {|:[1]|} in + Go.all ~configuration ~template ~source + |> Rewrite.all ~source ~rewrite_template + |> (function + | Some { rewritten_source; _ } -> print_string rewritten_source + | None -> print_string "BROKEN EXPECT"); + [%expect_exact {|`//`x|}] + +let%expect_test "tolerate_unbalanced_stuff_in_string_literals" = + let template = {|"("|} in + let source = {|"("|} in + let matches = all ~configuration template source in + print_matches matches; + [%expect_exact {|[ + { + "range": { + "start": { "offset": 0, "line": 1, "column": 1 }, + "end": { "offset": 3, "line": 1, "column": 4 } + }, + "environment": [], + "matched": "\"(\"" + } +]|}] + +let%expect_test "base_literal_matching" = + let source = {|"hello"|} in + let match_template = {|":[1]"|} in + let rewrite_template = {|:[1]|} in + all match_template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact {|hello|}] + +let%expect_test "base_literal_matching" = + let source = {|rewrite ("hello") this string|} in + let match_template = {|rewrite (":[1]") this string|} in + let rewrite_template = {|:[1]|} in + all match_template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact {|hello|}] + +let%expect_test "match_string_literals" = + let source = {|rewrite (".") this string|} in + let match_template = {|rewrite (":[1]") this string|} in + let rewrite_template = {|:[1]|} in + all match_template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact {|.|}] + +let%expect_test "match_string_literals" = + let source = {|rewrite ("") this string|} in + let match_template = {|rewrite (":[1]") this string|} in + let rewrite_template = {|:[1]|} in + all match_template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact {||}] + +let%expect_test "match_string_literals" = + let source = {|"(" match "a""a" this "(" |} in + let match_template = {|match :[1] this|} in + let rewrite_template = {|:[1]|} in + all match_template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact {|"(" "a""a" "(" |}] + +(* this tests special functionality in non-literal hole parser + but which must still ignore unbalanced delims within strings *) +let%expect_test "match_string_literals" = + let source = {|"(" match "(""(" this "(" |} in + let match_template = {|match :[1] this|} in + let rewrite_template = {|:[1]|} in + all match_template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact {|"(" "(""(" "(" |}] + +let%expect_test "match_string_literals" = + let source = {|rewrite ("") this string|} in + let match_template = {|rewrite (:[1]) this string|} in + let rewrite_template = {|:[1]|} in + all match_template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact {|""|}] + +let%expect_test "base_literal_matching" = + let source = {|"("|} in + let match_template = {|":[1]"|} in + let rewrite_template = {|:[1]|} in + all match_template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact {|(|}] + +let%expect_test "base_literal_matching" = + let source = {|"(""("|} in + let match_template = {|":[1]"|} in + let rewrite_template = {|:[1]|} in + all match_template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact {|((|}] + +let%expect_test "base_literal_matching" = + let source = {|"(""("|} in + let match_template = {|":[1]"|} in + let rewrite_template = {|:[1]|} in + all match_template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact {|((|}] + +let%expect_test "base_literal_matching" = + let source = {|"hello world"|} in + let match_template = {|":[x] :[y]"|} in + let rewrite_template = {|:[x] :[y]|} in + all match_template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact {|hello world|}] + +(* complex test: basically, we are checking that the inside of this literal is only matched by the val b part *) +let%expect_test "base_literal_matching" = + let source = {|val a = "class = ${String::class}" val b = "not $a"|} in + let match_template = {|":[x]$:[[y]]"|} in + let rewrite_template = {|(rewritten part: (:[x]) ([y]))|} in + all match_template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact {|val a = "class = ${String::class}" val b = (rewritten part: (not ) ([y]))|}] + +let%expect_test "base_literal_matching" = + let source = {|get("type") rekt ("enabled", True)|} in + let match_template = {|(":[1]", :[[3]])|} in + let rewrite_template = {|(rewritten part: (:[1]) (:[3]))|} in + all match_template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact {|get("type") rekt (rewritten part: (enabled) (True))|}] + +let%expect_test "rewrite_string_literals_8" = + let source = {|match "\"" this|} in + let match_template = {|match "\"" this|} in + let rewrite_template = "" in + + all match_template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact {||}] + +let%expect_test "rewrite_string_literals_8" = + let source = {|match "\"" this|} in + let match_template = {|match :[1] this|} in + let rewrite_template = ":[1]" in + + all match_template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact {|"\""|}] + +let%expect_test "rewrite_string_literals_8" = + let source = {|match "\"\"" this|} in + let match_template = {|match :[1] this|} in + let rewrite_template = ":[1]" in + + all match_template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact {|"\"\""|}] + +let%expect_test "rewrite_string_literals_8" = + let source = {|match "\"(\"" "(\"" this|} in + let match_template = {|match :[1] this|} in + let rewrite_template = ":[1]" in + + all match_template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact {|"\"(\"" "(\""|}] + +let%expect_test "rewrite_string_literals_8" = + let source = {|match "\"(\"" "(\"" this|} in + let match_template = {|match ":[1]" ":[2]" this|} in + let rewrite_template = {|:[1] :[2]|} in + + all match_template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact {|\"(\" (\"|}] + +let%expect_test "rewrite_string_literals_8" = + let source = {|match 'sin(gle' 'quo(tes' this|} in + let match_template = {|:[1]|} in + let rewrite_template = {|:[1]|} in + + all match_template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact {|match 'sin(gle' 'quo(tes' this|}] + +let%expect_test "rewrite_string_literals_8" = + let source = {|match '\''|} in + let match_template = {|:[1]|} in + let rewrite_template = {|:[1]|} in + + all match_template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact {|match '\''|}] + +let%expect_test "rewrite_string_literals_8" = + let source = {|match 'asdf'|} in + let match_template = {|':[1]'|} in + let rewrite_template = {|:[1]|} in + + all match_template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact {|match asdf|}] + +let%expect_test "rewrite_string_literals_8" = + let source = {|match '\''|} in + let match_template = {|':[1]'|} in + let rewrite_template = {|:[1]|} in + + all match_template source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact {|match \'|}] + + +let%expect_test "go_raw_string_literals" = + let source = + {| + x = x + y = `multi-line + raw str(ing literal` + z = `other multi-line + raw stri(ng literal` + |} + in + let match_template = {|`:[1]`|} in + let rewrite_template = {|:[1]|} in + + Go.all ~configuration ~source ~template:match_template + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact {| + x = x + y = multi-line + raw str(ing literal + z = other multi-line + raw stri(ng literal + |}] + +let%expect_test "go_raw_string_literals" = + let source = {|blah `(` quux|} in + let match_template = {|:[1]|} in + let rewrite_template = {|:[1]|} in + + Go.all ~configuration ~source ~template:match_template + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact {|blah `(` quux|}] + +let%expect_test "match_string_literals" = + let source = {|`(` match `(``(` this `(` |} in + let match_template = {|match :[1] this|} in + let rewrite_template = {|:[1]|} in + Go.all ~configuration ~template:match_template ~source + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact {|`(` `(``(` `(` |}] + + +let%expect_test "go_raw_string_literals" = + let source = + {| + x = x + y = `multi-line + raw "str"(ing literal` + z = `other multi-line + raw '"'\"\\s\\\\\tr\ni(ng literal` + |} + in + let match_template = {|`:[1]`|} in + let rewrite_template = {|:[1]|} in + + Go.all ~configuration ~source ~template:match_template + |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) + |> (fun { rewritten_source; _ } -> rewritten_source) + |> print_string; + [%expect_exact {| + x = x + y = multi-line + raw "str"(ing literal + z = other multi-line + raw '"'\"\\s\\\\\tr\ni(ng literal + |}] + +let%expect_test "regression_matching_kubernetes" = + let source = {|"\n" y = 5|} in + let template = {|y = :[1]|} in + let rewrite_template = {|:[1]|} in + Go.all ~configuration ~template ~source + |> Rewrite.all ~source ~rewrite_template + |> (function + | Some { rewritten_source; _ } -> print_string rewritten_source + | None -> print_string "BROKEN EXPECT"); + [%expect_exact {|"\n" 5|}] + + +let%expect_test "match_escaped_any_char" = + let source = {|printf("hello world\n");|} in + let template = {|printf(":[1]");|} in + let rewrite_template = {|:[1]|} in + Go.all ~configuration ~template ~source + |> Rewrite.all ~source ~rewrite_template + |> (function + | Some { rewritten_source; _ } -> print_string rewritten_source + | None -> print_string "BROKEN EXPECT"); + [%expect_exact {|hello world\n|}] + +let%expect_test "match_escaped_escaped" = + let source = {|printf("hello world\n\\");|} in + let template = {|printf(":[1]");|} in + let rewrite_template = {|:[1]|} in + Go.all ~configuration ~template ~source + |> Rewrite.all ~source ~rewrite_template + |> (function + | Some { rewritten_source; _ } -> print_string rewritten_source + | None -> print_string "BROKEN EXPECT"); + [%expect_exact {|hello world\n\\|}] + +let%expect_test "match_escaped_escaped" = + let source = {|printf("hello world\n\");|} in + let template = {|printf(":[1]");|} in + let rewrite_template = {|:[1]|} in + Go.all ~configuration ~template ~source + |> Rewrite.all ~source ~rewrite_template + |> (function + | Some { rewritten_source; _ } -> print_string rewritten_source + | None -> print_string "EXPECT SUCCESS"); + [%expect_exact {|EXPECT SUCCESS|}]