From f7bf5c7b40e1a599e6003e456c202640f0d1584a Mon Sep 17 00:00:00 2001 From: Rijnard van Tonder Date: Mon, 19 Apr 2021 01:17:42 -0700 Subject: [PATCH] Rewrite template refactor (#277) --- lib/app/comby.mli | 5 +- .../configuration/command_configuration.ml | 6 +- lib/app/configuration/specification.ml | 2 +- lib/app/pipeline/pipeline.ml | 10 +- lib/kernel/comby_kernel.mli | 5 +- lib/kernel/language/alpha_rule.ml | 7 +- lib/kernel/language/omega_rule.ml | 7 +- lib/kernel/language/types.ml | 1 + lib/kernel/matchers/alpha.ml | 2 +- lib/kernel/rewriter/dune | 2 +- lib/kernel/rewriter/rewrite.ml | 44 +++- lib/kernel/rewriter/rewrite_template.ml | 195 +++++++++++++---- lib/kernel/rewriter/rewrite_template.mli | 17 +- src/main.ml | 24 +- test/alpha/test_custom_metasyntax.ml | 36 ++- test/common/dune | 6 +- test/common/test_cli.ml | 46 ++++ test/common/test_extract_regex.ml | 2 +- test/common/test_parse_rewrite_template.ml | 37 ++++ test/common/test_rewrite_parts.ml | 29 ++- test/common/test_rewrite_parts_alpha.ml | 205 ------------------ test/common/test_rewrite_parts_omega.ml | 205 ------------------ test/common/test_substitute.ml | 31 +++ .../metasyntax/default-metasyntax.json | 13 ++ test/example/metasyntax/dolla.json | 8 + 25 files changed, 445 insertions(+), 500 deletions(-) create mode 100644 test/common/test_parse_rewrite_template.ml delete mode 100644 test/common/test_rewrite_parts_alpha.ml delete mode 100644 test/common/test_rewrite_parts_omega.ml create mode 100644 test/common/test_substitute.ml create mode 100644 test/example/metasyntax/default-metasyntax.json create mode 100644 test/example/metasyntax/dolla.json diff --git a/lib/app/comby.mli b/lib/app/comby.mli index b46b0cc..08cd670 100644 --- a/lib/app/comby.mli +++ b/lib/app/comby.mli @@ -465,15 +465,16 @@ module Rule : sig (** [create] parses and creates a rule. *) val create : string -> t Or_error.t - (** [apply matcher substitute_in_place fresh rule env] applies a [rule] + (** [apply matcher substitute_in_place fresh metasyntax rule env] applies a [rule] according to some [matcher] for existing matches in [env]. If [substitute_in_place] is true, rewrite rules substitute their values in place (default true). [fresh] introduces fresh variables for evaluating - rules. *) + rules. [metasyntax] uses the custom metasyntax definition. *) val apply : ?matcher:(module Matchers.Matcher.S) -> ?substitute_in_place:bool -> ?fresh:(unit -> string) + -> ?metasyntax:Matchers.Metasyntax.t -> t -> Match.environment -> result diff --git a/lib/app/configuration/command_configuration.ml b/lib/app/configuration/command_configuration.ml index 3d1e2c0..d806fb0 100644 --- a/lib/app/configuration/command_configuration.ml +++ b/lib/app/configuration/command_configuration.ml @@ -663,20 +663,20 @@ let select_matcher custom_metasyntax custom_matcher override_matcher file_filter let metasyntax = metasyntax custom_metasyntax in let syntax = syntax custom_matcher in E.create ~metasyntax syntax, None, Some metasyntax - (* forced language, optional custom metasyntax *) | _, Some language, custom_metasyntax -> + (* forced language, optional custom metasyntax *) let metasyntax = metasyntax custom_metasyntax in let (module Metasyntax) = Matchers.Metasyntax.create metasyntax in let (module Language) = force_language language in (module (E.Make (Language) (Metasyntax)) : Matchers.Matcher.S), None, Some metasyntax - (* infer language from file filters, definite custom metasyntax *) | _, _, Some custom_metasyntax -> + (* infer language from file filters, definite custom metasyntax *) let metasyntax = metasyntax (Some custom_metasyntax) in let (module Metasyntax) = Matchers.Metasyntax.create metasyntax in let (module Language) = force_language (extension file_filters) in (module (E.Make (Language) (Metasyntax)) : Matchers.Matcher.S), None, Some metasyntax - (* infer language from file filters, use default metasyntax *) | _, _, None -> + (* infer language from file filters, use default metasyntax *) of_extension (module E) file_filters let regex_of_specifications specifications = diff --git a/lib/app/configuration/specification.ml b/lib/app/configuration/specification.ml index f2ced24..5e88b13 100644 --- a/lib/app/configuration/specification.ml +++ b/lib/app/configuration/specification.ml @@ -135,7 +135,7 @@ let to_regex { match_template; _ } = let extracted = List.map extracted ~f:(fun part -> Str.global_replace match_spaces {|\s+|} part) in (* ?s is modifier metasyntax where . matches all chars including newlines. See regular-expressions.info/modifier.html *) - Format.sprintf "(%s)" @@ String.concat extracted ~sep:")(?s:.)*?(" + Format.sprintf "(%s)" @@ String.concat extracted ~sep:")(\\n|.)*?(" let create ?rewrite_template ?rule ~match_template () = { match_template; rule; rewrite_template } diff --git a/lib/app/pipeline/pipeline.ml b/lib/app/pipeline/pipeline.ml index b1cf1ad..4ddbf03 100644 --- a/lib/app/pipeline/pipeline.ml +++ b/lib/app/pipeline/pipeline.ml @@ -27,15 +27,15 @@ let infer_equality_constraints environment = else acc) -let apply_rule ?(substitute_in_place = true) matcher omega rule matches = +let apply_rule ?(substitute_in_place = true) ?metasyntax matcher omega rule matches = let open Option in List.filter_map matches ~f:(fun ({ environment; _ } as matched) -> let rule = rule @ infer_equality_constraints environment in let apply = if omega then - Rule.Omega.apply + Rule.Omega.apply ?metasyntax else - Rule.Alpha.apply + Rule.Alpha.apply ?metasyntax in let fresh () = Uuid_unix.(Fn.compose Uuid.to_string create ()) in let sat, env = apply ~fresh ~substitute_in_place ~matcher rule environment in @@ -47,6 +47,7 @@ let timed_run ?(fast_offset_conversion = false) ?(omega = false) ?substitute_in_place + ?metasyntax ~configuration ~source ~specification:(Specification.{ match_template = template; rule; rewrite_template }) @@ -57,7 +58,7 @@ let timed_run let rule = Option.value rule ~default:[Ast.True] in let options = if omega then Rule.Omega.options rule else Rule.Alpha.options rule in let matches = Matcher.all ~nested:options.nested ~configuration ~template ~source () in - let matches = apply_rule ?substitute_in_place (module Matcher) omega rule matches in + let matches = apply_rule ?substitute_in_place ?metasyntax (module Matcher) omega rule matches in List.map matches ~f:(Match.convert_offset ~fast:fast_offset_conversion ~source) type output = @@ -102,6 +103,7 @@ let process_single_source with_timeout timeout source ~f:(fun () -> timed_run matcher + ?metasyntax ~substitute_in_place ~omega ~fast_offset_conversion diff --git a/lib/kernel/comby_kernel.mli b/lib/kernel/comby_kernel.mli index 426b3e1..68fbd02 100644 --- a/lib/kernel/comby_kernel.mli +++ b/lib/kernel/comby_kernel.mli @@ -465,15 +465,16 @@ module Rule : sig (** [create] parses and creates a rule. *) val create : string -> t Or_error.t - (** [apply matcher substitute_in_place fresh rule env] applies a [rule] + (** [apply matcher substitute_in_place fresh metasyntax rule env] applies a [rule] according to some [matcher] for existing matches in [env]. If [substitute_in_place] is true, rewrite rules substitute their values in place (default true). [fresh] introduces fresh variables for evaluating - rules. *) + rules. [metasyntax] uses the custom metasyntax definition. *) val apply : ?matcher:(module Matchers.Matcher.S) -> ?substitute_in_place:bool -> ?fresh:(unit -> string) + -> ?metasyntax:Matchers.Metasyntax.t -> t -> Match.environment -> result diff --git a/lib/kernel/language/alpha_rule.ml b/lib/kernel/language/alpha_rule.ml index 2596b35..1bacd4d 100644 --- a/lib/kernel/language/alpha_rule.ml +++ b/lib/kernel/language/alpha_rule.ml @@ -46,6 +46,7 @@ let rec apply ?(matcher = (module Matchers.Alpha.Generic : Matchers.Matcher.S)) ?(substitute_in_place = true) ?(fresh = counter) + ?metasyntax predicates env = let open Option in @@ -107,7 +108,7 @@ let rec apply in Option.value_map result ~f:ident ~default:(false, Some env) | Match (String template, cases) -> - let source, _ = Rewriter.Rewrite_template.substitute template env in + let source, _ = Rewriter.Rewrite_template.substitute ?metasyntax template env in let fresh_var = fresh () in let env = Environment.add env fresh_var source in rule_match env (Match (Variable fresh_var, cases)) @@ -139,11 +140,11 @@ let rec apply let configuration = Configuration.create ~match_kind:Fuzzy () in let matches = Matcher.all ~configuration ~template ~source () in let source = if substitute_in_place then Some source else None in - let result = Rewrite.all ?source ~rewrite_template matches in + let result = Rewrite.all ?metasyntax ?source ~rewrite_template matches in match result with | Some { rewritten_source; _ } -> (* substitute for variables that are in the outside scope *) - let rewritten_source, _ = Rewrite_template.substitute rewritten_source env in + let rewritten_source, _ = Rewrite_template.substitute ?metasyntax rewritten_source env in let env = Environment.update env variable rewritten_source in return (true, Some env) | None -> diff --git a/lib/kernel/language/omega_rule.ml b/lib/kernel/language/omega_rule.ml index 9a089f1..b0b5fe1 100644 --- a/lib/kernel/language/omega_rule.ml +++ b/lib/kernel/language/omega_rule.ml @@ -51,6 +51,7 @@ let rec apply ?(matcher = (module Matchers.Omega.Generic : Matchers.Matcher.S)) ?(substitute_in_place = true) ?(fresh = counter) + ?metasyntax predicates env = let open Option in @@ -112,7 +113,7 @@ let rec apply in Option.value_map result ~f:ident ~default:(false, Some env) | Match (String template, cases) -> - let source, _ = Rewriter.Rewrite_template.substitute template env in + let source, _ = Rewriter.Rewrite_template.substitute ?metasyntax template env in let fresh_var = fresh () in let env = Environment.add env fresh_var source in rule_match env (Match (Variable fresh_var, cases)) @@ -123,7 +124,7 @@ let rec apply | Some { variable; _ } -> (* FIXME(RVT) assumes only contextual rewrite for now. *) let env = - Rewrite_template.substitute rewrite_template env + Rewrite_template.substitute ?metasyntax rewrite_template env |> fst |> fun replacement' -> Environment.update env variable replacement' @@ -144,7 +145,7 @@ let rec apply let configuration = Configuration.create ~match_kind:Fuzzy () in let matches = Matcher.all ~configuration ~template ~source () in let source = if substitute_in_place then Some source else None in - let result = Rewrite.all ?source ~rewrite_template matches in + let result = Rewrite.all ?metasyntax ?source ~rewrite_template matches in match result with | Some { rewritten_source; _ } -> (* substitute for variables that are in the outside scope *) diff --git a/lib/kernel/language/types.ml b/lib/kernel/language/types.ml index de502a6..63179ed 100644 --- a/lib/kernel/language/types.ml +++ b/lib/kernel/language/types.ml @@ -18,6 +18,7 @@ module type Engine = sig : ?matcher:(module Matcher.S) -> ?substitute_in_place:bool -> ?fresh:(unit -> string) + -> ?metasyntax:Matchers.Metasyntax.t -> t -> environment -> result diff --git a/lib/kernel/matchers/alpha.ml b/lib/kernel/matchers/alpha.ml index f7f8393..2062890 100644 --- a/lib/kernel/matchers/alpha.ml +++ b/lib/kernel/matchers/alpha.ml @@ -1064,7 +1064,7 @@ module Make (Language : Language.S) (Metasyntax : Metasyntax.S) = struct else shift, extract_matched_text original_source match_start match_end in - if debug then Format.printf "Extracted matched: %s" matched; + if debug then Format.printf "Extracted matched: %s@." matched; let result = { result with matched } in if shift >= String.length original_source then result :: acc diff --git a/lib/kernel/rewriter/dune b/lib/kernel/rewriter/dune index 17b6589..7b1e965 100644 --- a/lib/kernel/rewriter/dune +++ b/lib/kernel/rewriter/dune @@ -2,5 +2,5 @@ (name rewriter) (public_name comby-kernel.rewriter) (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_deriving_yojson)) + (preprocess (pps ppx_deriving_yojson ppx_sexp_message ppx_sexp_conv)) (libraries comby-kernel.matchers comby-kernel.replacement core_kernel)) diff --git a/lib/kernel/rewriter/rewrite.ml b/lib/kernel/rewriter/rewrite.ml index 5709295..50dfc5b 100644 --- a/lib/kernel/rewriter/rewrite.ml +++ b/lib/kernel/rewriter/rewrite.ml @@ -8,7 +8,19 @@ let debug = | exception Not_found -> false | _ -> true -let substitute_match_contexts ?fresh ?metasyntax (matches: Match.t list) source replacements = +(* override default metasyntax for identifiers to accomodate fresh variable generation and UUID + identifiers that contain -, etc. *) +let match_context_syntax = + let identifier = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-" in + Matchers.Metasyntax.{ default_metasyntax with identifier } + +let match_context_metasyntax = + Matchers.Metasyntax.(create match_context_syntax) + +module Match_context_metasyntax = (val match_context_metasyntax) +module Match_context_template = Rewrite_template.Make(Match_context_metasyntax) + +let substitute_match_contexts ?fresh (matches: Match.t list) source replacements = if debug then Format.printf "Matches: %d | Replacements: %d@." (List.length matches) (List.length replacements); let rewrite_template, environment = List.fold2_exn @@ -19,8 +31,8 @@ let substitute_match_contexts ?fresh ?metasyntax (matches: Match.t list) source { replacement_content; _ } -> (* create a hole in the rewrite template based on this match context *) let sub_fresh = Option.map fresh ~f:(fun f -> fun () -> ("sub_" ^ f ())) in (* ensure custom fresh function is unique for substition. *) - let hole_id, rewrite_template = Rewrite_template.of_match_context ?metasyntax ?fresh:sub_fresh match_ ~source:rewrite_template in - if debug then Format.printf "Hole: %s in %s@." hole_id rewrite_template; + let hole_id, rewrite_template = Rewrite_template.of_match_context ?fresh:sub_fresh match_ ~source:rewrite_template in + if debug then Format.printf "Created rewrite template with hole var %s: %s @." hole_id rewrite_template; (* add this match context replacement to the environment *) let accumulator_environment = Environment.add accumulator_environment hole_id replacement_content in (* update match context replacements offset *) @@ -28,8 +40,10 @@ let substitute_match_contexts ?fresh ?metasyntax (matches: Match.t list) source in if debug then Format.printf "Env:@.%s" (Environment.to_string environment); if debug then Format.printf "Rewrite in:@.%s@." rewrite_template; - let rewritten_source = Rewrite_template.substitute ?metasyntax ?fresh rewrite_template environment |> fst in - let offsets = Rewrite_template.get_offsets_for_holes ?metasyntax rewrite_template (Environment.vars environment) in + let variables = Match_context_template.variables rewrite_template in + let rewritten_source = Rewrite_template.substitute ~metasyntax:match_context_syntax ?fresh rewrite_template environment |> fst in + if debug then Format.printf "Rewritten source:@.%s@." rewritten_source; + let offsets = Rewrite_template.get_offsets_for_holes variables rewrite_template in if debug then Format.printf "Replacements: %d | Offsets 1: %d@." (List.length replacements) (List.length offsets); let offsets = Rewrite_template.get_offsets_after_substitution offsets environment in @@ -47,21 +61,29 @@ let substitute_match_contexts ?fresh ?metasyntax (matches: Match.t list) source ; in_place_substitutions } - (* +(** store range information for this match_context replacement: (a) its offset in the original source (b) its replacement context (to calculate the range) (c) an environment of values that are updated to reflect their relative offset in the rewrite template - *) -let substitute_in_rewrite_template ?fresh ?metasyntax rewrite_template ({ environment; _ } : Match.t) = +*) +let substitute_in_rewrite_template + ?fresh + ?(metasyntax = Matchers.Metasyntax.default_metasyntax) + rewrite_template + ({ environment; _ } : Match.t) = + let (module M) = Matchers.Metasyntax.create metasyntax in + let module Template_parser = Rewrite_template.Make(M) in + let variables = Template_parser.variables rewrite_template in + let replacement_content, vars_substituted_for = Rewrite_template.substitute - ?metasyntax + ~metasyntax ?fresh rewrite_template environment in - let offsets = Rewrite_template.get_offsets_for_holes ?metasyntax rewrite_template (Environment.vars environment) in + let offsets = Rewrite_template.get_offsets_for_holes variables rewrite_template in let offsets = Rewrite_template.get_offsets_after_substitution offsets environment in let environment = List.fold offsets ~init:(Environment.create ()) ~f:(fun acc (var, relative_offset) -> @@ -101,7 +123,7 @@ let all ?source ?metasyntax ?fresh ~rewrite_template matches : result option = let matches : Match.t list = List.rev matches in matches |> List.map ~f:(substitute_in_rewrite_template ?metasyntax ?fresh rewrite_template) - |> substitute_match_contexts ?metasyntax ?fresh matches source + |> substitute_match_contexts ?fresh matches source |> Option.some (* no in place substitution, emit result separated by newlines *) | None -> diff --git a/lib/kernel/rewriter/rewrite_template.ml b/lib/kernel/rewriter/rewrite_template.ml index 05d5e78..0b21381 100644 --- a/lib/kernel/rewriter/rewrite_template.ml +++ b/lib/kernel/rewriter/rewrite_template.ml @@ -8,6 +8,129 @@ let debug = | exception Not_found -> false | _ -> true +type syntax = { variable: string; pattern: string } +[@@deriving sexp_of] + +type extracted = + | Hole of syntax + | Constant of string +[@@deriving sexp_of] + +module Make (Metasyntax : Matchers.Metasyntax.S) = struct + + let alphanum = + satisfy (function + | 'a' .. 'z' + | 'A' .. 'Z' + | '0' .. '9' -> true + | _ -> false) + + let blank = + choice + [ char ' ' + ; char '\t' + ] + + let ignore p = + p *> return () + + let p = function + | Some delim -> ignore @@ (string delim) + | None -> return () + + let any_char_except ~reserved = + List.fold reserved + ~init:(return `OK) + ~f:(fun acc reserved_sequence -> + option `End_of_input + (peek_string (String.length reserved_sequence) + >>= fun s -> + if String.equal s reserved_sequence then + return `Reserved_sequence + else + acc)) + >>= function + | `OK -> any_char + | `End_of_input -> any_char + | `Reserved_sequence -> fail "reserved sequence hit" + + let identifier () = + choice @@ List.map ~f:char (String.to_list Metasyntax.identifier) + + let identifier () = + both + (option false (char '?' >>| fun _ -> true)) + (many1 (identifier ()) >>| String.of_char_list) + + let regex_expression suffix = + fix (fun expr -> + choice + [ lift (fun x -> Format.sprintf "[%s]" @@ String.concat x) (char '[' *> many1 expr <* char ']') + ; lift (fun c -> Format.sprintf {|\%c|} c) (char '\\' *> any_char) + ; lift String.of_char (any_char_except ~reserved:[suffix]) + ]) + + let regex_body separator suffix = + lift2 + (fun v e -> v, e) + (identifier ()) + (char separator *> many1 (regex_expression suffix)) + + let hole_parsers = + (* Fold left to respect order of definitions in custom metasyntax for + matching, where we attempt to parse in order. Note this is significant if + a syntax like $X~regex should be tried before shortcircuiting on $X, in + which case it should be defined _after_ the $X syntax (most general + should be first). *) + List.fold ~init:[] Metasyntax.syntax ~f:(fun acc v -> + let v = + match v with + | Hole (_, Delimited (left, right)) -> + p left *> identifier () <* p right >>| + fun (o, v) -> + Format.sprintf "%s%s%s%s" (Option.value left ~default:"") (if o then "?" else "") v (Option.value right ~default:""), + v + | Regex (left, separator, right) -> + p (Some left) *> regex_body separator right <* p (Some right) >>| + fun ((_, v), expr) -> + (Format.sprintf "%s%s%c%s%s" left v separator (String.concat expr) right), + v + in + v::acc) + + let hole_prefixes = + List.map Metasyntax.syntax ~f:(function + | Hole (_, Delimited (Some left, _)) + | Regex (left, _, _) -> Some left + | _ -> None) + |> List.filter_opt + + (** Not smart enough: only looks for hole prefix to stop scanning constant, + because there isn't a good 'not' parser *) + let parse_template : extracted list Angstrom.t = + let hole = choice hole_parsers in + many @@ choice + [ (hole >>| fun (pattern, variable) -> Hole { pattern; variable } ) + ; (((many1 @@ any_char_except ~reserved:hole_prefixes)) >>| fun c -> Constant (String.of_char_list c)) + ; any_char >>| fun c -> Constant (Char.to_string c) (* accept anything as constant not accepted by attempting holes above *) + ] + + let parse template = + match parse_string ~consume:All parse_template template with + | Ok result -> Some result + | Error e -> failwith ("No rewrite template parse: "^e) + + let variables template = + parse template + |> function + | Some result -> + List.filter_map result ~f:(function + | Hole { pattern; variable } -> Some { pattern; variable } + | _ -> None) + | None -> + [] +end + let counter = let uuid_for_id_counter = ref 0 in fun () -> @@ -50,7 +173,10 @@ let parse_first_label ?(metasyntax = Matchers.Metasyntax.default_metasyntax) tem | Ok label -> List.find_map label ~f:ident | Error _ -> None -let substitute_fresh ?(metasyntax = Matchers.Metasyntax.default_metasyntax) ?(fresh = counter) template = +let substitute_fresh + ?(metasyntax = Matchers.Metasyntax.default_metasyntax) + ?(fresh = counter) + template = let label_table = String.Table.create () in let template_ref = ref template in let current_label_ref = ref (parse_first_label ~metasyntax !template_ref) in @@ -72,34 +198,28 @@ let substitute_fresh ?(metasyntax = Matchers.Metasyntax.default_metasyntax) ?(fr done; !template_ref -let formats_of_metasyntax metasyntax = - let open Matchers.Metasyntax in - List.filter_map metasyntax ~f:(function - | Hole (_, Delimited (left, right)) -> - let left = Option.value left ~default:"" in - let right = Option.value right ~default:"" in - Some [(left, right); (left^"?", right)] - | _ -> None) - |> List.concat - let substitute ?(metasyntax = Matchers.Metasyntax.default_metasyntax) ?fresh template env = - let substitution_formats = formats_of_metasyntax metasyntax.syntax in + let (module M) = Matchers.Metasyntax.create metasyntax in + let module Template_parser = Make(M) in + let vars = Template_parser.variables template in let template = substitute_fresh ~metasyntax ?fresh template in - Environment.vars env - |> List.fold ~init:(template, []) ~f:(fun (acc, vars) variable -> + if debug then Format.printf "Template after substituting fresh: %s@." template; + + List.fold vars ~init:(template, []) ~f:(fun (acc, vars) { variable; pattern } -> match Environment.lookup env variable with | Some value -> - List.find_map substitution_formats ~f:(fun (left,right) -> - let pattern = left^variable^right in - if Option.is_some (String.substr_index template ~pattern) then - Some (String.substr_replace_all acc ~pattern ~with_:value, variable::vars) - else - None) - |> Option.value ~default:(acc,vars) - | None -> acc, vars) + if Option.is_some (String.substr_index template ~pattern) then + String.substr_replace_all acc ~pattern ~with_:value, variable::vars + else + acc, vars + | None -> + acc, vars) +(** Uses metasyntax to substitute fresh variables in the match_context that + will be replaced. It returns (id * rewrite_template) where id is the part + that will be substituted with match_context, and rewrite_template is the + source that's been templatized. *) let of_match_context - ?(metasyntax = Matchers.Metasyntax.default_metasyntax) ?(fresh = sub_counter) { range = { match_start = { offset = start_index; _ } @@ -116,36 +236,37 @@ let of_match_context in let after_part = String.slice source end_index (String.length source) in let hole_id = fresh () in - let left, right = replacement_sentinel metasyntax in + let left, right = replacement_sentinel Matchers.Metasyntax.default_metasyntax in let rewrite_template = String.concat [before_part; left; hole_id; right; after_part] in hole_id, rewrite_template -(* return the offset for holes (specified by variables) in a given match template *) -let get_offsets_for_holes ?(metasyntax = Matchers.Metasyntax.default_metasyntax) rewrite_template variables = - let left, right = replacement_sentinel metasyntax in +(** return the offset for holes (specified by variables) in a given match template *) +let get_offsets_for_holes + variables + rewrite_template = let sorted_variables = - List.fold variables ~init:[] ~f:(fun acc variable -> - match String.substr_index rewrite_template ~pattern:(left^variable^right) with - | Some index -> - (variable, index)::acc + List.fold variables ~init:[] ~f:(fun acc { variable; pattern } -> + match String.substr_index rewrite_template ~pattern with + | Some index -> ((variable, pattern), index)::acc | None -> acc) |> List.sort ~compare:(fun (_, i1) (_, i2) -> i1 - i2) |> List.map ~f:fst in - List.fold sorted_variables ~init:(rewrite_template, []) ~f:(fun (rewrite_template, acc) variable -> - match String.substr_index rewrite_template ~pattern:(left^variable^right) with + List.fold sorted_variables ~init:(rewrite_template, []) ~f:(fun (rewrite_template, acc) (variable, pattern) -> + match String.substr_index rewrite_template ~pattern with | Some index -> let rewrite_template = - String.substr_replace_all rewrite_template ~pattern:(left^variable^right) ~with_:"" in + String.substr_replace_all rewrite_template ~pattern ~with_:"" in rewrite_template, (variable, index)::acc | None -> rewrite_template, acc) |> snd -(* pretend we substituted vars in offsets with environment. return what the offsets are after *) +(** pretend we substituted vars in offsets with environment. return what the offsets are after *) let get_offsets_after_substitution offsets environment = - List.fold_right offsets ~init:([],0) ~f:(fun (var, offset) (acc, shift) -> + if debug then Format.printf "Environment: %s@." @@ Match.Environment.to_string environment; + List.fold_right offsets ~init:([],0 ) ~f:(fun (var, offset) (acc, shift) -> match Environment.lookup environment var with - | None -> failwith "Expected var" + | None -> acc, shift | Some s -> let offset' = offset + shift in let shift = shift + String.length s in diff --git a/lib/kernel/rewriter/rewrite_template.mli b/lib/kernel/rewriter/rewrite_template.mli index 0c18023..bd043f8 100644 --- a/lib/kernel/rewriter/rewrite_template.mli +++ b/lib/kernel/rewriter/rewrite_template.mli @@ -1,6 +1,19 @@ open Matchers open Match +type syntax = { variable: string; pattern: string } +[@@deriving sexp_of] + +type extracted = + | Hole of syntax + | Constant of string +[@@deriving sexp_of] + +module Make : Metasyntax.S -> sig + val parse : string -> extracted list option + val variables : string -> syntax list + end + (** if [fresh] is set, then substitute the pattern :[id()] starting at 1, and incrementing subsequent IDs. If [fresh] is unset, then by default substitute the pattern :[id()] starting at 1, and increment for each occurence of @@ -10,8 +23,8 @@ val substitute_fresh : ?metasyntax:Metasyntax.t -> ?fresh:(unit -> string) -> st (** substitute returns the result and variables substituted for *) val substitute : ?metasyntax:Metasyntax.t -> ?fresh:(unit -> string) -> string -> Environment.t -> (string * string list) -val of_match_context : ?metasyntax:Metasyntax.t -> ?fresh:(unit -> string) -> Match.t -> source:string -> (string * string) +val of_match_context : ?fresh:(unit -> string) -> Match.t -> source:string -> (string * string) -val get_offsets_for_holes : ?metasyntax:Metasyntax.t -> string -> string list -> (string * int) list +val get_offsets_for_holes : syntax list -> string -> (string * int) list val get_offsets_after_substitution : (string * int) list -> Environment.t -> (string * int) list diff --git a/src/main.ml b/src/main.ml index 3230b9b..274cb97 100644 --- a/src/main.ml +++ b/src/main.ml @@ -39,7 +39,25 @@ let list_supported_languages_and_exit omega = Format.printf "%s%!" list; exit 0 -let substitute_environment_only_and_exit anonymous_arguments json_environment = +let substitute_environment_only_and_exit metasyntax_path anonymous_arguments json_environment = + let metasyntax = + (* FIXME this is copy pasta of command_configuration *) + match metasyntax_path with + | None -> Matchers.Metasyntax.default_metasyntax + | Some metasyntax_path -> + match Sys.file_exists metasyntax_path with + | `No | `Unknown -> + Format.eprintf "Could not open file: %s@." metasyntax_path; + exit 1 + | `Yes -> + Yojson.Safe.from_file metasyntax_path + |> Matchers.Metasyntax.of_yojson + |> function + | Ok c -> c + | Error error -> + Format.eprintf "%s@." error; + exit 1 + in let rewrite_template = match anonymous_arguments with | Some { rewrite_template; _ } -> rewrite_template @@ -56,7 +74,7 @@ let substitute_environment_only_and_exit anonymous_arguments json_environment = Match.Environment.of_yojson json |> function | Ok environment -> - let substituted, _ = Rewriter.Rewrite_template.substitute rewrite_template environment in + let substituted, _ = Rewriter.Rewrite_template.substitute ~metasyntax rewrite_template environment in Format.printf "%s@." substituted; exit 0 | Error err -> @@ -152,7 +170,7 @@ let base_command_parameters : (unit -> 'result) Command.Param.t = in if list then list_supported_languages_and_exit omega; if Option.is_some substitute_environment then - substitute_environment_only_and_exit anonymous_arguments substitute_environment; + substitute_environment_only_and_exit custom_metasyntax anonymous_arguments substitute_environment; let interactive_review = let default_editor = let f = Option.some in diff --git a/test/alpha/test_custom_metasyntax.ml b/test/alpha/test_custom_metasyntax.ml index 2119603..49e41cc 100644 --- a/test/alpha/test_custom_metasyntax.ml +++ b/test/alpha/test_custom_metasyntax.ml @@ -103,6 +103,37 @@ let%expect_test "custom_metasyntax_underscore" = [%expect_exact {|{"uri":null,"matches":[{"range":{"start":{"offset":0,"line":1,"column":1},"end":{"offset":11,"line":1,"column":12}},"environment":[{"variable":"_","value":"simple","range":{"start":{"offset":0,"line":1,"column":1},"end":{"offset":6,"line":1,"column":7}}}],"matched":"simple(bar)"}]} |}] +let%expect_test "custom_metasyntax_equivalence" = + let matcher = create + [ Hole (Everything, Delimited (Some "$", None)) + ; Regex ("$", '~', "$") + ] + in + + run matcher "foo(foo)" {|$A($A~\w+$)|} ""; + [%expect_exact {|{"uri":null,"matches":[{"range":{"start":{"offset":0,"line":1,"column":1},"end":{"offset":8,"line":1,"column":9}},"environment":[{"variable":"!@#$000000000006_A_equal","value":"foo","range":{"start":{"offset":4,"line":1,"column":5},"end":{"offset":7,"line":1,"column":8}}},{"variable":"A","value":"foo","range":{"start":{"offset":0,"line":1,"column":1},"end":{"offset":3,"line":1,"column":4}}}],"matched":"foo(foo)"}]} +|}] + +let%expect_test "custom_metasyntax_definition_order" = + let matcher = create + [ Regex ("$", '~', "$") + ; Hole (Everything, Delimited (Some "$", None)) + ] + in + + run matcher "simple(bar)baz" {|$A($B)$C~\w+$|} ""; + [%expect_exact {|No matches.|}]; + + let matcher = create + [ Hole (Everything, Delimited (Some "$", None)) + ; Regex ("$", '~', "$") + ] + in + + run matcher "simple(bar)baz" {|$A($B)$C~\w+$|} ""; + [%expect_exact {|{"uri":null,"matches":[{"range":{"start":{"offset":0,"line":1,"column":1},"end":{"offset":14,"line":1,"column":15}},"environment":[{"variable":"A","value":"simple","range":{"start":{"offset":0,"line":1,"column":1},"end":{"offset":6,"line":1,"column":7}}},{"variable":"B","value":"bar","range":{"start":{"offset":7,"line":1,"column":8},"end":{"offset":10,"line":1,"column":11}}},{"variable":"C","value":"baz","range":{"start":{"offset":11,"line":1,"column":12},"end":{"offset":14,"line":1,"column":15}}}],"matched":"simple(bar)baz"}]} +|}] + let%expect_test "custom_metasyntax_rewrite" = let syntax = let open Matchers.Metasyntax in @@ -113,6 +144,9 @@ let%expect_test "custom_metasyntax_rewrite" = let metasyntax = Matchers.Metasyntax.{ syntax; identifier = "ABCDEFGHIJKLMNOPQRSTUVWXYZ_" } in let matcher = Option.value_exn (Matchers.Alpha.select_with_extension ~metasyntax ".go") in + (* KNOWN LIMITATION/BUG: if ? is a prefix it conflicts with ? optional syntax + for variable names and substitution. Expect should be ?bar here. Remove + optional syntax. *) let specification = Configuration.Specification.create ~match_template:"$A(?B)" ~rewrite_template:"??B -> $A$A" () in let result = Pipeline.execute matcher ~metasyntax (String "simple(bar)") specification in let output = match result with @@ -121,7 +155,7 @@ let%expect_test "custom_metasyntax_rewrite" = | Nothing -> "nothing" in print_string output; - [%expect_exact {|?bar -> simplesimple|}]; + [%expect_exact {|bar -> simplesimple|}]; let specification = Configuration.Specification.create ~match_template:"$A(?B)" ~rewrite_template:"$id() $id(a) $id(a)" () in let result = Pipeline.execute matcher ~metasyntax (String "simple(bar)") specification in diff --git a/test/common/dune b/test/common/dune index be680d8..c402595 100644 --- a/test/common/dune +++ b/test/common/dune @@ -14,6 +14,9 @@ test_statistics test_offset_conversion test_parse_rule + test_rewrite_parts + test_parse_rewrite_template + test_substitute test_rewrite_rule_alpha test_rewrite_rule_omega @@ -57,9 +60,6 @@ test_pipeline_alpha test_pipeline_omega - test_rewrite_parts_alpha - test_rewrite_parts_omega - test_user_defined_language_alpha test_user_defined_language_omega diff --git a/test/common/test_cli.ml b/test/common/test_cli.ml index 168859a..d1d9c8e 100644 --- a/test/common/test_cli.ml +++ b/test/common/test_cli.ml @@ -215,6 +215,7 @@ let%expect_test "with_rewrite_rule" = |}] let%expect_test "with_rewrite_rule_stdin_default_no_extension" = + (* echo "hello world" | ./comby ':[[2]] :[[1]]' ':[1]' -rule 'where rewrite :[1] { ":[_]" -> ":[2]" }' -stdin *) let source = "hello world" in let match_template = ":[[2]] :[[1]]" in let rewrite_template = ":[1]" in @@ -1207,3 +1208,48 @@ let%expect_test "dot_comby_with_flags" = @|-1,1 +1,1 ============================================================ -|main(void) +|main(rewrite)|}] + +let%expect_test "test_custom_metasyntax_replace" = + let source = "a(b)" in + let metasyntax_path = "example" ^/ "metasyntax" ^/ "dolla.json" in + let command_args = + Format.sprintf "'$A($B~\\w+$)' '$A $B' -stdin -sequential -custom-metasyntax %s -stdout" metasyntax_path + in + let command = Format.sprintf "%s %s" binary_path command_args in + let result = read_expect_stdin_and_stdout command source in + print_string result; + [%expect "a b"] + +let%expect_test "test_custom_metasyntax_replace" = + let source = "a(b)" in + let metasyntax_path = "example" ^/ "metasyntax" ^/ "dolla.json" in + let command_args = + Format.sprintf "'$A($B~\\w+$)' '$A~x$ $B~\\w+$' -stdin -sequential -custom-metasyntax %s -stdout" metasyntax_path + in + let command = Format.sprintf "%s %s" binary_path command_args in + let result = read_expect_stdin_and_stdout command source in + print_string result; + [%expect "a b"] + +let%expect_test "test_custom_metasyntax_substitute" = + let source = "IGNORED" in + let metasyntax_path = "example" ^/ "metasyntax" ^/ "dolla.json" in + let env = {|[{"variable":"B", "value":"hello" }]|} in + let command_args = + Format.sprintf "'IGNORED' '$A $B~\\w+$' -stdin -sequential -custom-metasyntax %s -substitute-only '%s'" metasyntax_path env + in + let command = Format.sprintf "%s %s" binary_path command_args in + let result = read_expect_stdin_and_stdout command source in + print_string result; + [%expect "$A hello"] + +let%expect_test "test_custom_metasyntax_partial_rule_support" = + let source = "a(b)" in + let metasyntax_path = "example" ^/ "metasyntax" ^/ "dolla.json" in + let command_args = + Format.sprintf {|'$A($B)' '$A $B' -rule 'where rewrite :[A] { "$C~a$" -> "$C" }' -stdin -custom-metasyntax %s -stdout|} metasyntax_path + in + let command = Format.sprintf "%s %s" binary_path command_args in + let result = read_expect_stdin_and_stdout command source in + print_string result; + [%expect "a b"] diff --git a/test/common/test_extract_regex.ml b/test/common/test_extract_regex.ml index 2112b02..f7a7201 100644 --- a/test/common/test_extract_regex.ml +++ b/test/common/test_extract_regex.ml @@ -7,4 +7,4 @@ let%expect_test "basic" = let spec = Specification.create ~match_template () in let result = Specification.to_regex spec in print_string result; - [%expect_exact {|(for\s+)(?s:.)*?(,\s+)(?s:.)*?(\s+:=\s+range\s+)(?s:.)*?(\s+\{)(?s:.)*?(\})|}]; + [%expect_exact {|(for\s+)(\n|.)*?(,\s+)(\n|.)*?(\s+:=\s+range\s+)(\n|.)*?(\s+\{)(\n|.)*?(\})|}]; diff --git a/test/common/test_parse_rewrite_template.ml b/test/common/test_parse_rewrite_template.ml new file mode 100644 index 0000000..ef562e2 --- /dev/null +++ b/test/common/test_parse_rewrite_template.ml @@ -0,0 +1,37 @@ +open Core + +open Rewriter +open Rewrite_template + +let parse metasyntax template = + let (module M) = Matchers.Metasyntax.create metasyntax in + let module Template_parser = Make(M) in + let tree = Template_parser.parse template in + match tree with + | Some tree -> Sexp.to_string_hum (sexp_of_list sexp_of_extracted tree) + | None -> "ERROR: NO PARSE" + +let%expect_test "interpret_incomplete_hole_as_constant" = + let template = ":[B :[A]" in + parse Matchers.Metasyntax.default_metasyntax template |> print_string; + [%expect_exact {|((Constant :) (Constant "[B ") (Hole ((variable A) (pattern :[A]))))|}]; + + let template = ":[B :[A~x]" in + parse Matchers.Metasyntax.default_metasyntax template |> print_string; + [%expect_exact {|((Constant :) (Constant "[B ") (Hole ((variable A) (pattern :[A~x]))))|}] + +let%expect_test "interpret_incomplete_hole_as_constant_metasyntax" = + let template = "$:x $B:x $A" in + let metasyntax = + Matchers.Metasyntax.{ + syntax = + [ Hole (Everything, Delimited (Some "$", None)) + ; Hole (Alphanum, Delimited (Some "$$", None)) + ; Regex ("$", ':', " ") + ] + ; identifier = "AB" + } + in + parse metasyntax template |> print_string; + [%expect_exact {|((Constant $) (Constant ":x ") (Hole ((variable B) (pattern "$B:x "))) + (Constant " ") (Hole ((variable A) (pattern $A))))|}]; diff --git a/test/common/test_rewrite_parts.ml b/test/common/test_rewrite_parts.ml index 8badf9b..a453fce 100644 --- a/test/common/test_rewrite_parts.ml +++ b/test/common/test_rewrite_parts.ml @@ -1,19 +1,29 @@ open Core open Match -open Matchers open Rewriter +open Test_helpers + +include Test_alpha + +let all ?(configuration = configuration) template source = + C.all ~configuration ~template ~source () + +module Template_parser = Rewrite_template.Make(Matchers.Metasyntax.Default) + let%expect_test "get_offsets_for_holes" = let rewrite_template = {|1234:[1]1234:[2]|} in - let result = Rewrite_template.get_offsets_for_holes rewrite_template ["1"; "2"] in - print_s [%message (result : (string * int) list)]; - [%expect_exact {|(result ((2 8) (1 4))) + let variables = Template_parser.variables rewrite_template in + let offsets = Rewrite_template.get_offsets_for_holes variables rewrite_template in + print_s [%message (offsets : (string * int) list)]; + [%expect_exact {|(offsets ((2 8) (1 4))) |}] let%expect_test "get_offsets_for_holes_after_substitution_1" = let rewrite_template = {|1234:[1]1234:[2]|} in - let offsets = Rewrite_template.get_offsets_for_holes rewrite_template ["1"; "2"] in + let variables = Template_parser.variables rewrite_template in + let offsets = Rewrite_template.get_offsets_for_holes variables rewrite_template in let environment = Environment.create () |> (fun environment -> Environment.add environment "1" "333") @@ -26,7 +36,8 @@ let%expect_test "get_offsets_for_holes_after_substitution_1" = let%expect_test "get_offsets_for_holes_after_substitution_1" = let rewrite_template = {|1234:[1]1234:[3]11:[2]|} in - let offsets = Rewrite_template.get_offsets_for_holes rewrite_template ["1"; "3"; "2"] in + let variables = Template_parser.variables rewrite_template in + let offsets = Rewrite_template.get_offsets_for_holes variables rewrite_template in let environment = Environment.create () |> (fun environment -> Environment.add environment "1" "333") @@ -38,12 +49,6 @@ let%expect_test "get_offsets_for_holes_after_substitution_1" = [%expect_exact {|(result ((2 16) (3 11) (1 4))) |}] - -let configuration = Configuration.create ~match_kind:Fuzzy () - -let all ?(configuration = configuration) template source = - C.all ~configuration ~template ~source - let%expect_test "comments_in_string_literals_should_not_be_treated_as_comments_by_fuzzy" = let source = {|123433312343331122|} in let match_template = {|1234:[1]1234:[3]11:[2]|} in diff --git a/test/common/test_rewrite_parts_alpha.ml b/test/common/test_rewrite_parts_alpha.ml deleted file mode 100644 index 5f5d243..0000000 --- a/test/common/test_rewrite_parts_alpha.ml +++ /dev/null @@ -1,205 +0,0 @@ -open Core - -open Match -open Rewriter - -open Test_helpers - -include Test_alpha - -let all ?(configuration = configuration) template source = - C.all ~configuration ~template ~source () - -let%expect_test "get_offsets_for_holes" = - let rewrite_template = {|1234:[1]1234:[2]|} in - let result = Rewrite_template.get_offsets_for_holes rewrite_template ["1"; "2"] in - print_s [%message (result : (string * int) list)]; - [%expect_exact {|(result ((2 8) (1 4))) -|}] - -let%expect_test "get_offsets_for_holes_after_substitution_1" = - let rewrite_template = {|1234:[1]1234:[2]|} in - let offsets = Rewrite_template.get_offsets_for_holes rewrite_template ["1"; "2"] in - let environment = - Environment.create () - |> (fun environment -> Environment.add environment "1" "333") - |> (fun environment -> Environment.add environment "2" "22") - in - let result = Rewrite_template.get_offsets_after_substitution offsets environment in - print_s [%message (result : (string * int) list)]; - [%expect_exact {|(result ((2 11) (1 4))) -|}] - -let%expect_test "get_offsets_for_holes_after_substitution_1" = - let rewrite_template = {|1234:[1]1234:[3]11:[2]|} in - let offsets = Rewrite_template.get_offsets_for_holes rewrite_template ["1"; "3"; "2"] in - let environment = - Environment.create () - |> (fun environment -> Environment.add environment "1" "333") - |> (fun environment -> Environment.add environment "3" "333") - |> (fun environment -> Environment.add environment "2" "22") - in - let result = Rewrite_template.get_offsets_after_substitution offsets environment in - print_s [%message (result : (string * int) list)]; - [%expect_exact {|(result ((2 16) (3 11) (1 4))) -|}] - -let%expect_test "comments_in_string_literals_should_not_be_treated_as_comments_by_fuzzy" = - let source = {|123433312343331122|} in - let match_template = {|1234:[1]1234:[3]11:[2]|} in - let rewrite_template = {|1234:[1]1234:[3]11:[2]|} in - all match_template source - |> Rewrite.all ~source ~rewrite_template - |> (function - | Some rewrite_result -> print_string (Yojson.Safe.pretty_to_string (Replacement.result_to_yojson rewrite_result)) - | None -> print_string "BROKEN EXPECT"); - [%expect_exact {|{ - "rewritten_source": "123433312343331122", - "in_place_substitutions": [ - { - "range": { - "start": { "offset": 0, "line": -1, "column": -1 }, - "end": { "offset": 18, "line": -1, "column": -1 } - }, - "replacement_content": "123433312343331122", - "environment": [ - { - "variable": "1", - "value": "333", - "range": { - "start": { "offset": 4, "line": -1, "column": -1 }, - "end": { "offset": 7, "line": -1, "column": -1 } - } - }, - { - "variable": "2", - "value": "22", - "range": { - "start": { "offset": 16, "line": -1, "column": -1 }, - "end": { "offset": 18, "line": -1, "column": -1 } - } - }, - { - "variable": "3", - "value": "333", - "range": { - "start": { "offset": 11, "line": -1, "column": -1 }, - "end": { "offset": 14, "line": -1, "column": -1 } - } - } - ] - } - ] -}|}] - -let%expect_test "comments_in_string_literals_should_not_be_treated_as_comments_by_fuzzy" = - let source = {|123433312343331122;123433312343331122;|} in - let match_template = {|1234:[1]1234:[3]11:[2];|} in - let rewrite_template = {|1234:[1]1234:[3]11:[2];|} in - all match_template source - |> Rewrite.all ~source ~rewrite_template - |> (function - | Some rewrite_result -> print_string (Yojson.Safe.pretty_to_string (Replacement.result_to_yojson rewrite_result)) - | None -> print_string "BROKEN EXPECT"); - [%expect_exact {|{ - "rewritten_source": "123433312343331122;123433312343331122;", - "in_place_substitutions": [ - { - "range": { - "start": { "offset": 19, "line": -1, "column": -1 }, - "end": { "offset": 38, "line": -1, "column": -1 } - }, - "replacement_content": "123433312343331122;", - "environment": [ - { - "variable": "1", - "value": "333", - "range": { - "start": { "offset": 4, "line": -1, "column": -1 }, - "end": { "offset": 7, "line": -1, "column": -1 } - } - }, - { - "variable": "2", - "value": "22", - "range": { - "start": { "offset": 16, "line": -1, "column": -1 }, - "end": { "offset": 18, "line": -1, "column": -1 } - } - }, - { - "variable": "3", - "value": "333", - "range": { - "start": { "offset": 11, "line": -1, "column": -1 }, - "end": { "offset": 14, "line": -1, "column": -1 } - } - } - ] - }, - { - "range": { - "start": { "offset": 0, "line": -1, "column": -1 }, - "end": { "offset": 19, "line": -1, "column": -1 } - }, - "replacement_content": "123433312343331122;", - "environment": [ - { - "variable": "1", - "value": "333", - "range": { - "start": { "offset": 4, "line": -1, "column": -1 }, - "end": { "offset": 7, "line": -1, "column": -1 } - } - }, - { - "variable": "2", - "value": "22", - "range": { - "start": { "offset": 16, "line": -1, "column": -1 }, - "end": { "offset": 18, "line": -1, "column": -1 } - } - }, - { - "variable": "3", - "value": "333", - "range": { - "start": { "offset": 11, "line": -1, "column": -1 }, - "end": { "offset": 14, "line": -1, "column": -1 } - } - } - ] - } - ] -}|}] - -let%expect_test "multiple_contextual_substitutions" = - let source = {|foo bar foo|} in - let match_template = {|foo|} in - let rewrite_template = {|xxxx|} in - all match_template source - |> Rewrite.all ~source ~rewrite_template - |> (function - | Some rewrite_result -> print_string (Yojson.Safe.pretty_to_string (Replacement.result_to_yojson rewrite_result)) - | None -> print_string "BROKEN EXPECT"); - [%expect_exact {|{ - "rewritten_source": "xxxx bar xxxx", - "in_place_substitutions": [ - { - "range": { - "start": { "offset": 9, "line": -1, "column": -1 }, - "end": { "offset": 13, "line": -1, "column": -1 } - }, - "replacement_content": "xxxx", - "environment": [] - }, - { - "range": { - "start": { "offset": 0, "line": -1, "column": -1 }, - "end": { "offset": 4, "line": -1, "column": -1 } - }, - "replacement_content": "xxxx", - "environment": [] - } - ] -}|}] diff --git a/test/common/test_rewrite_parts_omega.ml b/test/common/test_rewrite_parts_omega.ml deleted file mode 100644 index a8d3923..0000000 --- a/test/common/test_rewrite_parts_omega.ml +++ /dev/null @@ -1,205 +0,0 @@ -open Core - -open Match -open Rewriter - -open Test_helpers - -include Test_omega - -let all ?(configuration = configuration) template source = - C.all ~configuration ~template ~source () - -let%expect_test "get_offsets_for_holes" = - let rewrite_template = {|1234:[1]1234:[2]|} in - let result = Rewrite_template.get_offsets_for_holes rewrite_template ["1"; "2"] in - print_s [%message (result : (string * int) list)]; - [%expect_exact {|(result ((2 8) (1 4))) -|}] - -let%expect_test "get_offsets_for_holes_after_substitution_1" = - let rewrite_template = {|1234:[1]1234:[2]|} in - let offsets = Rewrite_template.get_offsets_for_holes rewrite_template ["1"; "2"] in - let environment = - Environment.create () - |> (fun environment -> Environment.add environment "1" "333") - |> (fun environment -> Environment.add environment "2" "22") - in - let result = Rewrite_template.get_offsets_after_substitution offsets environment in - print_s [%message (result : (string * int) list)]; - [%expect_exact {|(result ((2 11) (1 4))) -|}] - -let%expect_test "get_offsets_for_holes_after_substitution_1" = - let rewrite_template = {|1234:[1]1234:[3]11:[2]|} in - let offsets = Rewrite_template.get_offsets_for_holes rewrite_template ["1"; "3"; "2"] in - let environment = - Environment.create () - |> (fun environment -> Environment.add environment "1" "333") - |> (fun environment -> Environment.add environment "3" "333") - |> (fun environment -> Environment.add environment "2" "22") - in - let result = Rewrite_template.get_offsets_after_substitution offsets environment in - print_s [%message (result : (string * int) list)]; - [%expect_exact {|(result ((2 16) (3 11) (1 4))) -|}] - -let%expect_test "comments_in_string_literals_should_not_be_treated_as_comments_by_fuzzy" = - let source = {|123433312343331122|} in - let match_template = {|1234:[1]1234:[3]11:[2]|} in - let rewrite_template = {|1234:[1]1234:[3]11:[2]|} in - all match_template source - |> Rewrite.all ~source ~rewrite_template - |> (function - | Some rewrite_result -> print_string (Yojson.Safe.pretty_to_string (Replacement.result_to_yojson rewrite_result)) - | None -> print_string "BROKEN EXPECT"); - [%expect_exact {|{ - "rewritten_source": "123433312343331122", - "in_place_substitutions": [ - { - "range": { - "start": { "offset": 0, "line": -1, "column": -1 }, - "end": { "offset": 18, "line": -1, "column": -1 } - }, - "replacement_content": "123433312343331122", - "environment": [ - { - "variable": "1", - "value": "333", - "range": { - "start": { "offset": 4, "line": -1, "column": -1 }, - "end": { "offset": 7, "line": -1, "column": -1 } - } - }, - { - "variable": "2", - "value": "22", - "range": { - "start": { "offset": 16, "line": -1, "column": -1 }, - "end": { "offset": 18, "line": -1, "column": -1 } - } - }, - { - "variable": "3", - "value": "333", - "range": { - "start": { "offset": 11, "line": -1, "column": -1 }, - "end": { "offset": 14, "line": -1, "column": -1 } - } - } - ] - } - ] -}|}] - -let%expect_test "comments_in_string_literals_should_not_be_treated_as_comments_by_fuzzy" = - let source = {|123433312343331122;123433312343331122;|} in - let match_template = {|1234:[1]1234:[3]11:[2];|} in - let rewrite_template = {|1234:[1]1234:[3]11:[2];|} in - all match_template source - |> Rewrite.all ~source ~rewrite_template - |> (function - | Some rewrite_result -> print_string (Yojson.Safe.pretty_to_string (Replacement.result_to_yojson rewrite_result)) - | None -> print_string "BROKEN EXPECT"); - [%expect_exact {|{ - "rewritten_source": "123433312343331122;123433312343331122;", - "in_place_substitutions": [ - { - "range": { - "start": { "offset": 19, "line": -1, "column": -1 }, - "end": { "offset": 38, "line": -1, "column": -1 } - }, - "replacement_content": "123433312343331122;", - "environment": [ - { - "variable": "1", - "value": "333", - "range": { - "start": { "offset": 4, "line": -1, "column": -1 }, - "end": { "offset": 7, "line": -1, "column": -1 } - } - }, - { - "variable": "2", - "value": "22", - "range": { - "start": { "offset": 16, "line": -1, "column": -1 }, - "end": { "offset": 18, "line": -1, "column": -1 } - } - }, - { - "variable": "3", - "value": "333", - "range": { - "start": { "offset": 11, "line": -1, "column": -1 }, - "end": { "offset": 14, "line": -1, "column": -1 } - } - } - ] - }, - { - "range": { - "start": { "offset": 0, "line": -1, "column": -1 }, - "end": { "offset": 19, "line": -1, "column": -1 } - }, - "replacement_content": "123433312343331122;", - "environment": [ - { - "variable": "1", - "value": "333", - "range": { - "start": { "offset": 4, "line": -1, "column": -1 }, - "end": { "offset": 7, "line": -1, "column": -1 } - } - }, - { - "variable": "2", - "value": "22", - "range": { - "start": { "offset": 16, "line": -1, "column": -1 }, - "end": { "offset": 18, "line": -1, "column": -1 } - } - }, - { - "variable": "3", - "value": "333", - "range": { - "start": { "offset": 11, "line": -1, "column": -1 }, - "end": { "offset": 14, "line": -1, "column": -1 } - } - } - ] - } - ] -}|}] - -let%expect_test "multiple_contextual_substitutions" = - let source = {|foo bar foo|} in - let match_template = {|foo|} in - let rewrite_template = {|xxxx|} in - all match_template source - |> Rewrite.all ~source ~rewrite_template - |> (function - | Some rewrite_result -> print_string (Yojson.Safe.pretty_to_string (Replacement.result_to_yojson rewrite_result)) - | None -> print_string "BROKEN EXPECT"); - [%expect_exact {|{ - "rewritten_source": "xxxx bar xxxx", - "in_place_substitutions": [ - { - "range": { - "start": { "offset": 9, "line": -1, "column": -1 }, - "end": { "offset": 13, "line": -1, "column": -1 } - }, - "replacement_content": "xxxx", - "environment": [] - }, - { - "range": { - "start": { "offset": 0, "line": -1, "column": -1 }, - "end": { "offset": 4, "line": -1, "column": -1 } - }, - "replacement_content": "xxxx", - "environment": [] - } - ] -}|}] diff --git a/test/common/test_substitute.ml b/test/common/test_substitute.ml new file mode 100644 index 0000000..ef953cf --- /dev/null +++ b/test/common/test_substitute.ml @@ -0,0 +1,31 @@ +open Core + +open Match +open Rewriter +open Rewrite_template + +let parse metasyntax template = + let (module M) = Matchers.Metasyntax.create metasyntax in + let module Template_parser = Make(M) in + let tree = Template_parser.parse template in + match tree with + | Some tree -> Sexp.to_string_hum (sexp_of_list sexp_of_extracted tree) + | None -> "ERROR: NO PARSE" + +let%expect_test "substitute_entire_regex_pattern_in_custom_metasyntax" = + let metasyntax = + Matchers.Metasyntax.{ + syntax = + [ Hole (Everything, Delimited (Some "$", None)) + ; Hole (Alphanum, Delimited (Some "$$", None)) + ; Regex ("$", ':', " ") + ] + ; identifier = "AB" + } + in + (* Don't just substitute for `$B`, but for `$B:\w+ `. This depends on Regex (more specific syntax) being defined _after_ the general syntax. *) + let template = {|$A $B:\w+ |} in + let environment = Environment.add (Environment.create ()) "B" "hello" in + let result, _ = Rewrite_template.substitute ~metasyntax template environment in + print_string result; + [%expect_exact {|$A hello|}] diff --git a/test/example/metasyntax/default-metasyntax.json b/test/example/metasyntax/default-metasyntax.json new file mode 100644 index 0000000..40bf216 --- /dev/null +++ b/test/example/metasyntax/default-metasyntax.json @@ -0,0 +1,13 @@ +{ + "syntax": [ + [ "Hole", [ "Everything" ], [ "Delimited", ":[", "]" ] ], + [ "Hole", [ "Expression" ], [ "Delimited", ":[", ":e]" ] ], + [ "Hole", [ "Alphanum" ], [ "Delimited", ":[[", "]]" ] ], + [ "Hole", [ "Non_space" ], [ "Delimited", ":[", ".]" ] ], + [ "Hole", [ "Line" ], [ "Delimited", ":[", "\\n]" ] ], + [ "Hole", [ "Blank" ], [ "Delimited", ":[ ", "]" ] ], + [ "Regex", ":[", "~", "]" ] + ], + "identifier": + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_" +} diff --git a/test/example/metasyntax/dolla.json b/test/example/metasyntax/dolla.json new file mode 100644 index 0000000..bee9e87 --- /dev/null +++ b/test/example/metasyntax/dolla.json @@ -0,0 +1,8 @@ +{ + "syntax": [ + [ "Hole", [ "Everything" ], [ "Delimited", "$", null ] ], + [ "Regex", "$", "~", "$" ] // order is significant! + ], + "identifier": + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_" +}