Rewrite template refactor (#277)

lib/app/comby.mli

@@ -465,15 +465,16 @@ module Rule : sig
   (** [create] parses and creates a rule. *)
   val create : string -> t Or_error.t
 
-  (** [apply matcher substitute_in_place fresh rule env] applies a [rule]
+  (** [apply matcher substitute_in_place fresh metasyntax rule env] applies a [rule]
       according to some [matcher] for existing matches in [env]. If
       [substitute_in_place] is true, rewrite rules substitute their values in
       place (default true). [fresh] introduces fresh variables for evaluating
-      rules. *)
+      rules. [metasyntax] uses the custom metasyntax definition. *)
   val apply
     :  ?matcher:(module Matchers.Matcher.S)
     -> ?substitute_in_place:bool
     -> ?fresh:(unit -> string)
+    -> ?metasyntax:Matchers.Metasyntax.t
     -> t
     -> Match.environment
     -> result

lib/app/configuration/command_configuration.ml

@@ -663,20 +663,20 @@ let select_matcher custom_metasyntax custom_matcher override_matcher file_filter
     let metasyntax = metasyntax custom_metasyntax in
     let syntax = syntax custom_matcher in
     E.create ~metasyntax syntax, None, Some metasyntax
-  (* forced language, optional custom metasyntax *)
   | _, Some language, custom_metasyntax ->
+    (* forced language, optional custom metasyntax *)
     let metasyntax = metasyntax custom_metasyntax in
     let (module Metasyntax) = Matchers.Metasyntax.create metasyntax in
     let (module Language) = force_language language in
     (module (E.Make (Language) (Metasyntax)) : Matchers.Matcher.S), None, Some metasyntax
-  (* infer language from file filters, definite custom metasyntax *)
   | _, _, Some custom_metasyntax ->
+    (* infer language from file filters, definite custom metasyntax *)
     let metasyntax = metasyntax (Some custom_metasyntax) in
     let (module Metasyntax) = Matchers.Metasyntax.create metasyntax in
     let (module Language) = force_language (extension file_filters) in
     (module (E.Make (Language) (Metasyntax)) : Matchers.Matcher.S), None, Some metasyntax
-  (* infer language from file filters, use default metasyntax *)
   | _, _, None ->
+    (* infer language from file filters, use default metasyntax *)
     of_extension (module E) file_filters
 
 let regex_of_specifications specifications =

lib/app/configuration/specification.ml

@@ -135,7 +135,7 @@ let to_regex { match_template; _ } =
   let extracted = List.map extracted ~f:(fun part -> Str.global_replace match_spaces {|\s+|} part) in
   (* ?s is modifier metasyntax where . matches all chars including newlines. See
      regular-expressions.info/modifier.html *)
-  Format.sprintf "(%s)" @@ String.concat extracted ~sep:")(?s:.)*?("
+  Format.sprintf "(%s)" @@ String.concat extracted ~sep:")(\\n|.)*?("
 
 let create ?rewrite_template ?rule ~match_template () =
   { match_template; rule; rewrite_template }

lib/app/pipeline/pipeline.ml

@@ -27,15 +27,15 @@ let infer_equality_constraints environment =
       else
         acc)
 
-let apply_rule ?(substitute_in_place = true) matcher omega rule matches =
+let apply_rule ?(substitute_in_place = true) ?metasyntax matcher omega rule matches =
   let open Option in
   List.filter_map matches ~f:(fun ({ environment; _ } as matched) ->
       let rule = rule @ infer_equality_constraints environment in
       let apply =
         if omega then
-          Rule.Omega.apply
+          Rule.Omega.apply ?metasyntax
         else
-          Rule.Alpha.apply
+          Rule.Alpha.apply ?metasyntax
       in
       let fresh () = Uuid_unix.(Fn.compose Uuid.to_string create ()) in
       let sat, env = apply ~fresh ~substitute_in_place ~matcher rule environment in
@@ -47,6 +47,7 @@ let timed_run
     ?(fast_offset_conversion = false)
     ?(omega = false)
     ?substitute_in_place
+    ?metasyntax
     ~configuration
     ~source
     ~specification:(Specification.{ match_template = template; rule; rewrite_template })
@@ -57,7 +58,7 @@ let timed_run
   let rule = Option.value rule ~default:[Ast.True] in
   let options = if omega then Rule.Omega.options rule else Rule.Alpha.options rule in
   let matches = Matcher.all ~nested:options.nested ~configuration ~template ~source () in
-  let matches = apply_rule ?substitute_in_place (module Matcher) omega rule matches in
+  let matches = apply_rule ?substitute_in_place ?metasyntax (module Matcher) omega rule matches in
   List.map matches ~f:(Match.convert_offset ~fast:fast_offset_conversion ~source)
 
 type output =
@@ -102,6 +103,7 @@ let process_single_source
       with_timeout timeout source ~f:(fun () ->
           timed_run
             matcher
+            ?metasyntax
             ~substitute_in_place
             ~omega
             ~fast_offset_conversion

lib/kernel/comby_kernel.mli

@@ -465,15 +465,16 @@ module Rule : sig
   (** [create] parses and creates a rule. *)
   val create : string -> t Or_error.t
 
-  (** [apply matcher substitute_in_place fresh rule env] applies a [rule]
+  (** [apply matcher substitute_in_place fresh metasyntax rule env] applies a [rule]
       according to some [matcher] for existing matches in [env]. If
       [substitute_in_place] is true, rewrite rules substitute their values in
       place (default true). [fresh] introduces fresh variables for evaluating
-      rules. *)
+      rules. [metasyntax] uses the custom metasyntax definition. *)
   val apply
     :  ?matcher:(module Matchers.Matcher.S)
     -> ?substitute_in_place:bool
     -> ?fresh:(unit -> string)
+    -> ?metasyntax:Matchers.Metasyntax.t
     -> t
     -> Match.environment
     -> result

lib/kernel/language/alpha_rule.ml

@@ -46,6 +46,7 @@ let rec apply
     ?(matcher = (module Matchers.Alpha.Generic : Matchers.Matcher.S))
     ?(substitute_in_place = true)
     ?(fresh = counter)
+    ?metasyntax
     predicates
     env =
   let open Option in
@@ -107,7 +108,7 @@ let rec apply
       in
       Option.value_map result ~f:ident ~default:(false, Some env)
     | Match (String template, cases) ->
-      let source, _ = Rewriter.Rewrite_template.substitute template env in
+      let source, _ = Rewriter.Rewrite_template.substitute ?metasyntax template env in
       let fresh_var = fresh () in
       let env = Environment.add env fresh_var source in
       rule_match env (Match (Variable fresh_var, cases))
@@ -139,11 +140,11 @@ let rec apply
             let configuration = Configuration.create ~match_kind:Fuzzy () in
             let matches = Matcher.all ~configuration ~template ~source () in
             let source = if substitute_in_place then Some source else None in
-            let result = Rewrite.all ?source ~rewrite_template matches in
+            let result = Rewrite.all ?metasyntax ?source ~rewrite_template matches in
             match result with
             | Some { rewritten_source; _ } ->
               (* substitute for variables that are in the outside scope *)
-              let rewritten_source, _ = Rewrite_template.substitute rewritten_source env in
+              let rewritten_source, _ = Rewrite_template.substitute ?metasyntax rewritten_source env in
               let env = Environment.update env variable rewritten_source in
               return (true, Some env)
             | None ->

lib/kernel/language/omega_rule.ml

@@ -51,6 +51,7 @@ let rec apply
     ?(matcher = (module Matchers.Omega.Generic : Matchers.Matcher.S))
     ?(substitute_in_place = true)
     ?(fresh = counter)
+    ?metasyntax
     predicates
     env =
   let open Option in
@@ -112,7 +113,7 @@ let rec apply
       in
       Option.value_map result ~f:ident ~default:(false, Some env)
     | Match (String template, cases) ->
-      let source, _ = Rewriter.Rewrite_template.substitute template env in
+      let source, _ = Rewriter.Rewrite_template.substitute ?metasyntax template env in
       let fresh_var = fresh () in
       let env = Environment.add env fresh_var source in
       rule_match env (Match (Variable fresh_var, cases))
@@ -123,7 +124,7 @@ let rec apply
         | Some { variable; _ } ->
           (* FIXME(RVT) assumes only contextual rewrite for now. *)
           let env =
-            Rewrite_template.substitute rewrite_template env
+            Rewrite_template.substitute ?metasyntax rewrite_template env
             |> fst
             |> fun replacement' ->
             Environment.update env variable replacement'
@@ -144,7 +145,7 @@ let rec apply
             let configuration = Configuration.create ~match_kind:Fuzzy () in
             let matches = Matcher.all ~configuration ~template ~source () in
             let source = if substitute_in_place then Some source else None in
-            let result = Rewrite.all ?source ~rewrite_template matches in
+            let result = Rewrite.all ?metasyntax ?source ~rewrite_template matches in
             match result with
             | Some { rewritten_source; _ } ->
               (* substitute for variables that are in the outside scope *)

lib/kernel/language/types.ml

@@ -18,6 +18,7 @@ module type Engine = sig
     :  ?matcher:(module Matcher.S)
     -> ?substitute_in_place:bool
     -> ?fresh:(unit -> string)
+    -> ?metasyntax:Matchers.Metasyntax.t
     -> t
     -> environment
     -> result

lib/kernel/matchers/alpha.ml

@@ -1064,7 +1064,7 @@ module Make (Language : Language.S) (Metasyntax : Metasyntax.S) = struct
               else
                 shift, extract_matched_text original_source match_start match_end
             in
-            if debug then Format.printf "Extracted matched: %s" matched;
+            if debug then Format.printf "Extracted matched: %s@." matched;
             let result = { result with matched } in
             if shift >= String.length original_source then
               result :: acc

lib/kernel/rewriter/dune

@@ -2,5 +2,5 @@
   (name rewriter)
   (public_name comby-kernel.rewriter)
   (instrumentation (backend bisect_ppx))
-  (preprocess (pps ppx_deriving_yojson))
+  (preprocess (pps ppx_deriving_yojson ppx_sexp_message ppx_sexp_conv))
   (libraries comby-kernel.matchers comby-kernel.replacement core_kernel))

lib/kernel/rewriter/rewrite.ml

@@ -8,7 +8,19 @@ let debug =
   | exception Not_found -> false
   | _ -> true
 
-let substitute_match_contexts ?fresh ?metasyntax (matches: Match.t list) source replacements =
+(* override default metasyntax for identifiers to accomodate fresh variable generation and UUID
+   identifiers that contain -, etc. *)
+let match_context_syntax =
+  let identifier = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-" in
+  Matchers.Metasyntax.{ default_metasyntax with identifier }
+
+let match_context_metasyntax =
+  Matchers.Metasyntax.(create match_context_syntax)
+
+module Match_context_metasyntax = (val match_context_metasyntax)
+module Match_context_template = Rewrite_template.Make(Match_context_metasyntax)
+
+let substitute_match_contexts ?fresh (matches: Match.t list) source replacements =
   if debug then Format.printf "Matches: %d | Replacements: %d@." (List.length matches) (List.length replacements);
   let rewrite_template, environment =
     List.fold2_exn
@@ -19,8 +31,8 @@ let substitute_match_contexts ?fresh ?metasyntax (matches: Match.t list) source
            { replacement_content; _ } ->
            (* create a hole in the rewrite template based on this match context *)
            let sub_fresh = Option.map fresh ~f:(fun f -> fun () -> ("sub_" ^ f ())) in (* ensure custom fresh function is unique for substition. *)
-           let hole_id, rewrite_template = Rewrite_template.of_match_context ?metasyntax ?fresh:sub_fresh match_ ~source:rewrite_template in
-           if debug then Format.printf "Hole: %s in %s@." hole_id rewrite_template;
+           let hole_id, rewrite_template = Rewrite_template.of_match_context ?fresh:sub_fresh match_ ~source:rewrite_template in
+           if debug then Format.printf "Created rewrite template with hole var %s: %s @." hole_id rewrite_template;
            (* add this match context replacement to the environment *)
            let accumulator_environment = Environment.add accumulator_environment hole_id replacement_content in
            (* update match context replacements offset *)
@@ -28,8 +40,10 @@ let substitute_match_contexts ?fresh ?metasyntax (matches: Match.t list) source
   in
   if debug then Format.printf "Env:@.%s" (Environment.to_string environment);
   if debug then Format.printf "Rewrite in:@.%s@." rewrite_template;
-  let rewritten_source = Rewrite_template.substitute ?metasyntax ?fresh rewrite_template environment |> fst in
-  let offsets = Rewrite_template.get_offsets_for_holes ?metasyntax rewrite_template (Environment.vars environment) in
+  let variables = Match_context_template.variables rewrite_template in
+  let rewritten_source = Rewrite_template.substitute ~metasyntax:match_context_syntax ?fresh rewrite_template environment |> fst in
+  if debug then Format.printf "Rewritten source:@.%s@." rewritten_source;
+  let offsets = Rewrite_template.get_offsets_for_holes variables rewrite_template in
   if debug then
     Format.printf "Replacements: %d | Offsets 1: %d@." (List.length replacements) (List.length offsets);
   let offsets = Rewrite_template.get_offsets_after_substitution offsets environment in
@@ -47,21 +61,29 @@ let substitute_match_contexts ?fresh ?metasyntax (matches: Match.t list) source
   ; in_place_substitutions
   }
 
-  (*
+(**
    store range information for this match_context replacement:
    (a) its offset in the original source
    (b) its replacement context (to calculate the range)
    (c) an environment of values that are updated to reflect their relative offset in the rewrite template
-   *)
-let substitute_in_rewrite_template ?fresh ?metasyntax rewrite_template ({ environment; _ } : Match.t) =
+*)
+let substitute_in_rewrite_template
+    ?fresh
+    ?(metasyntax = Matchers.Metasyntax.default_metasyntax)
+    rewrite_template
+    ({ environment; _ } : Match.t) =
+  let (module M) = Matchers.Metasyntax.create metasyntax in
+  let module Template_parser = Rewrite_template.Make(M) in
+  let variables = Template_parser.variables rewrite_template in
+
   let replacement_content, vars_substituted_for =
     Rewrite_template.substitute
-      ?metasyntax
+      ~metasyntax
       ?fresh
       rewrite_template
       environment
   in
-  let offsets = Rewrite_template.get_offsets_for_holes ?metasyntax rewrite_template (Environment.vars environment) in
+  let offsets = Rewrite_template.get_offsets_for_holes variables rewrite_template in
   let offsets = Rewrite_template.get_offsets_after_substitution offsets environment in
   let environment =
     List.fold offsets ~init:(Environment.create ()) ~f:(fun acc (var, relative_offset) ->
@@ -101,7 +123,7 @@ let all ?source ?metasyntax ?fresh ~rewrite_template matches : result option =
       let matches : Match.t list = List.rev matches in
       matches
       |> List.map ~f:(substitute_in_rewrite_template ?metasyntax ?fresh rewrite_template)
-      |> substitute_match_contexts ?metasyntax ?fresh matches source
+      |> substitute_match_contexts ?fresh matches source
       |> Option.some
     (* no in place substitution, emit result separated by newlines *)
     | None ->

lib/kernel/rewriter/rewrite_template.ml

@@ -8,6 +8,129 @@ let debug =
   | exception Not_found -> false
   | _ -> true
 
+type syntax = { variable: string; pattern: string }
+[@@deriving sexp_of]
+
+type extracted =
+  | Hole of syntax
+  | Constant of string
+[@@deriving sexp_of]
+
+module Make (Metasyntax : Matchers.Metasyntax.S) = struct
+
+  let alphanum =
+    satisfy (function
+        | 'a' .. 'z'
+        | 'A' .. 'Z'
+        | '0' .. '9' -> true
+        | _ -> false)
+
+  let blank =
+    choice
+      [ char ' '
+      ; char '\t'
+      ]
+
+  let ignore p =
+    p *> return ()
+
+  let p = function
+    | Some delim -> ignore @@ (string delim)
+    | None -> return ()
+
+  let any_char_except ~reserved =
+    List.fold reserved
+      ~init:(return `OK)
+      ~f:(fun acc reserved_sequence ->
+          option `End_of_input
+            (peek_string (String.length reserved_sequence)
+             >>= fun s ->
+             if String.equal s reserved_sequence then
+               return `Reserved_sequence
+             else
+               acc))
+    >>= function
+    | `OK -> any_char
+    | `End_of_input -> any_char
+    | `Reserved_sequence -> fail "reserved sequence hit"
+
+  let identifier () =
+    choice @@ List.map ~f:char (String.to_list Metasyntax.identifier)
+
+  let identifier () =
+    both
+      (option false (char '?' >>| fun _ -> true))
+      (many1 (identifier ()) >>| String.of_char_list)
+
+  let regex_expression suffix =
+    fix (fun expr ->
+        choice
+          [ lift (fun x -> Format.sprintf "[%s]" @@ String.concat x) (char '[' *> many1 expr <* char ']')
+          ; lift (fun c -> Format.sprintf {|\%c|} c) (char '\\' *> any_char)
+          ; lift String.of_char (any_char_except ~reserved:[suffix])
+          ])
+
+  let regex_body separator suffix =
+    lift2
+      (fun v e -> v, e)
+      (identifier ())
+      (char separator *> many1 (regex_expression suffix))
+
+  let hole_parsers =
+    (* Fold left to respect order of definitions in custom metasyntax for
+       matching, where we attempt to parse in order. Note this is significant if
+       a syntax like $X~regex should be tried before shortcircuiting on $X, in
+       which case it should be defined _after_ the $X syntax (most general
+       should be first). *)
+    List.fold ~init:[] Metasyntax.syntax ~f:(fun acc v ->
+        let v =
+          match v with
+          | Hole (_, Delimited (left, right)) ->
+            p left *> identifier () <* p right >>|
+            fun (o, v) ->
+            Format.sprintf "%s%s%s%s" (Option.value left ~default:"") (if o then "?" else "") v (Option.value right ~default:""),
+            v
+          | Regex (left, separator, right) ->
+            p (Some left) *> regex_body separator right <* p (Some right) >>|
+            fun ((_, v), expr) ->
+            (Format.sprintf "%s%s%c%s%s" left v separator (String.concat expr) right),
+            v
+        in
+        v::acc)
+
+  let hole_prefixes =
+    List.map Metasyntax.syntax ~f:(function
+        | Hole (_, Delimited (Some left, _))
+        | Regex (left, _, _) -> Some left
+        | _ -> None)
+    |> List.filter_opt
+
+  (** Not smart enough: only looks for hole prefix to stop scanning constant,
+      because there isn't a good 'not' parser *)
+  let parse_template : extracted list Angstrom.t =
+    let hole = choice hole_parsers in
+    many @@ choice
+      [ (hole >>| fun (pattern, variable) -> Hole { pattern; variable } )
+      ; (((many1 @@ any_char_except ~reserved:hole_prefixes)) >>| fun c -> Constant (String.of_char_list c))
+      ; any_char >>| fun c -> Constant (Char.to_string c) (* accept anything as constant not accepted by attempting holes above *)
+      ]
+
+  let parse template =
+    match parse_string ~consume:All parse_template template with
+    | Ok result -> Some result
+    | Error e -> failwith ("No rewrite template parse: "^e)
+
+  let variables template =
+    parse template
+    |> function
+    | Some result ->
+      List.filter_map result ~f:(function
+          | Hole { pattern; variable } -> Some { pattern; variable }
+          | _ -> None)
+    | None ->
+      []
+end
+
 let counter =
   let uuid_for_id_counter = ref 0 in
   fun () ->
@@ -50,7 +173,10 @@ let parse_first_label ?(metasyntax = Matchers.Metasyntax.default_metasyntax) tem
   | Ok label -> List.find_map label ~f:ident
   | Error _ -> None
 
-let substitute_fresh ?(metasyntax = Matchers.Metasyntax.default_metasyntax) ?(fresh = counter) template =
+let substitute_fresh
+    ?(metasyntax = Matchers.Metasyntax.default_metasyntax)
+    ?(fresh = counter)
+    template =
   let label_table = String.Table.create () in
   let template_ref = ref template in
   let current_label_ref = ref (parse_first_label ~metasyntax !template_ref) in
@@ -72,34 +198,28 @@ let substitute_fresh ?(metasyntax = Matchers.Metasyntax.default_metasyntax) ?(fr
   done;
   !template_ref
 
-let formats_of_metasyntax metasyntax =
-  let open Matchers.Metasyntax in
-  List.filter_map metasyntax ~f:(function
-      | Hole (_, Delimited (left, right)) ->
-        let left = Option.value left ~default:"" in
-        let right = Option.value right ~default:"" in
-        Some [(left, right); (left^"?", right)]
-      | _ -> None)
-  |> List.concat
-
 let substitute ?(metasyntax = Matchers.Metasyntax.default_metasyntax) ?fresh template env =
-  let substitution_formats = formats_of_metasyntax metasyntax.syntax in
+  let (module M) = Matchers.Metasyntax.create metasyntax in
+  let module Template_parser = Make(M) in
+  let vars = Template_parser.variables template in
   let template = substitute_fresh ~metasyntax ?fresh template in
-  Environment.vars env
-  |> List.fold ~init:(template, []) ~f:(fun (acc, vars) variable ->
+  if debug then Format.printf "Template after substituting fresh: %s@." template;
+
+  List.fold vars ~init:(template, []) ~f:(fun (acc, vars) { variable; pattern } ->
       match Environment.lookup env variable with
       | Some value ->
-        List.find_map substitution_formats ~f:(fun (left,right) ->
-            let pattern = left^variable^right in
-            if Option.is_some (String.substr_index template ~pattern) then
-              Some (String.substr_replace_all acc ~pattern ~with_:value, variable::vars)
-            else
-              None)
-        |> Option.value ~default:(acc,vars)
-      | None -> acc, vars)
+        if Option.is_some (String.substr_index template ~pattern) then
+          String.substr_replace_all acc ~pattern ~with_:value, variable::vars
+        else
+          acc, vars
+      | None ->
+        acc, vars)
 
+(** Uses metasyntax to substitute fresh variables in the match_context that
+    will be replaced. It returns (id * rewrite_template) where id is the part
+    that will be substituted with match_context, and rewrite_template is the
+    source that's been templatized. *)
 let of_match_context
-    ?(metasyntax = Matchers.Metasyntax.default_metasyntax)
     ?(fresh = sub_counter)
     { range =
         { match_start = { offset = start_index; _ }
@@ -116,36 +236,37 @@ let of_match_context
   in
   let after_part = String.slice source end_index (String.length source) in
   let hole_id = fresh () in
-  let left, right = replacement_sentinel metasyntax in
+  let left, right = replacement_sentinel Matchers.Metasyntax.default_metasyntax in
   let rewrite_template = String.concat [before_part; left; hole_id; right; after_part] in
   hole_id, rewrite_template
 
-(* return the offset for holes (specified by variables) in a given match template *)
-let get_offsets_for_holes ?(metasyntax = Matchers.Metasyntax.default_metasyntax) rewrite_template variables =
-  let left, right = replacement_sentinel metasyntax in
+(** return the offset for holes (specified by variables) in a given match template *)
+let get_offsets_for_holes
+    variables
+    rewrite_template =
   let sorted_variables =
-    List.fold variables ~init:[] ~f:(fun acc variable ->
-        match String.substr_index rewrite_template ~pattern:(left^variable^right) with
-        | Some index ->
-          (variable, index)::acc
+    List.fold variables ~init:[] ~f:(fun acc { variable; pattern } ->
+        match String.substr_index rewrite_template ~pattern with
+        | Some index -> ((variable, pattern), index)::acc
         | None -> acc)
     |> List.sort ~compare:(fun (_, i1) (_, i2) -> i1 - i2)
     |> List.map ~f:fst
   in
-  List.fold sorted_variables ~init:(rewrite_template, []) ~f:(fun (rewrite_template, acc) variable ->
-      match String.substr_index rewrite_template ~pattern:(left^variable^right) with
+  List.fold sorted_variables ~init:(rewrite_template, []) ~f:(fun (rewrite_template, acc) (variable, pattern) ->
+      match String.substr_index rewrite_template ~pattern with
       | Some index ->
         let rewrite_template =
-          String.substr_replace_all rewrite_template ~pattern:(left^variable^right) ~with_:"" in
+          String.substr_replace_all rewrite_template ~pattern ~with_:"" in
         rewrite_template, (variable, index)::acc
       | None -> rewrite_template, acc)
   |> snd
 
-(* pretend we substituted vars in offsets with environment. return what the offsets are after *)
+(** pretend we substituted vars in offsets with environment. return what the offsets are after *)
 let get_offsets_after_substitution offsets environment =
-  List.fold_right offsets ~init:([],0) ~f:(fun (var, offset) (acc, shift)  ->
+  if debug then Format.printf "Environment: %s@." @@ Match.Environment.to_string environment;
+  List.fold_right offsets ~init:([],0 ) ~f:(fun (var, offset) (acc, shift)  ->
       match Environment.lookup environment var with
-      | None -> failwith "Expected var"
+      | None -> acc, shift
       | Some s ->
         let offset' = offset + shift in
         let shift = shift + String.length s in

lib/kernel/rewriter/rewrite_template.mli

@@ -1,6 +1,19 @@
 open Matchers
 open Match
 
+type syntax = { variable: string; pattern: string }
+[@@deriving sexp_of]
+
+type extracted =
+  | Hole of syntax
+  | Constant of string
+[@@deriving sexp_of]
+
+module Make : Metasyntax.S -> sig
+    val parse : string -> extracted list option
+    val variables : string -> syntax list
+  end
+
 (** if [fresh] is set, then substitute the pattern :[id()] starting at 1, and
     incrementing subsequent IDs. If [fresh] is unset, then by default substitute
     the pattern :[id()] starting at 1, and increment for each occurence of
@@ -10,8 +23,8 @@ val substitute_fresh : ?metasyntax:Metasyntax.t -> ?fresh:(unit -> string) -> st
 (** substitute returns the result and variables substituted for *)
 val substitute : ?metasyntax:Metasyntax.t -> ?fresh:(unit -> string) -> string -> Environment.t -> (string * string list)
 
-val of_match_context : ?metasyntax:Metasyntax.t -> ?fresh:(unit -> string) -> Match.t -> source:string -> (string * string)
+val of_match_context : ?fresh:(unit -> string) -> Match.t -> source:string -> (string * string)
 
-val get_offsets_for_holes : ?metasyntax:Metasyntax.t -> string -> string list -> (string * int) list
+val get_offsets_for_holes : syntax list -> string -> (string * int) list
 
 val get_offsets_after_substitution : (string * int) list -> Environment.t -> (string * int) list

src/main.ml

@@ -39,7 +39,25 @@ let list_supported_languages_and_exit omega =
   Format.printf "%s%!" list;
   exit 0
 
-let substitute_environment_only_and_exit anonymous_arguments json_environment =
+let substitute_environment_only_and_exit metasyntax_path anonymous_arguments json_environment =
+  let metasyntax =
+    (* FIXME this is copy pasta of command_configuration *)
+    match metasyntax_path with
+    | None -> Matchers.Metasyntax.default_metasyntax
+    | Some metasyntax_path ->
+      match Sys.file_exists metasyntax_path with
+      | `No | `Unknown ->
+        Format.eprintf "Could not open file: %s@." metasyntax_path;
+        exit 1
+      | `Yes ->
+        Yojson.Safe.from_file metasyntax_path
+        |> Matchers.Metasyntax.of_yojson
+        |> function
+        | Ok c -> c
+        | Error error ->
+          Format.eprintf "%s@." error;
+          exit 1
+  in
   let rewrite_template =
     match anonymous_arguments with
     | Some { rewrite_template; _ } -> rewrite_template
@@ -56,7 +74,7 @@ let substitute_environment_only_and_exit anonymous_arguments json_environment =
       Match.Environment.of_yojson json
       |> function
       | Ok environment ->
-        let substituted, _ = Rewriter.Rewrite_template.substitute rewrite_template environment in
+        let substituted, _ = Rewriter.Rewrite_template.substitute ~metasyntax rewrite_template environment in
         Format.printf "%s@." substituted;
         exit 0
       | Error err ->
@@ -152,7 +170,7 @@ let base_command_parameters : (unit -> 'result) Command.Param.t =
     in
     if list then list_supported_languages_and_exit omega;
     if Option.is_some substitute_environment then
-      substitute_environment_only_and_exit anonymous_arguments substitute_environment;
+      substitute_environment_only_and_exit custom_metasyntax anonymous_arguments substitute_environment;
     let interactive_review =
       let default_editor =
         let f = Option.some in

test/alpha/test_custom_metasyntax.ml

@@ -103,6 +103,37 @@ let%expect_test "custom_metasyntax_underscore" =
   [%expect_exact {|{"uri":null,"matches":[{"range":{"start":{"offset":0,"line":1,"column":1},"end":{"offset":11,"line":1,"column":12}},"environment":[{"variable":"_","value":"simple","range":{"start":{"offset":0,"line":1,"column":1},"end":{"offset":6,"line":1,"column":7}}}],"matched":"simple(bar)"}]}
 |}]
 
+let%expect_test "custom_metasyntax_equivalence" =
+  let matcher = create
+      [ Hole (Everything, Delimited (Some "$", None))
+      ; Regex ("$", '~', "$")
+      ]
+  in
+
+  run matcher "foo(foo)" {|$A($A~\w+$)|} "";
+  [%expect_exact {|{"uri":null,"matches":[{"range":{"start":{"offset":0,"line":1,"column":1},"end":{"offset":8,"line":1,"column":9}},"environment":[{"variable":"!@#$000000000006_A_equal","value":"foo","range":{"start":{"offset":4,"line":1,"column":5},"end":{"offset":7,"line":1,"column":8}}},{"variable":"A","value":"foo","range":{"start":{"offset":0,"line":1,"column":1},"end":{"offset":3,"line":1,"column":4}}}],"matched":"foo(foo)"}]}
+|}]
+
+let%expect_test "custom_metasyntax_definition_order" =
+  let matcher = create
+      [ Regex ("$", '~', "$")
+      ; Hole (Everything, Delimited (Some "$", None))
+      ]
+  in
+
+  run matcher "simple(bar)baz" {|$A($B)$C~\w+$|} "";
+  [%expect_exact {|No matches.|}];
+
+  let matcher = create
+      [ Hole (Everything, Delimited (Some "$", None))
+      ; Regex ("$", '~', "$")
+      ]
+  in
+
+  run matcher "simple(bar)baz" {|$A($B)$C~\w+$|} "";
+  [%expect_exact {|{"uri":null,"matches":[{"range":{"start":{"offset":0,"line":1,"column":1},"end":{"offset":14,"line":1,"column":15}},"environment":[{"variable":"A","value":"simple","range":{"start":{"offset":0,"line":1,"column":1},"end":{"offset":6,"line":1,"column":7}}},{"variable":"B","value":"bar","range":{"start":{"offset":7,"line":1,"column":8},"end":{"offset":10,"line":1,"column":11}}},{"variable":"C","value":"baz","range":{"start":{"offset":11,"line":1,"column":12},"end":{"offset":14,"line":1,"column":15}}}],"matched":"simple(bar)baz"}]}
+|}]
+
 let%expect_test "custom_metasyntax_rewrite" =
   let syntax =
     let open Matchers.Metasyntax in
@@ -113,6 +144,9 @@ let%expect_test "custom_metasyntax_rewrite" =
   let metasyntax = Matchers.Metasyntax.{ syntax; identifier = "ABCDEFGHIJKLMNOPQRSTUVWXYZ_" } in
   let matcher = Option.value_exn (Matchers.Alpha.select_with_extension ~metasyntax ".go") in
 
+  (* KNOWN LIMITATION/BUG: if ? is a prefix it conflicts with ? optional syntax
+     for variable names and substitution. Expect should be ?bar here. Remove
+     optional syntax. *)
   let specification = Configuration.Specification.create ~match_template:"$A(?B)" ~rewrite_template:"??B -> $A$A" () in
   let result = Pipeline.execute matcher ~metasyntax (String "simple(bar)") specification in
   let output = match result with
@@ -121,7 +155,7 @@ let%expect_test "custom_metasyntax_rewrite" =
     | Nothing -> "nothing"
   in
   print_string output;
-  [%expect_exact {|?bar -> simplesimple|}];
+  [%expect_exact {|bar -> simplesimple|}];
 
   let specification = Configuration.Specification.create ~match_template:"$A(?B)" ~rewrite_template:"$id() $id(a) $id(a)" () in
   let result = Pipeline.execute matcher ~metasyntax (String "simple(bar)") specification in

test/common/dune

@@ -14,6 +14,9 @@
   test_statistics
   test_offset_conversion
   test_parse_rule
+  test_rewrite_parts
+  test_parse_rewrite_template
+  test_substitute
 
   test_rewrite_rule_alpha
   test_rewrite_rule_omega
@@ -57,9 +60,6 @@
   test_pipeline_alpha
   test_pipeline_omega
 
-  test_rewrite_parts_alpha
-  test_rewrite_parts_omega
-
   test_user_defined_language_alpha
   test_user_defined_language_omega
 

test/common/test_cli.ml

@@ -215,6 +215,7 @@ let%expect_test "with_rewrite_rule" =
 |}]
 
 let%expect_test "with_rewrite_rule_stdin_default_no_extension" =
+  (* echo "hello world" | ./comby ':[[2]] :[[1]]' ':[1]' -rule 'where rewrite :[1] { ":[_]" -> ":[2]" }' -stdin *)
   let source = "hello world" in
   let match_template = ":[[2]] :[[1]]" in
   let rewrite_template = ":[1]" in
@@ -1207,3 +1208,48 @@ let%expect_test "dot_comby_with_flags" =
     @|-1,1 +1,1 ============================================================
     -|main(void)
     +|main(rewrite)|}]
+
+let%expect_test "test_custom_metasyntax_replace" =
+  let source = "a(b)" in
+  let metasyntax_path = "example" ^/ "metasyntax" ^/ "dolla.json" in
+  let command_args =
+    Format.sprintf "'$A($B~\\w+$)' '$A $B' -stdin -sequential -custom-metasyntax %s -stdout" metasyntax_path
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect "a b"]
+
+let%expect_test "test_custom_metasyntax_replace" =
+  let source = "a(b)" in
+  let metasyntax_path = "example" ^/ "metasyntax" ^/ "dolla.json" in
+  let command_args =
+    Format.sprintf "'$A($B~\\w+$)' '$A~x$ $B~\\w+$' -stdin -sequential -custom-metasyntax %s -stdout" metasyntax_path
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect "a b"]
+
+let%expect_test "test_custom_metasyntax_substitute" =
+  let source = "IGNORED" in
+  let metasyntax_path = "example" ^/ "metasyntax" ^/ "dolla.json" in
+  let env = {|[{"variable":"B", "value":"hello" }]|} in
+  let command_args =
+    Format.sprintf "'IGNORED' '$A $B~\\w+$' -stdin -sequential -custom-metasyntax %s -substitute-only '%s'" metasyntax_path env
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect "$A hello"]
+
+let%expect_test "test_custom_metasyntax_partial_rule_support" =
+  let source = "a(b)" in
+  let metasyntax_path = "example" ^/ "metasyntax" ^/ "dolla.json" in
+  let command_args =
+    Format.sprintf {|'$A($B)' '$A $B' -rule 'where rewrite :[A] { "$C~a$" -> "$C" }' -stdin -custom-metasyntax %s -stdout|} metasyntax_path
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect "a b"]

test/common/test_extract_regex.ml

@@ -7,4 +7,4 @@ let%expect_test "basic" =
   let spec = Specification.create ~match_template () in
   let result = Specification.to_regex spec in
   print_string result;
-  [%expect_exact {|(for\s+)(?s:.)*?(,\s+)(?s:.)*?(\s+:=\s+range\s+)(?s:.)*?(\s+\{)(?s:.)*?(\})|}];
+  [%expect_exact {|(for\s+)(\n|.)*?(,\s+)(\n|.)*?(\s+:=\s+range\s+)(\n|.)*?(\s+\{)(\n|.)*?(\})|}];

test/common/test_parse_rewrite_template.ml

@@ -0,0 +1,37 @@
+open Core
+
+open Rewriter
+open Rewrite_template
+
+let parse metasyntax template =
+  let (module M) = Matchers.Metasyntax.create metasyntax in
+  let module Template_parser = Make(M) in
+  let tree = Template_parser.parse template in
+  match tree with
+  | Some tree -> Sexp.to_string_hum (sexp_of_list sexp_of_extracted tree)
+  | None -> "ERROR: NO PARSE"
+
+let%expect_test "interpret_incomplete_hole_as_constant" =
+  let template = ":[B :[A]" in
+  parse Matchers.Metasyntax.default_metasyntax template |> print_string;
+  [%expect_exact {|((Constant :) (Constant "[B ") (Hole ((variable A) (pattern :[A]))))|}];
+
+  let template = ":[B :[A~x]" in
+  parse Matchers.Metasyntax.default_metasyntax template |> print_string;
+  [%expect_exact {|((Constant :) (Constant "[B ") (Hole ((variable A) (pattern :[A~x]))))|}]
+
+let%expect_test "interpret_incomplete_hole_as_constant_metasyntax" =
+  let template = "$:x $B:x  $A" in
+  let metasyntax =
+    Matchers.Metasyntax.{
+      syntax =
+        [ Hole (Everything, Delimited (Some "$", None))
+        ; Hole (Alphanum, Delimited (Some "$$", None))
+        ; Regex ("$", ':', " ")
+        ]
+    ; identifier = "AB"
+    }
+  in
+  parse metasyntax template |> print_string;
+  [%expect_exact {|((Constant $) (Constant ":x ") (Hole ((variable B) (pattern "$B:x ")))
+ (Constant " ") (Hole ((variable A) (pattern $A))))|}];

test/common/test_rewrite_parts.ml

@@ -1,19 +1,29 @@
 open Core
 
 open Match
-open Matchers
 open Rewriter
 
+open Test_helpers
+
+include Test_alpha
+
+let all ?(configuration = configuration) template source =
+  C.all ~configuration ~template ~source ()
+
+module Template_parser = Rewrite_template.Make(Matchers.Metasyntax.Default)
+
 let%expect_test "get_offsets_for_holes" =
   let rewrite_template = {|1234:[1]1234:[2]|} in
-  let result = Rewrite_template.get_offsets_for_holes rewrite_template ["1"; "2"] in
-  print_s [%message (result : (string * int) list)];
-  [%expect_exact {|(result ((2 8) (1 4)))
+  let variables = Template_parser.variables rewrite_template in
+  let offsets = Rewrite_template.get_offsets_for_holes variables rewrite_template in
+  print_s [%message (offsets : (string * int) list)];
+  [%expect_exact {|(offsets ((2 8) (1 4)))
 |}]
 
 let%expect_test "get_offsets_for_holes_after_substitution_1" =
   let rewrite_template = {|1234:[1]1234:[2]|} in
-  let offsets = Rewrite_template.get_offsets_for_holes rewrite_template ["1"; "2"] in
+  let variables = Template_parser.variables rewrite_template in
+  let offsets = Rewrite_template.get_offsets_for_holes variables rewrite_template in
   let environment =
     Environment.create ()
     |> (fun environment -> Environment.add environment "1" "333")
@@ -26,7 +36,8 @@ let%expect_test "get_offsets_for_holes_after_substitution_1" =
 
 let%expect_test "get_offsets_for_holes_after_substitution_1" =
   let rewrite_template = {|1234:[1]1234:[3]11:[2]|} in
-  let offsets = Rewrite_template.get_offsets_for_holes rewrite_template ["1"; "3"; "2"] in
+  let variables = Template_parser.variables rewrite_template in
+  let offsets = Rewrite_template.get_offsets_for_holes variables rewrite_template in
   let environment =
     Environment.create ()
     |> (fun environment -> Environment.add environment "1" "333")
@@ -38,12 +49,6 @@ let%expect_test "get_offsets_for_holes_after_substitution_1" =
   [%expect_exact {|(result ((2 16) (3 11) (1 4)))
 |}]
 
-
-let configuration = Configuration.create ~match_kind:Fuzzy ()
-
-let all ?(configuration = configuration) template source =
-  C.all ~configuration ~template ~source
-
 let%expect_test "comments_in_string_literals_should_not_be_treated_as_comments_by_fuzzy" =
   let source = {|123433312343331122|} in
   let match_template = {|1234:[1]1234:[3]11:[2]|} in

test/common/test_rewrite_parts_alpha.ml

@@ -1,205 +0,0 @@
-open Core
-
-open Match
-open Rewriter
-
-open Test_helpers
-
-include Test_alpha
-
-let all ?(configuration = configuration) template source =
-  C.all ~configuration ~template ~source ()
-
-let%expect_test "get_offsets_for_holes" =
-  let rewrite_template = {|1234:[1]1234:[2]|} in
-  let result = Rewrite_template.get_offsets_for_holes rewrite_template ["1"; "2"] in
-  print_s [%message (result : (string * int) list)];
-  [%expect_exact {|(result ((2 8) (1 4)))
-|}]
-
-let%expect_test "get_offsets_for_holes_after_substitution_1" =
-  let rewrite_template = {|1234:[1]1234:[2]|} in
-  let offsets = Rewrite_template.get_offsets_for_holes rewrite_template ["1"; "2"] in
-  let environment =
-    Environment.create ()
-    |> (fun environment -> Environment.add environment "1" "333")
-    |> (fun environment -> Environment.add environment "2" "22")
-  in
-  let result = Rewrite_template.get_offsets_after_substitution offsets environment in
-  print_s [%message (result : (string * int) list)];
-  [%expect_exact {|(result ((2 11) (1 4)))
-|}]
-
-let%expect_test "get_offsets_for_holes_after_substitution_1" =
-  let rewrite_template = {|1234:[1]1234:[3]11:[2]|} in
-  let offsets = Rewrite_template.get_offsets_for_holes rewrite_template ["1"; "3"; "2"] in
-  let environment =
-    Environment.create ()
-    |> (fun environment -> Environment.add environment "1" "333")
-    |> (fun environment -> Environment.add environment "3" "333")
-    |> (fun environment -> Environment.add environment "2" "22")
-  in
-  let result = Rewrite_template.get_offsets_after_substitution offsets environment in
-  print_s [%message (result : (string * int) list)];
-  [%expect_exact {|(result ((2 16) (3 11) (1 4)))
-|}]
-
-let%expect_test "comments_in_string_literals_should_not_be_treated_as_comments_by_fuzzy" =
-  let source = {|123433312343331122|} in
-  let match_template = {|1234:[1]1234:[3]11:[2]|} in
-  let rewrite_template = {|1234:[1]1234:[3]11:[2]|} in
-  all match_template source
-  |> Rewrite.all ~source ~rewrite_template
-  |> (function
-      | Some rewrite_result -> print_string (Yojson.Safe.pretty_to_string (Replacement.result_to_yojson rewrite_result))
-      | None -> print_string "BROKEN EXPECT");
-  [%expect_exact {|{
-  "rewritten_source": "123433312343331122",
-  "in_place_substitutions": [
-    {
-      "range": {
-        "start": { "offset": 0, "line": -1, "column": -1 },
-        "end": { "offset": 18, "line": -1, "column": -1 }
-      },
-      "replacement_content": "123433312343331122",
-      "environment": [
-        {
-          "variable": "1",
-          "value": "333",
-          "range": {
-            "start": { "offset": 4, "line": -1, "column": -1 },
-            "end": { "offset": 7, "line": -1, "column": -1 }
-          }
-        },
-        {
-          "variable": "2",
-          "value": "22",
-          "range": {
-            "start": { "offset": 16, "line": -1, "column": -1 },
-            "end": { "offset": 18, "line": -1, "column": -1 }
-          }
-        },
-        {
-          "variable": "3",
-          "value": "333",
-          "range": {
-            "start": { "offset": 11, "line": -1, "column": -1 },
-            "end": { "offset": 14, "line": -1, "column": -1 }
-          }
-        }
-      ]
-    }
-  ]
-}|}]
-
-let%expect_test "comments_in_string_literals_should_not_be_treated_as_comments_by_fuzzy" =
-  let source = {|123433312343331122;123433312343331122;|} in
-  let match_template = {|1234:[1]1234:[3]11:[2];|} in
-  let rewrite_template = {|1234:[1]1234:[3]11:[2];|} in
-  all match_template source
-  |> Rewrite.all ~source ~rewrite_template
-  |> (function
-      | Some rewrite_result -> print_string (Yojson.Safe.pretty_to_string (Replacement.result_to_yojson rewrite_result))
-      | None -> print_string "BROKEN EXPECT");
-  [%expect_exact {|{
-  "rewritten_source": "123433312343331122;123433312343331122;",
-  "in_place_substitutions": [
-    {
-      "range": {
-        "start": { "offset": 19, "line": -1, "column": -1 },
-        "end": { "offset": 38, "line": -1, "column": -1 }
-      },
-      "replacement_content": "123433312343331122;",
-      "environment": [
-        {
-          "variable": "1",
-          "value": "333",
-          "range": {
-            "start": { "offset": 4, "line": -1, "column": -1 },
-            "end": { "offset": 7, "line": -1, "column": -1 }
-          }
-        },
-        {
-          "variable": "2",
-          "value": "22",
-          "range": {
-            "start": { "offset": 16, "line": -1, "column": -1 },
-            "end": { "offset": 18, "line": -1, "column": -1 }
-          }
-        },
-        {
-          "variable": "3",
-          "value": "333",
-          "range": {
-            "start": { "offset": 11, "line": -1, "column": -1 },
-            "end": { "offset": 14, "line": -1, "column": -1 }
-          }
-        }
-      ]
-    },
-    {
-      "range": {
-        "start": { "offset": 0, "line": -1, "column": -1 },
-        "end": { "offset": 19, "line": -1, "column": -1 }
-      },
-      "replacement_content": "123433312343331122;",
-      "environment": [
-        {
-          "variable": "1",
-          "value": "333",
-          "range": {
-            "start": { "offset": 4, "line": -1, "column": -1 },
-            "end": { "offset": 7, "line": -1, "column": -1 }
-          }
-        },
-        {
-          "variable": "2",
-          "value": "22",
-          "range": {
-            "start": { "offset": 16, "line": -1, "column": -1 },
-            "end": { "offset": 18, "line": -1, "column": -1 }
-          }
-        },
-        {
-          "variable": "3",
-          "value": "333",
-          "range": {
-            "start": { "offset": 11, "line": -1, "column": -1 },
-            "end": { "offset": 14, "line": -1, "column": -1 }
-          }
-        }
-      ]
-    }
-  ]
-}|}]
-
-let%expect_test "multiple_contextual_substitutions" =
-  let source = {|foo bar foo|} in
-  let match_template = {|foo|} in
-  let rewrite_template = {|xxxx|} in
-  all match_template source
-  |> Rewrite.all ~source ~rewrite_template
-  |> (function
-      | Some rewrite_result -> print_string (Yojson.Safe.pretty_to_string (Replacement.result_to_yojson rewrite_result))
-      | None -> print_string "BROKEN EXPECT");
-  [%expect_exact {|{
-  "rewritten_source": "xxxx bar xxxx",
-  "in_place_substitutions": [
-    {
-      "range": {
-        "start": { "offset": 9, "line": -1, "column": -1 },
-        "end": { "offset": 13, "line": -1, "column": -1 }
-      },
-      "replacement_content": "xxxx",
-      "environment": []
-    },
-    {
-      "range": {
-        "start": { "offset": 0, "line": -1, "column": -1 },
-        "end": { "offset": 4, "line": -1, "column": -1 }
-      },
-      "replacement_content": "xxxx",
-      "environment": []
-    }
-  ]
-}|}]

test/common/test_rewrite_parts_omega.ml

@@ -1,205 +0,0 @@
-open Core
-
-open Match
-open Rewriter
-
-open Test_helpers
-
-include Test_omega
-
-let all ?(configuration = configuration) template source =
-  C.all ~configuration ~template ~source ()
-
-let%expect_test "get_offsets_for_holes" =
-  let rewrite_template = {|1234:[1]1234:[2]|} in
-  let result = Rewrite_template.get_offsets_for_holes rewrite_template ["1"; "2"] in
-  print_s [%message (result : (string * int) list)];
-  [%expect_exact {|(result ((2 8) (1 4)))
-|}]
-
-let%expect_test "get_offsets_for_holes_after_substitution_1" =
-  let rewrite_template = {|1234:[1]1234:[2]|} in
-  let offsets = Rewrite_template.get_offsets_for_holes rewrite_template ["1"; "2"] in
-  let environment =
-    Environment.create ()
-    |> (fun environment -> Environment.add environment "1" "333")
-    |> (fun environment -> Environment.add environment "2" "22")
-  in
-  let result = Rewrite_template.get_offsets_after_substitution offsets environment in
-  print_s [%message (result : (string * int) list)];
-  [%expect_exact {|(result ((2 11) (1 4)))
-|}]
-
-let%expect_test "get_offsets_for_holes_after_substitution_1" =
-  let rewrite_template = {|1234:[1]1234:[3]11:[2]|} in
-  let offsets = Rewrite_template.get_offsets_for_holes rewrite_template ["1"; "3"; "2"] in
-  let environment =
-    Environment.create ()
-    |> (fun environment -> Environment.add environment "1" "333")
-    |> (fun environment -> Environment.add environment "3" "333")
-    |> (fun environment -> Environment.add environment "2" "22")
-  in
-  let result = Rewrite_template.get_offsets_after_substitution offsets environment in
-  print_s [%message (result : (string * int) list)];
-  [%expect_exact {|(result ((2 16) (3 11) (1 4)))
-|}]
-
-let%expect_test "comments_in_string_literals_should_not_be_treated_as_comments_by_fuzzy" =
-  let source = {|123433312343331122|} in
-  let match_template = {|1234:[1]1234:[3]11:[2]|} in
-  let rewrite_template = {|1234:[1]1234:[3]11:[2]|} in
-  all match_template source
-  |> Rewrite.all ~source ~rewrite_template
-  |> (function
-      | Some rewrite_result -> print_string (Yojson.Safe.pretty_to_string (Replacement.result_to_yojson rewrite_result))
-      | None -> print_string "BROKEN EXPECT");
-  [%expect_exact {|{
-  "rewritten_source": "123433312343331122",
-  "in_place_substitutions": [
-    {
-      "range": {
-        "start": { "offset": 0, "line": -1, "column": -1 },
-        "end": { "offset": 18, "line": -1, "column": -1 }
-      },
-      "replacement_content": "123433312343331122",
-      "environment": [
-        {
-          "variable": "1",
-          "value": "333",
-          "range": {
-            "start": { "offset": 4, "line": -1, "column": -1 },
-            "end": { "offset": 7, "line": -1, "column": -1 }
-          }
-        },
-        {
-          "variable": "2",
-          "value": "22",
-          "range": {
-            "start": { "offset": 16, "line": -1, "column": -1 },
-            "end": { "offset": 18, "line": -1, "column": -1 }
-          }
-        },
-        {
-          "variable": "3",
-          "value": "333",
-          "range": {
-            "start": { "offset": 11, "line": -1, "column": -1 },
-            "end": { "offset": 14, "line": -1, "column": -1 }
-          }
-        }
-      ]
-    }
-  ]
-}|}]
-
-let%expect_test "comments_in_string_literals_should_not_be_treated_as_comments_by_fuzzy" =
-  let source = {|123433312343331122;123433312343331122;|} in
-  let match_template = {|1234:[1]1234:[3]11:[2];|} in
-  let rewrite_template = {|1234:[1]1234:[3]11:[2];|} in
-  all match_template source
-  |> Rewrite.all ~source ~rewrite_template
-  |> (function
-      | Some rewrite_result -> print_string (Yojson.Safe.pretty_to_string (Replacement.result_to_yojson rewrite_result))
-      | None -> print_string "BROKEN EXPECT");
-  [%expect_exact {|{
-  "rewritten_source": "123433312343331122;123433312343331122;",
-  "in_place_substitutions": [
-    {
-      "range": {
-        "start": { "offset": 19, "line": -1, "column": -1 },
-        "end": { "offset": 38, "line": -1, "column": -1 }
-      },
-      "replacement_content": "123433312343331122;",
-      "environment": [
-        {
-          "variable": "1",
-          "value": "333",
-          "range": {
-            "start": { "offset": 4, "line": -1, "column": -1 },
-            "end": { "offset": 7, "line": -1, "column": -1 }
-          }
-        },
-        {
-          "variable": "2",
-          "value": "22",
-          "range": {
-            "start": { "offset": 16, "line": -1, "column": -1 },
-            "end": { "offset": 18, "line": -1, "column": -1 }
-          }
-        },
-        {
-          "variable": "3",
-          "value": "333",
-          "range": {
-            "start": { "offset": 11, "line": -1, "column": -1 },
-            "end": { "offset": 14, "line": -1, "column": -1 }
-          }
-        }
-      ]
-    },
-    {
-      "range": {
-        "start": { "offset": 0, "line": -1, "column": -1 },
-        "end": { "offset": 19, "line": -1, "column": -1 }
-      },
-      "replacement_content": "123433312343331122;",
-      "environment": [
-        {
-          "variable": "1",
-          "value": "333",
-          "range": {
-            "start": { "offset": 4, "line": -1, "column": -1 },
-            "end": { "offset": 7, "line": -1, "column": -1 }
-          }
-        },
-        {
-          "variable": "2",
-          "value": "22",
-          "range": {
-            "start": { "offset": 16, "line": -1, "column": -1 },
-            "end": { "offset": 18, "line": -1, "column": -1 }
-          }
-        },
-        {
-          "variable": "3",
-          "value": "333",
-          "range": {
-            "start": { "offset": 11, "line": -1, "column": -1 },
-            "end": { "offset": 14, "line": -1, "column": -1 }
-          }
-        }
-      ]
-    }
-  ]
-}|}]
-
-let%expect_test "multiple_contextual_substitutions" =
-  let source = {|foo bar foo|} in
-  let match_template = {|foo|} in
-  let rewrite_template = {|xxxx|} in
-  all match_template source
-  |> Rewrite.all ~source ~rewrite_template
-  |> (function
-      | Some rewrite_result -> print_string (Yojson.Safe.pretty_to_string (Replacement.result_to_yojson rewrite_result))
-      | None -> print_string "BROKEN EXPECT");
-  [%expect_exact {|{
-  "rewritten_source": "xxxx bar xxxx",
-  "in_place_substitutions": [
-    {
-      "range": {
-        "start": { "offset": 9, "line": -1, "column": -1 },
-        "end": { "offset": 13, "line": -1, "column": -1 }
-      },
-      "replacement_content": "xxxx",
-      "environment": []
-    },
-    {
-      "range": {
-        "start": { "offset": 0, "line": -1, "column": -1 },
-        "end": { "offset": 4, "line": -1, "column": -1 }
-      },
-      "replacement_content": "xxxx",
-      "environment": []
-    }
-  ]
-}|}]

test/common/test_substitute.ml

@@ -0,0 +1,31 @@
+open Core
+
+open Match
+open Rewriter
+open Rewrite_template
+
+let parse metasyntax template =
+  let (module M) = Matchers.Metasyntax.create metasyntax in
+  let module Template_parser = Make(M) in
+  let tree = Template_parser.parse template in
+  match tree with
+  | Some tree -> Sexp.to_string_hum (sexp_of_list sexp_of_extracted tree)
+  | None -> "ERROR: NO PARSE"
+
+let%expect_test "substitute_entire_regex_pattern_in_custom_metasyntax" =
+  let metasyntax =
+    Matchers.Metasyntax.{
+      syntax =
+        [ Hole (Everything, Delimited (Some "$", None))
+        ; Hole (Alphanum, Delimited (Some "$$", None))
+        ; Regex ("$", ':', " ")
+        ]
+    ; identifier = "AB"
+    }
+  in
+  (* Don't just substitute for `$B`, but for `$B:\w+ `. This depends on Regex (more specific syntax) being defined _after_ the general syntax. *)
+  let template = {|$A $B:\w+ |} in
+  let environment = Environment.add (Environment.create ()) "B" "hello" in
+  let result, _ = Rewrite_template.substitute ~metasyntax template environment in
+  print_string result;
+  [%expect_exact {|$A hello|}]

test/example/metasyntax/default-metasyntax.json

@@ -0,0 +1,13 @@
+{
+  "syntax": [
+    [ "Hole", [ "Everything" ], [ "Delimited", ":[", "]" ] ],
+    [ "Hole", [ "Expression" ], [ "Delimited", ":[", ":e]" ] ],
+    [ "Hole", [ "Alphanum" ], [ "Delimited", ":[[", "]]" ] ],
+    [ "Hole", [ "Non_space" ], [ "Delimited", ":[", ".]" ] ],
+    [ "Hole", [ "Line" ], [ "Delimited", ":[", "\\n]" ] ],
+    [ "Hole", [ "Blank" ], [ "Delimited", ":[ ", "]" ] ],
+    [ "Regex", ":[", "~", "]" ]
+  ],
+  "identifier":
+    "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_"
+}

test/example/metasyntax/dolla.json

@@ -0,0 +1,8 @@
+{
+  "syntax": [
+    [ "Hole", [ "Everything" ], [ "Delimited", "$", null ] ],
+    [ "Regex", "$", "~", "$" ] // order is significant!
+  ],
+  "identifier":
+    "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_"
+}