Add new matching engine (#127)

lib/matchers/alpha/matcher.ml

@@ -47,7 +47,7 @@ module Make (Syntax : Syntax.S) (Info : Info.S) = struct
      | Some { delimiters; escape_character } ->
        List.map delimiters ~f:(fun delimiter ->
            let module M =
-             Parsers.String_literals.Escapable.Make(struct
+             Parsers.String_literals.Alpha.Escapable.Make(struct
                let delimiter = delimiter
                let escape = escape_character
              end)
@@ -59,7 +59,7 @@ module Make (Syntax : Syntax.S) (Info : Info.S) = struct
   let raw_string_literal_parser (f : 'a literal_parser_callback) =
     List.map Syntax.raw_string_literals ~f:(fun (left_delimiter, right_delimiter) ->
         let module M =
-          Parsers.String_literals.Raw.Make(struct
+          Parsers.String_literals.Alpha.Raw.Make(struct
             let left_delimiter = left_delimiter
             let right_delimiter = right_delimiter
           end)
@@ -75,7 +75,7 @@ module Make (Syntax : Syntax.S) (Info : Info.S) = struct
       List.map syntax ~f:(function
           | Multiline (left, right) ->
             let module M =
-              Parsers.Comments.Multiline.Make(struct
+              Parsers.Comments.Alpha.Multiline.Make(struct
                 let left = left
                 let right = right
               end)
@@ -83,7 +83,7 @@ module Make (Syntax : Syntax.S) (Info : Info.S) = struct
             M.comment
           | Nested_multiline (left, right)  ->
             let module M =
-              Parsers.Comments.Nested_multiline.Make(struct
+              Parsers.Comments.Alpha.Nested_multiline.Make(struct
                 let left = left
                 let right = right
               end)
@@ -91,7 +91,7 @@ module Make (Syntax : Syntax.S) (Info : Info.S) = struct
             M.comment
           | Until_newline start ->
             let module M =
-              Parsers.Comments.Until_newline.Make(struct
+              Parsers.Comments.Alpha.Until_newline.Make(struct
                 let start = start
               end)
             in

lib/matchers/dune

@@ -1,5 +1,8 @@
+(copy_files# alpha/*.ml{,i})
+; (copy_files# omega/*.ml{,i})
+
 (library
   (name matchers)
   (public_name comby.matchers)
-  (preprocess (pps ppx_sexp_conv ppx_sexp_message ppx_deriving_yojson bisect_ppx --conditional))
-  (libraries comby.parsers comby.match ppxlib core core.uuid mparser mparser.pcre yojson ppx_deriving_yojson))
+  (preprocess (pps ppx_here ppx_sexp_conv ppx_sexp_message ppx_deriving_yojson bisect_ppx --conditional))
+  (libraries comby.parsers comby.match angstrom ppxlib core core.uuid mparser mparser.pcre yojson ppx_deriving_yojson))

lib/matchers/omega/matcher.ml

@@ -0,0 +1,510 @@
+open Core
+open Angstrom
+
+open Types
+open Omega
+
+let configuration_ref = ref (Configuration.create ())
+
+let matches_ref : Match.t list ref = ref []
+
+let source_ref : string ref = ref ""
+
+let current_environment_ref : Match.Environment.t ref = ref (Match.Environment.create ())
+
+let (|>>) p f =
+  p >>= fun x -> return (f x)
+
+let debug =
+  Sys.getenv "DEBUG_COMBY"
+  |> Option.is_some
+
+
+type signal_hole =
+  | Signal_a_hole of Types.hole
+  | Dont_care
+
+let record_match_context pos_before pos_after =
+  let open Match.Location in
+  if debug then Format.printf "match context start pos: %d@." pos_before;
+  if debug then Format.printf "match context end pos %d@." pos_after;
+  (* FIXME this may be slow. Try (a) collecting this
+     or (b) removing it by just doing a rewrite *)
+  let extract_matched_text source { offset = match_start; _ } { offset = match_end; _ } =
+    String.slice source match_start match_end
+  in
+  let match_context =
+    let match_start =
+      { default with offset = pos_before }
+    in
+    let match_end =
+      { default with offset = pos_after }
+    in
+    Match.
+      { range = { match_start; match_end }
+      ; environment = !current_environment_ref
+      ; matched = extract_matched_text !source_ref match_start match_end
+      }
+  in
+  matches_ref := match_context :: !matches_ref
+
+module Make (Syntax : Syntax.S) (Info : Info.S) = struct
+  include Info
+
+  (* This is the init we will pass in with a functor later *)
+  let acc = 0
+
+  (* This is the function we will pass in with a functor later *)
+  let f acc _production =
+    acc + 1
+
+  let r acc production : (signal_hole * 'a) t =
+    let open Match in
+    let open Range in
+    let acc = f acc production in
+    match production with
+    | String s ->
+      if debug then Format.printf "Matched String: %S@." s;
+      return (Dont_care, acc)
+    | Match { offset = pos_after; identifier; text = content } ->
+      (* using just pos after for now, because thats what we do in matcher. lol *)
+      if debug then Format.printf "Match: %S @@ %d for %s@." content pos_after identifier;
+      let before = Location.default in (* FIXME *)
+      let after = { Location.default with offset = pos_after } in
+      let range = { match_start = before; match_end = after } in
+      let environment = Environment.add ~range !current_environment_ref identifier content in
+      current_environment_ref := environment;
+      return (Dont_care, acc)
+    | _ -> return (Dont_care, acc)
+
+  let between left right p =
+    left *> p <* right
+
+  let zero =
+    fail ""
+
+  let comment_parser =
+    match Syntax.comments with
+    | [] -> zero
+    | syntax ->
+      List.map syntax ~f:(function
+          | Multiline (left, right) ->
+            let module M = Parsers.Comments.Omega.Multiline.Make(struct
+                let left = left
+                let right = right
+              end)
+            in
+            M.comment
+          | Until_newline start ->
+            let module M = Parsers.Comments.Omega.Until_newline.Make(struct
+                let start = start
+              end)
+            in
+            M.comment
+          | Nested_multiline (_, _) -> zero) (* FIXME unimplemented nested multiline comments *)
+      |> choice
+
+  let escapable_string_literal_parser =
+    (match Syntax.escapable_string_literals with
+     | None -> []
+     | Some { delimiters; escape_character } ->
+       List.map delimiters ~f:(fun delimiter ->
+           let module M =
+             Parsers.String_literals.Omega.Escapable.Make(struct
+               let delimiter = delimiter
+               let escape = escape_character
+             end)
+           in
+           M.base_string_literal >>= fun contents ->
+           (* FIXME figure out if we need to do the same callback thing here to communicate
+              forward that we entered a string *)
+           return contents
+         ))
+    |> choice
+
+  let until_of_from from =
+    Syntax.user_defined_delimiters
+    |> List.find_map ~f:(fun (from', until) -> if from = from' then Some until else None)
+    |> function
+    | Some until -> until
+    | None -> assert false
+
+  let alphanum =
+    satisfy (function
+        | 'a' .. 'z'
+        | 'A' .. 'Z'
+        | '0' .. '9' -> true
+        | _ -> false)
+
+  let is_whitespace = function
+    | ' ' | '\t' | '\r' | '\n' -> true
+    | _ -> false
+
+  let reserved_delimiters =
+    List.concat_map Syntax.user_defined_delimiters ~f:(fun (from, until) -> [from; until])
+    |> List.append [":["; "]"]
+    |> List.append [":[["; "]]"]
+
+  let reserved =
+    reserved_delimiters @ [" "; "\n"; "\t"; "\r"]
+    |> List.sort ~compare:(fun v2 v1 ->
+        String.length v1 - String.length v2)
+
+  (* XXX can shortcircuit *)
+  (* what if you hit a reserved
+     seqence "{" and then attempt
+     ":[[" and then say "end of
+     input" and then move ahead any_char. not good.
+     going from longest to shortest works though *)
+  let any_char_except ~reserved =
+    List.fold reserved
+      ~init:(return `OK)
+      ~f:(fun acc reserved_sequence ->
+          option `End_of_input
+            (peek_string (String.length reserved_sequence)
+             >>= fun s ->
+             if s = reserved_sequence then
+               return `Reserved_sequence
+             else
+               acc))
+    >>= function
+    | `OK -> any_char
+    | `End_of_input -> any_char
+    | `Reserved_sequence -> fail "reserved sequence hit"
+
+  let generate_single_hole_parser () =
+    (alphanum <|> char '_') |>> String.of_char
+
+  let generate_greedy_hole_parser
+      ?priority_left_delimiter:left_delimiter
+      ?priority_right_delimiter:right_delimiter
+      () =
+    let between_nested_delims p from =
+      let until = until_of_from from in
+      between (string from) (string until) p
+      >>= fun result -> return (String.concat @@ [from] @ result @ [until])
+    in
+    let between_nested_delims p =
+      (match left_delimiter, right_delimiter with
+       | Some left_delimiter, Some right_delimiter -> [ (left_delimiter, right_delimiter) ]
+       | _ -> Syntax.user_defined_delimiters)
+      |> List.map ~f:fst
+      |> List.map ~f:(between_nested_delims p)
+      |> choice
+    in
+    let reserved =
+      (match left_delimiter, right_delimiter with
+       | Some left_delimiter, Some right_delimiter -> [ (left_delimiter, right_delimiter) ]
+       | _ -> Syntax.user_defined_delimiters)
+      |> List.concat_map ~f:(fun (from, until) -> [from; until])
+    in
+    fix (fun grammar ->
+        let delimsx = between_nested_delims (many grammar) in
+        let other = any_char_except ~reserved |>> String.of_char in
+        (* FIXME holes does not handle space here, but does in alpha *)
+        choice
+          [ comment_parser
+          ; escapable_string_literal_parser
+          ; delimsx
+          ; other
+          ])
+
+  let cons x xs = x :: xs
+
+  let many_till p t =
+    fix (fun m -> (t *> return []) <|> (lift2 cons p m))
+
+  let many1_till p t =
+    lift2 cons p (many_till p t)
+
+  let sequence_chain (p_list : (signal_hole * 'a) t list) =
+    begin
+      let i = ref 0 in
+      List.fold_right p_list ~init:(return (Dont_care, acc)) ~f:(fun p acc ->
+          let result =
+            if debug then Format.printf "iterate fold_right %d@." !i;
+            match parse_string p "_signal_hole" with
+            | Error _ ->
+              if debug then Format.printf "Composing p with terminating parser@.";
+              p *> acc
+            | Ok (f,user_state) ->
+              (*Format.printf "Ok.@.";*)
+              match f with
+              | Signal_a_hole (Alphanum (identifier, _)) ->
+                pos >>= fun pos_before ->
+                many1 (generate_single_hole_parser ())
+                >>= fun value ->
+                (* acc must come after in order to sat. try mimic alpha to better express this. *)
+                acc >>= fun _ ->
+                r user_state
+                  (Match
+                     { offset = pos_before; identifier; text = (String.concat value) }
+                  )
+              | Signal_a_hole (Everything (identifier, _dimension)) ->
+                if debug then Format.printf "do hole %s@." identifier;
+                let first_pos = Set_once.create () in
+                let pparser =
+                  let until =
+                    (* if this is the base case (the first time we go around the
+                       loop backwards, when the first parser is a hole), then it
+                       means there's a hole at the end without anything following
+                       it in the template. So it should always match to
+                       end_of_input (not empty string) *)
+                    if !i = 0 then
+                      (if debug then Format.printf "Yes this case@.";
+                       end_of_input)
+                    else
+                      (if debug then Format.printf "Yes this second case@.";
+                       acc >>= fun _ -> return ())
+                  in
+                  (many_till
+                     (pos >>= fun pos -> Set_once.set_if_none first_pos [%here] pos;
+                      generate_greedy_hole_parser ())
+                     (pos >>= fun pos -> Set_once.set_if_none first_pos [%here] pos;
+                      until)
+                     (* it may be that the many till for the first parser
+                        succeeds on 'empty string', specifically in the :[1]:[2]
+                        case for :[1]. We won't capture the pos of :[1] in the
+                        first parser since it doesn't fire, so, so we have to
+                        set the pos right before the until parser below, if that
+                        happens. *)
+                  ) >>| String.concat
+                in
+                pparser >>= fun text ->
+                (*Format.printf "have results %d@." @@ List.length results;*)
+                let offset =
+                  match Set_once.get first_pos with
+                  | Some offset -> offset
+                  | _ -> failwith "Did not expect unset offset"
+                in
+                r
+                  user_state
+                  (Match
+                     { offset
+                     ; identifier
+                     ; text
+                     })
+              | _ -> assert false
+          in
+          i := !i + 1;
+          result)
+    end
+
+  (** must have at least one, otherwise spins on
+      the empty string *)
+  let spaces1 =
+    satisfy is_whitespace >>= fun c ->
+    (* XXX use skip_while once everything works.
+       we don't need the string *)
+    take_while is_whitespace >>= fun s ->
+    return (Format.sprintf "%c%s" c s)
+
+  let spaces =
+    take_while is_whitespace >>= fun s ->
+    return s
+
+  (* XXX change ignore to unit once everything works.
+     right now it's the string that was parsed by spaces1 *)
+  let generate_spaces_parser _ignored =
+    (* XXX still some parts ignored in the choice case in Alpha *)
+    if debug then Format.printf "Template_spaces(%s)@." _ignored;
+    spaces1 *> many comment_parser <* spaces
+    >>= fun result -> r acc (String (String.concat result))
+
+  (** All code can have comments interpolated *)
+  let generate_string_token_parser str =
+    if debug then Format.printf "Template_string(%s)@." str;
+    many comment_parser
+    *> string str
+    *> many comment_parser
+    >>= fun result -> r acc (String (String.concat result))
+
+  let generate_single_hole_parser () =
+    (alphanum <|> char '_') |>> String.of_char
+
+  let skip_unit p =
+    p |>> ignore
+
+  let identifier_parser () =
+    many (alphanum <|> char '_')
+    |>> String.of_char_list
+
+  let single_hole_parser () =
+    string ":[[" *> identifier_parser () <* string "]]"
+
+  let greedy_hole_parser () =
+    string ":[" *> identifier_parser () <* string "]"
+
+  let many1_till p t =
+    let cons x xs = x::xs in
+    lift2 cons p (many_till p t)
+
+  let hole_parser sort dimension : (signal_hole * 'a) t t =
+    match sort with
+    | `Single ->
+      single_hole_parser () |>> fun id ->
+      skip_unit (string "_signal_hole") |>> fun () ->
+      (Signal_a_hole (Alphanum (id, dimension)), acc)
+    | `Greedy ->
+      greedy_hole_parser () |>> fun id ->
+      skip_unit (string "_signal_hole") |>> fun () ->
+      (Signal_a_hole (Everything (id, dimension)), acc)
+
+  let general_parser_generator : (signal_hole * 'a) t t =
+    fix (fun (generator : (signal_hole * 'a) t list t) ->
+        if debug then Format.printf "Descends@.";
+        let nested =
+          (* FIXME nested needs comments and string literals (or does it not
+             need one for string literals because we handle it in fix? Unsure,
+             couldn't come up with a CLI test. Check against test suite. *)
+          if debug then Format.printf "Nested@.";
+          Syntax.user_defined_delimiters
+          |> List.map ~f:(fun (left_delimiter, right_delimiter) ->
+              (string left_delimiter
+               *> generator
+               <* string right_delimiter)
+              >>= fun (g: (signal_hole * 'a) t list) ->
+              if debug then Format.printf "G size: %d; delim %s@." (List.length g) left_delimiter;
+              (([string left_delimiter
+                 >>= fun result -> r acc (String result)]
+                @ g
+                @ [ string right_delimiter
+                    >>= fun result -> r acc (String result)])
+               |>
+               sequence_chain)
+              |> return)
+          |> choice
+        in
+        let spaces : (signal_hole * 'a) t t= spaces1 |>> generate_spaces_parser in
+        let escapable_string_literal_parser : (signal_hole * 'a) t t =
+          escapable_string_literal_parser
+          >>| fun string_literal_contents ->
+          (* FIXME incomplete likely, may need info about delims. also, no hole
+             matching yet. *)
+          generate_string_token_parser string_literal_contents
+        in
+        let other =
+          (* XXX many1_till would be cool, but it also parses the thing that
+             causes it to fail, which i need restored.  many_till is 'parse and
+             include the parse of the exception', whereas I want parse and
+             exclude the parse of the exception (hard to reintroduce ) *)
+   (*
+   (many1_till (any_char >>= fun c -> Format.printf "parsed %c@." c;
+   return c) (List.map reserved ~f:string |> choice >>= fun x ->
+   Format.printf "Fail on %s@." x; return x) |>> fun s ->
+   Format.printf "Chars: %s@." @@ String.of_char_list s;
+   String.of_char_list s) *)
+          (many1 (any_char_except ~reserved) |>> String.of_char_list)
+          |>> fun x ->
+          if debug then Format.printf "Other: %s@." x;
+          generate_string_token_parser x
+        in
+        if debug then Format.printf "Many... @.";
+        (* can't be many1 because then {} isn't a valid template (delimiters have to
+           contain something then and can't be empty *)
+        (* don't want it to be many because empty string will satisfy and
+           "" is a valid template, or even "{", because it generates 'seq' on chain *)
+        many @@
+        choice
+          [ hole_parser `Single Code
+          ; hole_parser `Greedy Code
+          ; escapable_string_literal_parser
+          ; spaces
+          ; nested
+          ; other
+          ]
+        >>= fun x ->
+        if debug then Format.printf "Produced %d parsers in main generator@." @@ List.length x;
+        return x
+      )
+    |>> fun p_list ->
+    p_list
+    |> sequence_chain
+    |> fun matcher ->
+    (* FIXME: skip_unit needs to be raw literals *)
+    (* XXX: what is the difference does many vs many1 make here? Semantically,
+       it should mean "0 or more matching contexts" vs "1 or more matching
+       contexts". We only care about the 1 case anyway, so... *)
+    many (skip_unit
+            (many_till (skip_unit comment_parser
+                        <|> skip_unit escapable_string_literal_parser
+                        <|> skip_unit any_char)
+               (
+                 at_end_of_input >>= fun res ->
+                 if debug then Format.printf "We are at the end? %b.@." res;
+                 if res then
+                   (if debug then Format.printf "We ended@.";
+                    fail "x")
+                 else
+                   (* we found a match *)
+                   pos >>= fun start_pos ->
+                   matcher >>= fun _access_last_production_herpe ->
+                   pos >>= fun end_pos ->
+                   record_match_context start_pos end_pos;
+                   current_environment_ref := Match.Environment.create ();
+                   return Unit)
+               (*<|>
+                 (end_of_input >>= fun () -> return Unit)*)
+            )
+         )
+    (*>>= fun _x -> end_of_input *)
+    >>= fun _x -> r acc Unit
+
+  let to_template template =
+    let state = Buffered.parse general_parser_generator in
+    let state = Buffered.feed state (`String template) in
+    Buffered.feed state `Eof
+    |> function
+    | Buffered.Done ({ len; _ }, p) ->
+      if len <> 0 then failwith @@
+        Format.sprintf "Input left over in template where not expected: %d" len;
+      Ok p
+    | _ -> Or_error.error_string "Template could not be parsed."
+
+  let run_the_parser_for_first p source : Match.t Or_error.t =
+    source_ref := source;
+    let state = Buffered.parse p in
+    let state = Buffered.feed state (`String source) in
+    let state = Buffered.feed state `Eof in
+    match state with
+    | Buffered.Done ({ len; off; _ }, _result) ->
+      if len <> 0 then
+        (if debug then
+           Format.eprintf "Input left over in parse where not expected: off(%d) len(%d)" off len;
+         Or_error.error_string "Does not match tempalte")
+      else
+        Ok (Match.create ()) (* Fake for now *)
+    | _ -> Or_error.error_string "No matches"
+
+  let first_broken ?configuration:_ ?shift:_ template source : Match.t Or_error.t =
+    match to_template template with
+    | Ok p ->
+      begin match run_the_parser_for_first p source with
+        | Ok _ -> (* matches passed, ok to access *)
+          begin
+            match !matches_ref with
+            | [] -> Or_error.error_string "not really"
+            | hd::_ -> Ok hd
+          end
+        | Error e -> (* parse failed *)
+          Error e
+      end
+    | Error e ->
+      Format.printf "Template FAIL %s@." @@ Error.to_string_hum e;
+      Error e
+
+  let all ?configuration:_ ~template ~source : Match.t list =
+    matches_ref := [];
+    match first_broken template source with
+    | Ok _
+    | Error _ -> List.rev !matches_ref
+
+  let first ?configuration ?shift:_ template source : Match.t Or_error.t =
+    matches_ref := [];
+    match all ?configuration ~template ~source with
+    | [] -> Or_error.error_string "nothing"
+    | (hd::_) as m ->
+      if debug then List.iter m ~f:(fun { environment; _ } ->
+          Format.printf "START:@.%s@.END@." (Match.Environment.to_string environment));
+      Ok hd
+end

lib/matchers/types.ml

@@ -55,11 +55,24 @@ type hole =
   | Line of (id * dimension)
   | Blank of (id * dimension)
 
+module Omega = struct
+  type omega_match_production =
+    { offset : int
+    ; identifier : string
+    ; text : string
+    }
+
+  type production =
+    | Unit
+    | String of string
+    | Hole of hole
+    | Match of omega_match_production
+end
+
 type production =
   | Unit
   | String of string
   | Hole of hole
-  | Match of (int * string * string)
 
 module Matcher = struct
   module type S = sig

lib/parsers/comments.ml

@@ -1,94 +1,155 @@
 open Core
-open MParser
 
-let to_string from until between : string =
-  from ^ (String.of_char_list between) ^ until
+module Omega = struct
+  open Angstrom
 
-let anything_including_newlines ~until =
-  (many
-     (not_followed_by (string until) ""
-      >>= fun () -> any_char_or_nl))
+  let (|>>) p f =
+    p >>= fun x -> return (f x)
 
-let anything_excluding_newlines ~until =
-  (many
-     (not_followed_by (string until) ""
-      >>= fun () -> any_char))
+  let between left _right p =
+    left *> p (*<* right*)
 
-(** a parser for comments with delimiters [from] and [until] that do not nest *)
-let non_nested_comment from until s =
-  (between
-     (string from)
-     (string until)
-     (anything_including_newlines ~until)
-   |>> to_string from until
-  ) s
+  let to_string from until between : string =
+    from ^ (String.of_char_list between) ^ until
 
-let until_newline start s =
-  (string start >> anything_excluding_newlines ~until:"\n"
-   |>> fun l -> start^(String.of_char_list l)) s
+  let anything_including_newlines ~until =
+    (* until is not consumed in Alpha. Angstrom consumes it. It needs to not
+       consume because we're doing that for 'between'; but now between is
+       changed to not expect it. The point is: it does the right thing but
+       diverges from Alpha and is a little bit... weird *)
+    (many_till any_char (string until))
 
-let any_newline comment_string s =
-  (string comment_string >> anything_excluding_newlines ~until:"\n" |>> fun l -> (comment_string^String.of_char_list l)) s
+  let anything_excluding_newlines () =
+    anything_including_newlines ~until:"\n"
 
-let is_not p s =
-  if is_ok (p s) then
-    Empty_failed (unknown_error s)
-  else
-    match read_char s with
-    | Some c ->
-      Consumed_ok (c, advance_state s 1, No_error)
-    | None ->
-      Empty_failed (unknown_error s)
+  let non_nested_comment from until =
+    between
+      (string from)
+      (string until)
+      (anything_including_newlines ~until)
+    |>> to_string from until
 
-(** A nested comment parser *)
-let nested_comment from until s =
-  let reserved = skip ((string from) <|> (string until)) in
-  let rec grammar s =
-    ((comment_delimiters >>= fun string -> return string)
-     <|>
-     (is_not reserved >>= fun c -> return (Char.to_string c)))
-      s
 
-  and comment_delimiters s =
+  module Multiline = struct
+    module type S = sig
+      val left : string
+      val right : string
+    end
+
+    module Make (M : S) = struct
+      let comment = non_nested_comment M.left M.right
+    end
+  end
+
+  (* XXX consumes the newline *)
+  let until_newline start =
+    (string start *> anything_excluding_newlines ()
+     |>> fun l -> start^(String.of_char_list l))
+
+  module Until_newline = struct
+    module type S = sig
+      val start : string
+    end
+
+    module Make (M : S) = struct
+      let comment = until_newline M.start
+    end
+  end
+
+end
+
+module Alpha = struct
+  open MParser
+
+  let to_string from until between : string =
+    from ^ (String.of_char_list between) ^ until
+
+  let anything_including_newlines ~until =
+    (many
+       (not_followed_by (string until) ""
+        >>= fun () -> any_char_or_nl))
+
+  let anything_excluding_newlines ~until =
+    (many
+       (not_followed_by (string until) ""
+        >>= fun () -> any_char))
+
+  (** a parser for comments with delimiters [from] and [until] that do not nest *)
+  let non_nested_comment from until s =
     (between
        (string from)
        (string until)
-       ((many grammar) >>= fun result ->
-        return (String.concat result)))
-      s
-  in
-  (comment_delimiters |>> fun content ->
-   from ^ content ^ until) s
+       (anything_including_newlines ~until)
+     |>> to_string from until
+    ) s
 
-(** a parser for, e.g., /* ... */ style block comments. Non-nested. *)
-module Multiline = struct
-  module type S = sig
-    val left : string
-    val right : string
+  let until_newline start s =
+    (string start >> anything_excluding_newlines ~until:"\n"
+     |>> fun l -> start^(String.of_char_list l)) s
+
+  let any_newline comment_string s =
+    (string comment_string >> anything_excluding_newlines ~until:"\n" |>> fun l -> (comment_string^String.of_char_list l)) s
+
+  let is_not p s =
+    if is_ok (p s) then
+      Empty_failed (unknown_error s)
+    else
+      match read_char s with
+      | Some c ->
+        Consumed_ok (c, advance_state s 1, No_error)
+      | None ->
+        Empty_failed (unknown_error s)
+
+  (** A nested comment parser *)
+  let nested_comment from until s =
+    let reserved = skip ((string from) <|> (string until)) in
+    let rec grammar s =
+      ((comment_delimiters >>= fun string -> return string)
+       <|>
+       (is_not reserved >>= fun c -> return (Char.to_string c)))
+        s
+
+    and comment_delimiters s =
+      (between
+         (string from)
+         (string until)
+         ((many grammar) >>= fun result ->
+          return (String.concat result)))
+        s
+    in
+    (comment_delimiters |>> fun content ->
+     from ^ content ^ until) s
+
+  (** a parser for, e.g., /* ... */ style block comments. Non-nested. *)
+  module Multiline = struct
+    module type S = sig
+      val left : string
+      val right : string
+    end
+
+    module Make (M : S) = struct
+      let comment s = non_nested_comment M.left M.right s
+    end
   end
 
-  module Make (M : S) = struct
-    let comment s = non_nested_comment M.left M.right s
-  end
-end
-
-module Until_newline = struct
-  module type S = sig
-    val start : string
-  end
-
-  module Make (M : S) = struct
-    let comment s = until_newline M.start s
-  end
-end
-
-module Nested_multiline = struct
-  module type S = sig
-    val left : string
-    val right : string
-  end
-
-  module Make (M : S) = struct
-    let comment s = nested_comment M.left M.right s
+  module Until_newline = struct
+    module type S = sig
+      val start : string
+    end
+
+    module Make (M : S) = struct
+      let comment s = until_newline M.start s
+    end
+  end
+
+  module Nested_multiline = struct
+    module type S = sig
+      val left : string
+      val right : string
+    end
+
+    module Make (M : S) = struct
+      let comment s = nested_comment M.left M.right s
+    end
   end
 end

lib/parsers/dune

@@ -2,4 +2,4 @@
   (name parsers)
   (public_name comby.parsers)
   (preprocess (pps ppx_sexp_conv bisect_ppx --conditional))
-  (libraries ppxlib core mparser mparser.pcre))
+  (libraries angstrom ppxlib core mparser mparser.pcre))

lib/parsers/string_literals.ml

@@ -1,57 +1,94 @@
 open Core
-open MParser
 
-(** Assumes the left and right delimiter are the same, and that these can be
-    escaped. Does not parse a string body containing newlines (as usual when
-    escaping with \n) *)
-module Escapable = struct
-  module type S = sig
-    val delimiter : string
-    val escape : char
-  end
+module Omega = struct
+  open Angstrom
 
-  module Make (M : S) = struct
-    (* delimiters can be escaped and parsing continues within the string body *)
-    let escaped_char_s s =
-      any_char s
+  let (|>>) p f =
+    p >>= fun x -> return (f x)
 
-    let char_token_s s =
-      ((char M.escape >> escaped_char_s >>= fun c -> return (Format.sprintf {|%c%c|} M.escape c))
-       <|> (any_char |>> String.of_char)
-      )
-        s
+  module Escapable = struct
+    module type S = sig
+      val delimiter : string
+      val escape : char
+    end
 
-    let base_string_literal s =
-      ((string M.delimiter >> (many_until char_token_s (string M.delimiter))
-        |>> String.concat)
-       >>= fun result ->
-       return (Format.sprintf {|%s%s%s|} M.delimiter result M.delimiter)
-      )
-        s
+    module Make (M : S) = struct
+      (* delimiters can be escaped and parsing continues within the string body *)
+      let escaped_char_s  =
+        any_char
+
+      let char_token_s =
+        ((char M.escape *> escaped_char_s >>= fun c -> return (Format.sprintf {|%c%c|} M.escape c))
+         <|> (any_char |>> String.of_char)
+        )
+
+
+      let base_string_literal =
+        ((string M.delimiter *> (many_till char_token_s (string M.delimiter))
+          |>> String.concat)
+         >>= fun result ->
+         return (Format.sprintf {|%s%s|} M.delimiter result)
+         (* unlike Alpha, do not suffix the ending delimiter, it was captured by the delimiter parser in many_till *)
+        )
+    end
   end
 end
 
-(** Quoted or raw strings. Allows different left and right delimiters, and
-    disallows any sort of escaping. Does not support raw strings with identifiers
-    yet, e.g., {blah|<string body>|blah} (OCaml) or delim`<string body>`delim
-    syntax (Go) *)
-module Raw = struct
-  module type S = sig
-    val left_delimiter : string
-    val right_delimiter : string
+module Alpha = struct
+  open MParser
+
+  (** Assumes the left and right delimiter are the same, and that these can be
+      escaped. Does not parse a string body containing newlines (as usual when
+      escaping with \n) *)
+  module Escapable = struct
+    module type S = sig
+      val delimiter : string
+      val escape : char
+    end
+
+    module Make (M : S) = struct
+      (* delimiters can be escaped and parsing continues within the string body *)
+      let escaped_char_s s =
+        any_char s
+
+      let char_token_s s =
+        ((char M.escape >> escaped_char_s >>= fun c -> return (Format.sprintf {|%c%c|} M.escape c))
+         <|> (any_char |>> String.of_char)
+        )
+          s
+
+      let base_string_literal s =
+        ((string M.delimiter >> (many_until char_token_s (string M.delimiter))
+          |>> String.concat)
+         >>= fun result ->
+         return (Format.sprintf {|%s%s%s|} M.delimiter result M.delimiter)
+        )
+          s
+    end
   end
 
-  module Make (M : S) = struct
-    let char_token_s s =
-      (any_char_or_nl |>> String.of_char) s
+  (** Quoted or raw strings. Allows different left and right delimiters, and
+      disallows any sort of escaping. Does not support raw strings with identifiers
+      yet, e.g., {blah|<string body>|blah} (OCaml) or delim`<string body>`delim
+      syntax (Go) *)
+  module Raw = struct
+    module type S = sig
+      val left_delimiter : string
+      val right_delimiter : string
+    end
 
-    let base_string_literal s =
-      ((
-        string M.left_delimiter >> (many_until char_token_s (string M.right_delimiter))
-        |>> String.concat <?> "raw string literal body")
-       >>= fun result ->
-       return (Format.sprintf {|%s%s%s|} M.left_delimiter result M.right_delimiter)
-      )
-        s
+    module Make (M : S) = struct
+      let char_token_s s =
+        (any_char_or_nl |>> String.of_char) s
+
+      let base_string_literal s =
+        ((
+          string M.left_delimiter >> (many_until char_token_s (string M.right_delimiter))
+          |>> String.concat <?> "raw string literal body")
+         >>= fun result ->
+         return (Format.sprintf {|%s%s%s|} M.left_delimiter result M.right_delimiter)
+        )
+          s
+    end
   end
 end

lib/rewriter/rewrite.ml

@@ -3,7 +3,14 @@ open Core
 open Match
 open Replacement
 
+let debug =
+  Sys.getenv "DEBUG_COMBY"
+  |> Option.is_some
+
+
 let substitute_match_contexts (matches: Match.t list) source replacements =
+  if debug then
+    Format.printf "Matches: %d | Replacements: %d@." (List.length matches) (List.length replacements);
   let rewrite_template, environment =
     List.fold2_exn
       matches replacements
@@ -13,14 +20,21 @@ let substitute_match_contexts (matches: Match.t list) source replacements =
            { replacement_content; _ } ->
            (* create a hole in the rewrite template based on this match context *)
            let hole_id, rewrite_template = Rewrite_template.of_match_context match_ ~source:rewrite_template in
+           if debug then Format.printf "Hole: %s in %s@." hole_id rewrite_template;
            (* add this match context replacement to the environment *)
            let accumulator_environment = Environment.add accumulator_environment hole_id replacement_content in
            (* update match context replacements offset *)
            rewrite_template, accumulator_environment)
   in
+  if debug then Format.printf "Env:@.%s" (Environment.to_string environment);
+  if debug then Format.printf "Rewrite in:@.%s@." rewrite_template;
   let rewritten_source = Rewrite_template.substitute rewrite_template environment |> fst in
   let offsets = Rewrite_template.get_offsets_for_holes rewrite_template (Environment.vars environment) in
+  if debug then
+    Format.printf "Replacements: %d | Offsets 1: %d@." (List.length replacements) (List.length offsets);
   let offsets = Rewrite_template.get_offsets_after_substitution offsets environment in
+  if debug then
+    Format.printf "Replacements: %d | Offsets 2: %d@." (List.length replacements) (List.length offsets);
   let in_place_substitutions =
     List.map2_exn replacements offsets ~f:(fun replacement (_uid, offset) ->
         let match_start = { Location.default with offset } in

lib/rewriter/rewrite_template.ml

@@ -2,6 +2,11 @@ open Core
 
 open Match
 
+let debug =
+  Sys.getenv "DEBUG_COMBY"
+  |> Option.is_some
+
+
 let substitute template env =
   let substitution_formats =
     [ ":[ ", "]"
@@ -31,6 +36,7 @@ let of_match_context
     ; _
     }
     ~source =
+  if debug then Format.printf "Start idx: %d@.End idx: %d@." start_index end_index;
   let before_part =
     if start_index = 0 then
       ""

test/alpha/dune

@@ -0,0 +1,27 @@
+(library
+ (name alpha_test_integration)
+ (modules
+  test_match_rule
+  test_hole_extensions
+  test_python_string_literals
+  test_integration
+  test_statistics
+  test_cli
+  test_c_style_comments
+  test_string_literals
+  test_special_matcher_cases
+  test_generic
+  test_rewrite_rule
+  test_server
+  test_substring_disabled)
+ (inline_tests)
+ (preprocess (pps ppx_expect ppx_sexp_message ppx_deriving_yojson ppx_deriving_yojson.runtime))
+ (libraries
+  comby
+  cohttp-lwt-unix
+  core
+  camlzip))
+
+(alias
+(name runtest)
+(deps (source_tree example) (source_tree example/src/.ignore-me)))

test/alpha/example

@@ -0,0 +1 @@
+../example

test/alpha/test_cli.ml

@@ -0,0 +1,944 @@
+open Core
+
+module Time = Core_kernel.Time_ns.Span
+
+let binary_path = "../../../../comby"
+
+let read_with_timeout read_from_channels =
+  let read_from_fds = List.map ~f:Unix.descr_of_in_channel read_from_channels in
+  let read_from_channels =
+    Unix.select
+      ~restart:true
+      ~read:read_from_fds
+      ~write:[]
+      ~except:[]
+      ~timeout:(`After (Time.of_int_sec 1))
+      ()
+    |> (fun { Unix.Select_fds.read; _ } -> read)
+    |> List.map ~f:Unix.in_channel_of_descr
+  in
+  List.map read_from_channels ~f:In_channel.input_all
+  |> String.concat ~sep:"\n"
+
+let read_output command =
+  let open Unix.Process_channels in
+  let { stdout; stderr; _ } =
+    Unix.open_process_full ~env:(Array.of_list ["COMBY_TEST=1"]) command
+  in
+  let stdout_result = In_channel.input_all stdout in
+  let stderr_result = In_channel.input_all stderr in
+  stdout_result ^ stderr_result
+
+let read_expect_stdin_and_stdout command source =
+  let open Unix.Process_channels in
+  let { stdin; stdout; stderr } =
+    Unix.open_process_full ~env:(Array.of_list ["COMBY_TEST=1"]) command
+  in
+  Out_channel.output_string stdin source;
+  Out_channel.flush stdin;
+  Out_channel.close stdin;
+  let stdout_result = In_channel.input_all stdout in
+  let stderr_result = In_channel.input_all stderr in
+  stdout_result ^ stderr_result
+
+let read_expect_stderr command source =
+  let open Unix.Process_channels in
+  let { stdin; stdout; stderr } =
+    Unix.open_process_full ~env:(Array.of_list ["COMBY_TEST=1"]) command
+  in
+  Out_channel.output_string stdin source;
+  Out_channel.flush stdin;
+  Out_channel.close stdin;
+  let _ = In_channel.input_all stdout in
+  let stderr_result = In_channel.input_all stderr in
+  stderr_result
+
+let%expect_test "json_lines_separates_by_line" =
+  let source = "hello world" in
+  let match_template = "o" in
+  let rewrite_template = "i" in
+  let command_args =
+    Format.sprintf "-stdin -sequential '%s' '%s' -f .c -json-lines" match_template rewrite_template
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect_exact {|{"uri":null,"rewritten_source":"helli wirld","in_place_substitutions":[{"range":{"start":{"offset":7,"line":-1,"column":-1},"end":{"offset":8,"line":-1,"column":-1}},"replacement_content":"i","environment":[]},{"range":{"start":{"offset":4,"line":-1,"column":-1},"end":{"offset":5,"line":-1,"column":-1}},"replacement_content":"i","environment":[]}],"diff":"--- /dev/null\n+++ /dev/null\n@@ -1,1 +1,1 @@\n-hello world\n+helli wirld"}
+|}]
+
+let%expect_test "json_lines_json_pretty_do_not_output_when_diff_null" =
+  let source = "hello world" in
+  let match_template = "asdf" in
+  let rewrite_template = "asdf" in
+  let command_args =
+    Format.sprintf "-stdin -sequential '%s' '%s' -f .c -json-lines" match_template rewrite_template
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect{| |}]
+
+let%expect_test "json_lines_do_not_output_when_diff_null" =
+  let source = "hello world" in
+  let match_template = "asdf" in
+  let rewrite_template = "asdf" in
+  let command_args =
+    Format.sprintf "-stdin -sequential '%s' '%s' -f .c -json-lines" match_template rewrite_template
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect{| |}]
+
+let%expect_test "error_on_zip_and_stdin" =
+  let command_args = "-zip x -stdin" in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command "none" in
+  print_string result;
+  [%expect_exact {|No templates specified. See -h to specify on the command line, or use -templates <directory-containing-templates>.
+Next error: -zip may not be used with -stdin.
+|}]
+
+let%expect_test "error_on_stdout_and_diff" =
+  let command_args = "'' '' -stdout -diff" in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command "none" in
+  print_string result;
+  [%expect_exact {|-stdout may not be used with -diff. Note: -stdout outputs the changed file contents and -diff outputs a unified diff. Choose one of these.
+|}]
+
+let%expect_test "error_on_invalid_templates_dir" =
+  let source = "hello world" in
+  let match_template = "hello :[1]" in
+  let rewrite_template = ":[1]" in
+  let command_args =
+    Format.sprintf "-stdin -sequential '%s' '%s' -f .c -templates nonexistent" match_template rewrite_template
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect_exact {|One or more directories specified with -templates is not a directory.
+|}]
+
+let%expect_test "warn_on_anonymous_and_templates_flag" =
+  let source = "hello world" in
+  let match_template = "hello :[1]" in
+  let rewrite_template = ":[1]" in
+  let command_args =
+    Format.sprintf "-stdin -sequential '%s' '%s' -f .c -templates example/templates/identity" match_template rewrite_template
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect_exact {|WARNING: Templates specified on the command line AND using -templates. Ignoring match
+      and rewrite templates on the command line and only using those in directories.
+|}]
+
+
+let%expect_test "stdin_command" =
+  let source = "hello world" in
+  let match_template = "hello :[1]" in
+  let rewrite_template = ":[1]" in
+  let command_args =
+    Format.sprintf "-stdin -sequential '%s' '%s' -f .c" match_template rewrite_template
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect_exact {|------ /dev/null
+++++++ /dev/null
+@|-1,1 +1,1 ============================================================
+!|hello world
+|}]
+
+let%expect_test "with_match_rule" =
+  let source = "hello world" in
+  let match_template = "hello :[1]" in
+  let rewrite_template = ":[1]" in
+  let rule = {|where :[1] == "world"|} in
+  let command_args =
+    Format.sprintf "-stdin -sequential '%s' '%s' -rule '%s' -f .c "
+      match_template rewrite_template rule
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect_exact {|------ /dev/null
+++++++ /dev/null
+@|-1,1 +1,1 ============================================================
+!|hello world
+|}];
+
+  let source = "hello world" in
+  let match_template = "hello :[1]" in
+  let rewrite_template = ":[1]" in
+  let rule = {|where :[1] != "world"|} in
+  let command_args =
+    Format.sprintf "-stdin -sequential '%s' '%s' -rule '%s' -f .c -stdout"
+      match_template rewrite_template rule
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect{|
+    hello world |}]
+
+let%expect_test "with_rewrite_rule" =
+  let source = "hello world" in
+  let match_template = ":[2] :[1]" in
+  let rewrite_template = ":[1]" in
+  let rule = {|where rewrite :[1] { ":[_]" -> ":[2]" }|} in
+  let command_args =
+    Format.sprintf "-stdin -sequential '%s' '%s' -rule '%s' -f .c "
+      match_template rewrite_template rule
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect_exact {|------ /dev/null
+++++++ /dev/null
+@|-1,1 +1,1 ============================================================
+!|hello world
+|}]
+
+let%expect_test "with_rewrite_rule_stdin_default_no_extension" =
+  let source = "hello world" in
+  let match_template = ":[2] :[1]" in
+  let rewrite_template = ":[1]" in
+  let rule = {|where rewrite :[1] { ":[_]" -> ":[2]" }|} in
+  let command_args =
+    Format.sprintf "-sequential '%s' '%s' -rule '%s' -stdin" match_template rewrite_template rule
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect_exact {|------ /dev/null
+++++++ /dev/null
+@|-1,1 +1,1 ============================================================
+!|hello world
+
+WARNING: the GENERIC matcher was used, because a language could not be inferred from the file extension(s). The GENERIC matcher may miss matches. See '-list' to set a matcher for a specific language and to remove this warning.
+|}]
+
+let%expect_test "generic_matcher" =
+  let source = {|\footnote{\small \url{https://github.com}}|} in
+  let match_template = {|\footnote{\small :[1]}|} in
+  let rewrite_template = {|\footnote{\scriptsize :[1]}|} in
+  let command_args =
+    Format.sprintf "-stdin -sequential '%s' '%s' -f .generic" match_template rewrite_template
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect_exact {|------ /dev/null
+++++++ /dev/null
+@|-1,1 +1,1 ============================================================
+-|\footnote{\small \url{https://github.com}}
++|\footnote{\scriptsize \url{https://github.com}}
+
+WARNING: the GENERIC matcher was used, because a language could not be inferred from the file extension(s). The GENERIC matcher may miss matches. See '-list' to set a matcher for a specific language and to remove this warning.
+|}]
+
+
+let%expect_test "json_output_option" =
+  let source = "a X c a Y c" in
+  let match_template = "a :[1] c" in
+  let rewrite_template = "c :[1] a" in
+  let command_args =
+    Format.sprintf "-stdin -sequential -json-lines '%s' '%s' -f .c "
+      match_template rewrite_template
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect_exact {|{"uri":null,"rewritten_source":"c X a c Y a","in_place_substitutions":[{"range":{"start":{"offset":6,"line":-1,"column":-1},"end":{"offset":11,"line":-1,"column":-1}},"replacement_content":"c Y a","environment":[{"variable":"1","value":"Y","range":{"start":{"offset":2,"line":-1,"column":-1},"end":{"offset":3,"line":-1,"column":-1}}}]},{"range":{"start":{"offset":0,"line":-1,"column":-1},"end":{"offset":5,"line":-1,"column":-1}},"replacement_content":"c X a","environment":[{"variable":"1","value":"X","range":{"start":{"offset":2,"line":-1,"column":-1},"end":{"offset":3,"line":-1,"column":-1}}}]}],"diff":"--- /dev/null\n+++ /dev/null\n@@ -1,1 +1,1 @@\n-a X c a Y c\n+c X a c Y a"}
+|}];
+
+  let source = "a X c a Y c" in
+  let match_template = "a :[1] c" in
+  let rewrite_template = "c :[1] a" in
+  let command_args =
+    Format.sprintf "-stdin -sequential -json-lines -match-only '%s' '%s' -f .c "
+      match_template rewrite_template
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect_exact {|{"uri":null,"matches":[{"range":{"start":{"offset":0,"line":1,"column":1},"end":{"offset":5,"line":1,"column":6}},"environment":[{"variable":"1","value":"X","range":{"start":{"offset":2,"line":1,"column":3},"end":{"offset":3,"line":1,"column":4}}}],"matched":"a X c"},{"range":{"start":{"offset":6,"line":1,"column":7},"end":{"offset":11,"line":1,"column":12}},"environment":[{"variable":"1","value":"Y","range":{"start":{"offset":8,"line":1,"column":9},"end":{"offset":9,"line":1,"column":10}}}],"matched":"a Y c"}]}|}]
+
+let with_zip f =
+  let file = Filename.temp_file "comby_" ".zip" in
+  let zip =  Zip.open_out file in
+  let entry_name = "main.ml" in
+  let entry_content = "hello world" in
+  Zip.add_entry entry_content zip entry_name;
+  Zip.close_out zip;
+  f file;
+  Unix.remove file
+
+let%expect_test "list_languages" =
+  let command_args = "-list" in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_output command in
+  print_string result;
+  [%expect_exact {|Option              Language  
+ -matcher .s        Assembly  
+ -matcher .sh       Bash      
+ -matcher .c        C         
+ -matcher .cs       C#        
+ -matcher .css      CSS       
+ -matcher .dart     Dart      
+ -matcher .dyck     Dyck      
+ -matcher .clj      Clojure   
+ -matcher .elm      Elm       
+ -matcher .erl      Erlang    
+ -matcher .ex       Elixir    
+ -matcher .f        Fortran   
+ -matcher .fsx      F#        
+ -matcher .go       Go        
+ -matcher .html     HTML      
+ -matcher .hs       Haskell   
+ -matcher .java     Java      
+ -matcher .js       Javascript/Typescript
+ -matcher .json     JSON      
+ -matcher .jl       Julia     
+ -matcher .kt       Kotlin    
+ -matcher .tex      LaTeX     
+ -matcher .lisp     Lisp      
+ -matcher .nim      Nim       
+ -matcher .ml       OCaml     
+ -matcher .paren    Paren     
+ -matcher .pas      Pascal    
+ -matcher .php      PHP       
+ -matcher .py       Python    
+ -matcher .re       Reason    
+ -matcher .rb       Ruby      
+ -matcher .rs       Rust      
+ -matcher .scala    Scala     
+ -matcher .sql      SQL       
+ -matcher .swift    Swift     
+ -matcher .txt      Text      
+ -matcher .xml      XML       
+ -matcher .generic  Generic   
+|}]
+
+
+let%expect_test "patdiff_and_zip" =
+  with_zip (fun file ->
+      let match_template = ":[2] :[1]" in
+      let rewrite_template = ":[1]" in
+      let command_args =
+        Format.sprintf "'%s' '%s' .ml -sequential -json-lines -zip %s"
+          match_template rewrite_template file
+      in
+      let command = Format.sprintf "%s %s" binary_path command_args in
+      let result = read_output command in
+      print_string result;
+      [%expect_exact {|{"uri":"main.ml","rewritten_source":"world","in_place_substitutions":[{"range":{"start":{"offset":0,"line":-1,"column":-1},"end":{"offset":5,"line":-1,"column":-1}},"replacement_content":"world","environment":[{"variable":"1","value":"world","range":{"start":{"offset":0,"line":-1,"column":-1},"end":{"offset":5,"line":-1,"column":-1}}}]}],"diff":"--- main.ml\n+++ main.ml\n@@ -1,1 +1,1 @@\n-hello world\n+world"}
+|}]
+    )
+
+let%expect_test "template_parsing_no_match_template" =
+  let source = "hello world" in
+  let template_dir = "example" ^/ "templates" ^/ "parse-no-match-template" in
+  let command_args = Format.sprintf "-stdin -sequential -f .c -templates %s" template_dir in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect_exact {|WARNING: Could not read required match file in example/templates/parse-no-match-template
+|}]
+
+let%expect_test "template_parsing_with_trailing_newline" =
+  let source = "hello world" in
+  let template_dir = "example" ^/ "templates" ^/ "parse-template-no-trailing-newline" in
+  let command_args = Format.sprintf "-stdin -sequential -f .c -templates %s -stdout" template_dir in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect{|
+    hello world |}]
+
+let%expect_test "template_parsing_with_trailing_newline" =
+  let source = "hello world" in
+  let template_dir = "example" ^/ "templates" ^/ "parse-template-with-trailing-newline" in
+  let command_args = Format.sprintf "-stdin -sequential -f .c -templates %s -stdout" template_dir in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect{|
+    hello world |}]
+
+let%expect_test "nested_templates" =
+  let source = "1 2 3" in
+  let template_dir = "example" ^/ "multiple-nested-templates" in
+  let command_args = Format.sprintf "-stdin -sequential -f .c -templates %s -stdout" template_dir in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect{|
+    +1 +2 +3WARNING: Could not read required match file in example/multiple-nested-templates/invalid-subdir |}]
+
+let%expect_test "diff_is_default" =
+  let source = "a X c a Y c" in
+  let match_template = "a :[1] c" in
+  let rewrite_template = "c :[1] a" in
+  let command_args =
+    Format.sprintf "-stdin -sequential '%s' '%s' -f .c"
+      match_template rewrite_template
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect_exact {|------ /dev/null
+++++++ /dev/null
+@|-1,1 +1,1 ============================================================
+-|a X c a Y c
++|c X a c Y a
+|}]
+
+let%expect_test "diff_option" =
+  let source = "a X c a Y c" in
+  let match_template = "a :[1] c" in
+  let rewrite_template = "c :[1] a" in
+  let command_args =
+    Format.sprintf "-stdin -sequential -diff '%s' '%s' -f .c"
+      match_template rewrite_template
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect_exact {|--- /dev/null
++++ /dev/null
+@@ -1,1 +1,1 @@
+-a X c a Y c
++c X a c Y a
+|}]
+
+let%expect_test "stdout_option" =
+  let source = "a X c a Y c" in
+  let match_template = "a :[1] c" in
+  let rewrite_template = "c :[1] a" in
+  let command_args =
+    Format.sprintf "-stdin -sequential -stdout '%s' '%s' -f .c"
+      match_template rewrite_template
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  read_expect_stdin_and_stdout command source
+  |> print_string;
+  [%expect_exact {|c X a c Y a|}]
+
+let%expect_test "only_color_prints_colored_diff" =
+  let source = "a X c a Y c" in
+  let match_template = "a :[1] c" in
+  let rewrite_template = "c :[1] a" in
+  let command_args =
+    Format.sprintf "-stdin -sequential '%s' '%s' -f .c -color"
+      match_template rewrite_template
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect_exact {|------ /dev/null
+++++++ /dev/null
+@|-1,1 +1,1 ============================================================
+-|a X c a Y c
++|c X a c Y a
+|}]
+
+let%expect_test "diff_explicit_color" =
+  let source = "a X c a Y c" in
+  let match_template = "a :[1] c" in
+  let rewrite_template = "c :[1] a" in
+  let command_args =
+    Format.sprintf "-stdin -sequential '%s' '%s' -f .c -diff -color"
+      match_template rewrite_template
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect_exact {|------ /dev/null
+++++++ /dev/null
+@|-1,1 +1,1 ============================================================
+-|a X c a Y c
++|c X a c Y a
+|}]
+
+let%expect_test "is_real_directory" =
+  let source = "hello world" in
+  let src_dir = "example" ^/ "src" ^/ "main.c" in
+  let command_args = Format.sprintf "'main' 'pain' -sequential -d %s -exclude-dir 'ignore' -diff" src_dir in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect{|
+    Directory specified with -d or -directory is not a directory. |}]
+
+let%expect_test "exclude_dir_option" =
+  let source = "hello world" in
+  let src_dir = "example" ^/ "src" in
+  let command_args = Format.sprintf "'main' 'pain' -sequential -d %s -exclude-dir 'ignore' -diff" src_dir in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect{|
+    --- example/src/honor-file-extensions/honor.pb.generic
+    +++ example/src/honor-file-extensions/honor.pb.generic
+    @@ -1,3 +1,3 @@
+    -func main() {
+    +func pain() {
+     // foo()
+     }
+    --- example/src/honor-file-extensions/honor.pb.go
+    +++ example/src/honor-file-extensions/honor.pb.go
+    @@ -1,4 +1,4 @@
+    -func main() {
+    +func pain() {
+     // in a comment foo()
+     foo()
+     }
+    --- example/src/main.c
+    +++ example/src/main.c
+    @@ -1,1 +1,1 @@
+    -int main() {}
+    +int pain() {}
+
+    WARNING: the GENERIC matcher was used, because a language could not be inferred from the file extension(s). The GENERIC matcher may miss matches. See '-list' to set a matcher for a specific language and to remove this warning. |}];
+
+  let src_dir = "example" ^/ "src" in
+  let command_args = Format.sprintf "'main' 'pain' -sequential -d %s -exclude-dir 'nonexist' -diff" src_dir in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect{|
+    --- example/src/honor-file-extensions/honor.pb.generic
+    +++ example/src/honor-file-extensions/honor.pb.generic
+    @@ -1,3 +1,3 @@
+    -func main() {
+    +func pain() {
+     // foo()
+     }
+    --- example/src/honor-file-extensions/honor.pb.go
+    +++ example/src/honor-file-extensions/honor.pb.go
+    @@ -1,4 +1,4 @@
+    -func main() {
+    +func pain() {
+     // in a comment foo()
+     foo()
+     }
+    --- example/src/ignore-me/main.c
+    +++ example/src/ignore-me/main.c
+    @@ -1,1 +1,1 @@
+    -int main() {}
+    +int pain() {}
+    --- example/src/main.c
+    +++ example/src/main.c
+    @@ -1,1 +1,1 @@
+    -int main() {}
+    +int pain() {}
+
+    WARNING: the GENERIC matcher was used, because a language could not be inferred from the file extension(s). The GENERIC matcher may miss matches. See '-list' to set a matcher for a specific language and to remove this warning. |}]
+
+let%expect_test "dir_depth_option" =
+  let source = "hello world" in
+  let src_dir = "example" ^/ "src" in
+  let command_args = Format.sprintf "'depth_' 'correct_depth_' -sequential -directory %s -depth %d -diff" src_dir (-1) in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect{| -depth must be 0 or greater. |}];
+
+  let source = "hello world" in
+  let src_dir = "example" ^/ "src" in
+  let command_args = Format.sprintf "'depth_' 'correct_depth_' -sequential -directory %s -depth %d -diff" src_dir 0 in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect{|
+    --- example/src/depth-0.c
+    +++ example/src/depth-0.c
+    @@ -1,1 +1,1 @@
+    -int depth_0() {}
+    +int correct_depth_0() {}
+
+    WARNING: the GENERIC matcher was used, because a language could not be inferred from the file extension(s). The GENERIC matcher may miss matches. See '-list' to set a matcher for a specific language and to remove this warning. |}];
+
+  let source = "hello world" in
+  let src_dir = "example" ^/ "src" in
+  let command_args = Format.sprintf "'depth_' 'correct_depth_' -sequential -directory %s -depth %d -diff" src_dir 1 in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect{|
+    --- example/src/depth-0.c
+    +++ example/src/depth-0.c
+    @@ -1,1 +1,1 @@
+    -int depth_0() {}
+    +int correct_depth_0() {}
+    --- example/src/depth-1/depth-1.c
+    +++ example/src/depth-1/depth-1.c
+    @@ -1,1 +1,1 @@
+    -int depth_1() {}
+    +int correct_depth_1() {}
+
+    WARNING: the GENERIC matcher was used, because a language could not be inferred from the file extension(s). The GENERIC matcher may miss matches. See '-list' to set a matcher for a specific language and to remove this warning. |}];
+
+  let source = "hello world" in
+  let src_dir = "example" ^/ "src" in
+  let command_args = Format.sprintf "'depth_' 'correct_depth_' -sequential -directory %s -depth %d -diff" src_dir 2 in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect{|
+    --- example/src/depth-0.c
+    +++ example/src/depth-0.c
+    @@ -1,1 +1,1 @@
+    -int depth_0() {}
+    +int correct_depth_0() {}
+    --- example/src/depth-1/depth-1.c
+    +++ example/src/depth-1/depth-1.c
+    @@ -1,1 +1,1 @@
+    -int depth_1() {}
+    +int correct_depth_1() {}
+    --- example/src/depth-1/depth-2/depth-2.c
+    +++ example/src/depth-1/depth-2/depth-2.c
+    @@ -1,1 +1,1 @@
+    -int depth_2() {}
+    +int correct_depth_2() {}
+
+    WARNING: the GENERIC matcher was used, because a language could not be inferred from the file extension(s). The GENERIC matcher may miss matches. See '-list' to set a matcher for a specific language and to remove this warning. |}];
+
+  let source = "hello world" in
+  let src_dir = "example" ^/ "src" in
+  let command_args = Format.sprintf "'depth_' 'correct_depth_' -sequential -directory %s -depth %d -diff" src_dir 1000 in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect{|
+    --- example/src/depth-0.c
+    +++ example/src/depth-0.c
+    @@ -1,1 +1,1 @@
+    -int depth_0() {}
+    +int correct_depth_0() {}
+    --- example/src/depth-1/depth-1.c
+    +++ example/src/depth-1/depth-1.c
+    @@ -1,1 +1,1 @@
+    -int depth_1() {}
+    +int correct_depth_1() {}
+    --- example/src/depth-1/depth-2/depth-2.c
+    +++ example/src/depth-1/depth-2/depth-2.c
+    @@ -1,1 +1,1 @@
+    -int depth_2() {}
+    +int correct_depth_2() {}
+
+    WARNING: the GENERIC matcher was used, because a language could not be inferred from the file extension(s). The GENERIC matcher may miss matches. See '-list' to set a matcher for a specific language and to remove this warning. |}]
+
+let%expect_test "matcher_override" =
+  let source = "hello world" in
+  let src_dir = "example" ^/ "src" in
+  let command_args = Format.sprintf "'(' '_unbalanced_match_' main.c -sequential -d %s -matcher .txt -diff" src_dir in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect{|
+    --- example/src/ignore-me/main.c
+    +++ example/src/ignore-me/main.c
+    @@ -1,1 +1,1 @@
+    -int main() {}
+    +int main_unbalanced_match_) {}
+    --- example/src/main.c
+    +++ example/src/main.c
+    @@ -1,1 +1,1 @@
+    -int main() {}
+    +int main_unbalanced_match_) {} |}];
+
+  let source = "hello world" in
+  let src_dir = "example" ^/ "src" in
+  let command_args = Format.sprintf "'(' '_unbalanced_match_' main.c -sequential -d %s -diff" src_dir in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect{| |}]
+
+let%expect_test "infer_and_honor_extensions" =
+  let source = "doesn't matter" in
+  let src_dir = "example" ^/ "src" ^/ "honor-file-extensions" in
+  let command_args = Format.sprintf "'foo()' 'bar()' .go -sequential -d %s -diff" src_dir in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect{|
+    --- example/src/honor-file-extensions/honor.pb.go
+    +++ example/src/honor-file-extensions/honor.pb.go
+    @@ -1,4 +1,4 @@
+     func main() {
+     // in a comment foo()
+    -foo()
+    +bar()
+     } |}];
+
+  let source = "doesn't matter" in
+  let src_dir = "example" ^/ "src" ^/ "honor-file-extensions" in
+  let command_args = Format.sprintf "'foo()' 'bar()' .generic -sequential -d %s -diff" src_dir in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect{|
+    --- example/src/honor-file-extensions/honor.pb.generic
+    +++ example/src/honor-file-extensions/honor.pb.generic
+    @@ -1,3 +1,3 @@
+     func main() {
+    -// foo()
+    +// bar()
+     }
+
+    WARNING: the GENERIC matcher was used, because a language could not be inferred from the file extension(s). The GENERIC matcher may miss matches. See '-list' to set a matcher for a specific language and to remove this warning. |}]
+
+let%expect_test "diff_only" =
+  let source = "hello world" in
+  let command_args = Format.sprintf "'hello' 'world' -stdin -sequential -json-lines -json-only-diff" in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect{|
+    {"uri":null,"diff":"--- /dev/null\n+++ /dev/null\n@@ -1,1 +1,1 @@\n-hello world\n+world world"}
+
+    WARNING: the GENERIC matcher was used, because a language could not be inferred from the file extension(s). The GENERIC matcher may miss matches. See '-list' to set a matcher for a specific language and to remove this warning. |}];
+
+  let source = "hello world" in
+  let command_args = Format.sprintf "'hello' 'world' -stdin -sequential -json-only-diff" in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect{|
+    -json-only-diff can only be supplied with -json-lines. |}]
+
+let%expect_test "zip_exclude_dir_with_extension" =
+  let source = "doesn't matter" in
+  let zip = "example" ^/ "zip-test" ^/ "sample-repo.zip" in
+  let exclude_dir = "sample-repo/vendor" in
+  let command_args = Format.sprintf "'main' 'pain' .go -zip %s -sequential -diff -exclude-dir %s" zip exclude_dir in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect{|
+    --- sample-repo/src/main.go
+    +++ sample-repo/src/main.go
+    @@ -1,2 +1,2 @@
+     // src
+    -func main() {}
+    +func pain() {} |}]
+
+let%expect_test "zip_exclude_dir_no_extension" =
+  let source = "doesn't matter" in
+  let zip = "example" ^/ "zip-test" ^/ "sample-repo.zip" in
+  let exclude_dir = "sample-repo/vendor" in
+  let command_args = Format.sprintf "'main' 'pain' -zip %s -sequential -diff -exclude-dir %s" zip exclude_dir in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect{|
+    --- sample-repo/src/main.go
+    +++ sample-repo/src/main.go
+    @@ -1,2 +1,2 @@
+     // src
+    -func main() {}
+    +func pain() {}
+
+    WARNING: the GENERIC matcher was used, because a language could not be inferred from the file extension(s). The GENERIC matcher may miss matches. See '-list' to set a matcher for a specific language and to remove this warning. |}]
+
+let%expect_test "invalid_path_with_error_message" =
+  let source = "doesn't matter" in
+  let command_args = Format.sprintf "'a' 'b' ./invalid/path" in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect{|
+    No such file or directory: ./invalid/path. Comby interprets patterns containing '/' as file paths. If a pattern does not contain '/' (like '.ml'), it is considered a pattern where file endings must match the pattern. Please supply only valid file paths or patterns. |}]
+
+let%expect_test "newline_separated_output"=
+  let source = "a b c" in
+  let match_template = ":[[1]]" in
+  let rewrite_template = ":[[1]]" in
+  let command_args =
+    Format.sprintf "-stdin -sequential -stdout '%s' '%s' -n -matcher .generic"
+      match_template rewrite_template
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  read_expect_stdin_and_stdout command source
+  |> print_string;
+  [%expect_exact {|a
+b
+c
+|}]
+
+let%expect_test "warn_on_stdin_and_in_place_flags" =
+  let source = "a b c" in
+  let match_template = ":[[1]]" in
+  let rewrite_template = ":[[1]]" in
+  let command_args =
+    Format.sprintf "-stdin -in-place '%s' '%s' -matcher .generic"
+      match_template rewrite_template
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  read_expect_stdin_and_stdout command source
+  |> print_string;
+  [%expect_exact {|WARNING: -in-place has no effect when -stdin is used. Ignoring -in-place.
+|}]
+
+let%expect_test "print_single_line_matches" =
+  let source = {|
+    let () = x in
+    let () = y in
+  |}
+  in
+  let match_template = "let ()" in
+  let rewrite_template = "dont care" in
+  let command_args =
+    Format.sprintf "-stdin '%s' '%s' -match-only -matcher .generic"
+      match_template rewrite_template
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  read_expect_stdin_and_stdout command source
+  |> print_string;
+  [%expect_exact {|let ()
+let ()
+|}]
+
+let%expect_test "print_multi_line_matches" =
+  let source = {|
+    let () = x in
+    let
+
+    ()
+in
+    let () = y in
+  |}
+  in
+  let match_template = "let ()" in
+  let rewrite_template = "dont care" in
+  let command_args =
+    Format.sprintf "-stdin '%s' '%s' -match-only -matcher .generic"
+      match_template rewrite_template
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  read_expect_stdin_and_stdout command source
+  |> print_string;
+  [%expect_exact {|let ()
+let\n\n    ()
+let ()
+|}];
+
+  let command_args =
+    Format.sprintf "-stdin '%s' '%s' -match-only -count -matcher .generic"
+      match_template rewrite_template
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  read_expect_stdin_and_stdout command source
+  |> print_string;
+  [%expect_exact {|3 matches
+|}];
+
+  let command_args =
+    Format.sprintf "-stdin '%s' '%s' -match-only -count -json-lines -matcher .generic"
+      match_template rewrite_template
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  read_expect_stdin_and_stdout command source
+  |> print_string;
+  [%expect_exact {|{"uri":null,"matches":[{"range":{"start":{"offset":5,"line":1,"column":6},"end":{"offset":11,"line":1,"column":12}},"environment":[],"matched":"let ()"},{"range":{"start":{"offset":23,"line":1,"column":24},"end":{"offset":34,"line":3,"column":7}},"environment":[],"matched":"let\n\n    ()"},{"range":{"start":{"offset":42,"line":1,"column":43},"end":{"offset":48,"line":1,"column":49}},"environment":[],"matched":"let ()"}]}WARNING: -count and -json-lines is specified. Ignoring -count.
+|}];
+
+  let command_args =
+    Format.sprintf "-stdin '%s' '%s' -count -matcher .generic"
+      match_template rewrite_template
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  read_expect_stdin_and_stdout command source
+  |> print_string;
+  [%expect_exact {|3 matches
+WARNING: -count only works with -match-only. Performing -match-only -count.
+|}]
+
+let%expect_test "unrecognized_matcher" =
+  let source = {|dont care|} in
+  let match_template = "dont care" in
+  let rewrite_template = "dont care" in
+  let command_args =
+    Format.sprintf "-stdin '%s' '%s' -matcher invalid"
+      match_template rewrite_template
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  read_expect_stdin_and_stdout command source
+  |> print_string;
+  [%expect_exact {|The matcher "invalid" is not supported. See -list for supported matchers
+|}]
+
+let%expect_test "generic_matcher_ok" =
+  let source = {|dont care|} in
+  let match_template = "dont care" in
+  let rewrite_template = "blah" in
+  let command_args =
+    Format.sprintf "-stdin '%s' '%s' -matcher .generic"
+      match_template rewrite_template
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  read_expect_stdin_and_stdout command source
+  |> print_string;
+  [%expect_exact {|------ /dev/null
+++++++ /dev/null
+@|-1,1 +1,1 ============================================================
+-|dont care
++|blah
+|}]
+
+let%expect_test "warn_on_anonymous_and_templates_flag" =
+  let source = "(fun i -> j) (fun x -> x)" in
+  let command_args =
+    Format.sprintf "-stdin -sequential 'ignore' 'ignore' -templates example/templates/implicit-equals -matcher .ml -stdout"
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let result = read_expect_stdin_and_stdout command source in
+  print_string result;
+  [%expect_exact {|(fun i -> j) identWARNING: Templates specified on the command line AND using -templates. Ignoring match
+      and rewrite templates on the command line and only using those in directories.
+|}]
+
+let%expect_test "dump_stats" =
+  let source = {|dont care|} in
+  let match_template = "care" in
+  let rewrite_template = "realy care" in
+  let command_args = Format.sprintf "-stdin '%s' '%s' -stats -matcher .txt" match_template rewrite_template in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  let stats_json = read_expect_stderr command source in
+  (match Statistics.of_yojson (Yojson.Safe.from_string stats_json) with
+   | Ok { number_of_files; lines_of_code; number_of_matches; _ } ->
+     Format.printf "number_of_files: %d, lines_of_code: %d, number_of_matches: %d"
+       number_of_files lines_of_code number_of_matches
+   | Error _ -> print_string "Unexpected error");
+  [%expect_exact {|number_of_files: 1, lines_of_code: 1, number_of_matches: 1|}]
+
+let%expect_test "substitute_bad_parse" =
+  let source = "dont care" in
+  let match_template = "dont care" in
+  let rewrite_template = "dont care" in
+  let command_args = Format.sprintf "%s %s -substitute 'json'" match_template rewrite_template in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  read_expect_stdin_and_stdout command source
+  |> print_string;
+  [%expect_exact {|Error, could not parse JSON to environment: Line 1, bytes 0-4:
+Invalid token 'json'
+|}]
+
+let%expect_test "substitute_ok" =
+  let source = "a match1 c d a match2 c d" in
+  let match_template = "ignored" in
+  let rewrite_template = ":[1] :[2]" in
+  let environment = {|[{"variable":"1","value":"hole_1"},{"variable":"2","value":"hole_2"}]|} in
+  let command_args =
+    Format.sprintf "'%s' '%s' -stdin -match-only -matcher .txt -substitute '%s'" match_template rewrite_template environment
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  read_expect_stdin_and_stdout command source
+  |> print_string;
+  [%expect_exact {|hole_1 hole_2
+|}]

test/alpha/test_server.ml

@@ -0,0 +1,204 @@
+open Core
+
+open Lwt.Infix
+
+open Match
+open Server_types
+
+let binary_path = "../../../../comby-server"
+
+let port = "9991"
+
+let pid' = ref None
+
+let launch port =
+  Unix.create_process ~prog:binary_path ~args:["-p"; port]
+  |> fun { pid; _ } -> pid' := Some pid
+
+let post endpoint json =
+  let uri =
+    let uri endpoint =
+      Uri.of_string ("http://127.0.0.1:" ^ port ^ "/" ^ endpoint)
+    in
+    match endpoint with
+    | `Match -> uri "match"
+    | `Rewrite -> uri "rewrite"
+    | `Substitute -> uri "substitute"
+  in
+  let thread =
+    Cohttp_lwt_unix.Client.post ~body:(`String json) uri >>= fun (_, response) ->
+    match response with
+    | `Stream response -> Lwt_stream.get response >>= fun result -> Lwt.return result
+    | _ -> Lwt.return None
+  in
+  match Lwt_unix.run thread with
+  | None -> "FAIL"
+  | Some result -> result
+
+(* FIXME(RVT) use wait *)
+let launch () =
+  launch port;
+  Unix.sleep 2
+
+let kill () =
+  match !pid' with
+  | None -> ()
+  | Some pid ->
+    match Signal.send Signal.kill (`Pid pid) with
+    | `Ok -> ()
+    | `No_such_process -> ()
+
+let () = launch ()
+
+let%expect_test "post_request" =
+
+  let source = "hello world" in
+  let match_template = "hello :[1]" in
+  let rule = Some {|where :[1] == "world"|} in
+  let language = "generic" in
+
+  In.{ source; match_template; rule; language; id = 0 }
+  |> In.match_request_to_yojson
+  |> Yojson.Safe.to_string
+  |> post `Match
+  |> print_string;
+
+  [%expect {|
+    {
+      "matches": [
+        {
+          "range": {
+            "start": { "offset": 0, "line": 1, "column": 1 },
+            "end": { "offset": 11, "line": 1, "column": 12 }
+          },
+          "environment": [
+            {
+              "variable": "1",
+              "value": "world",
+              "range": {
+                "start": { "offset": 6, "line": 1, "column": 7 },
+                "end": { "offset": 11, "line": 1, "column": 12 }
+              }
+            }
+          ],
+          "matched": "hello world"
+        }
+      ],
+      "source": "hello world",
+      "id": 0
+    } |}];
+
+
+  let source = "hello world" in
+  let match_template = "hello :[1]" in
+  let rule = Some {|where :[1] = "world"|} in
+  let language = "generic" in
+
+  In.{ source; match_template; rule; language; id = 0 }
+  |> In.match_request_to_yojson
+  |> Yojson.Safe.to_string
+  |> post `Match
+  |> print_string;
+
+  [%expect {|
+    Error in line 1, column 7:
+    where :[1] = "world"
+          ^
+    Expecting "false", "match", "rewrite" or "true"
+    Backtracking occurred after:
+      Error in line 1, column 12:
+      where :[1] = "world"
+                 ^
+      Expecting "!=" or "==" |}];
+
+  let substitution_kind = "in_place" in
+  let source = "hello world" in
+  let match_template = "hello :[1]" in
+  let rule = Some {|where :[1] == "world"|} in
+  let rewrite_template = ":[1], hello" in
+  let language = "generic" in
+
+  In.{ source; match_template; rewrite_template; rule; language; substitution_kind; id = 0}
+  |> In.rewrite_request_to_yojson
+  |> Yojson.Safe.to_string
+  |> post `Rewrite
+  |> print_string;
+
+  [%expect {|
+      {
+        "rewritten_source": "world, hello",
+        "in_place_substitutions": [
+          {
+            "range": {
+              "start": { "offset": 0, "line": -1, "column": -1 },
+              "end": { "offset": 12, "line": -1, "column": -1 }
+            },
+            "replacement_content": "world, hello",
+            "environment": [
+              {
+                "variable": "1",
+                "value": "world",
+                "range": {
+                  "start": { "offset": 0, "line": -1, "column": -1 },
+                  "end": { "offset": 5, "line": -1, "column": -1 }
+                }
+              }
+            ]
+          }
+        ],
+        "id": 0
+      } |}];
+
+  let substitution_kind = "newline_separated" in
+  let source = "hello world {} hello world" in
+  let match_template = "hello :[[1]]" in
+  let rule = Some {|where :[1] == "world"|} in
+  let rewrite_template = ":[1], hello" in
+  let language = "generic" in
+
+  In.{ source; match_template; rewrite_template; rule; language; substitution_kind; id = 0}
+  |> In.rewrite_request_to_yojson
+  |> Yojson.Safe.to_string
+  |> post `Rewrite
+  |> print_string;
+
+  [%expect {|
+      {
+        "rewritten_source": "world, hello\nworld, hello",
+        "in_place_substitutions": [],
+        "id": 0
+      } |}];
+
+  (* test there must be at least one predicate in a rule *)
+  let source = "hello world" in
+  let match_template = "hello :[1]" in
+  let rule = Some {|where |} in
+  let language = "generic" in
+
+  let request = In.{ source; match_template; rule; language; id = 0 } in
+  let json = In.match_request_to_yojson request |> Yojson.Safe.to_string in
+  let result = post `Match json in
+
+  print_string result;
+  [%expect {|
+    Error in line 1, column 7:
+    where
+          ^
+    Expecting ":[", "false", "match", "rewrite", "true" or string literal |}]
+
+let%expect_test "post_substitute" =
+
+  let rewrite_template = ":[1] hi :[2]" in
+  let environment = Environment.create () in
+  let environment = Environment.add environment "1" "oh" in
+  let environment = Environment.add environment "2" "there" in
+
+  In.{ rewrite_template; environment; id = 0 }
+  |> In.substitution_request_to_yojson
+  |> Yojson.Safe.to_string
+  |> post `Substitute
+  |> print_string;
+
+  [%expect {| { "result": "oh hi there", "id": 0 } |}]
+
+let () = kill ()

test/common/dune

@@ -0,0 +1,22 @@
+(library
+ (name common_test_integration)
+ (modules
+  test_nested_comments
+  test_c
+  test_bash
+  test_go
+  test_c_separators
+  test_pipeline
+  test_rewrite_parts
+  test_user_defined_language)
+ (inline_tests)
+ (preprocess (pps ppx_expect ppx_sexp_message ppx_deriving_yojson ppx_deriving_yojson.runtime))
+ (libraries
+  comby
+  cohttp-lwt-unix
+  core
+  camlzip))
+
+(alias
+(name runtest)
+(deps (source_tree example) (source_tree example/src/.ignore-me)))

test/dune

@@ -1,35 +1,2 @@
-(library
- (name test_integration)
- (modules
-  test_match_rule
-  test_hole_extensions
-  test_python_string_literals
-  test_nested_comments
-  test_integration
-  test_statistics
-  test_c
-  test_cli
-  test_bash
-  test_go
-  test_c_style_comments
-  test_c_separators
-  test_string_literals
-  test_special_matcher_cases
-  test_generic
-  test_pipeline
-  test_rewrite_parts
-  test_rewrite_rule
-  test_server
-  test_substring_disabled
-  test_user_defined_language)
- (inline_tests)
- (preprocess (pps ppx_expect ppx_sexp_message ppx_deriving_yojson ppx_deriving_yojson.runtime))
- (libraries
-  comby
-  cohttp-lwt-unix
-  core
-  camlzip))
-
-(alias
-(name runtest)
-(deps (source_tree example) (source_tree example/src/.ignore-me)))
+(dirs common alpha)
+; (dirs common omega)

test/omega/dune

@@ -0,0 +1,31 @@
+(library
+ (name omega_test_integration)
+ (modules
+;
+; TODO
+;
+  test_generic
+  test_match_rule
+  test_hole_extensions
+  test_python_string_literals
+  test_integration
+  test_statistics
+  test_cli
+  test_c_style_comments
+  test_string_literals
+  test_special_matcher_cases
+  test_generic
+  test_rewrite_rule
+  test_server
+  test_substring_disabled)
+ (inline_tests)
+ (preprocess (pps ppx_expect ppx_sexp_message ppx_deriving_yojson ppx_deriving_yojson.runtime))
+ (libraries
+  comby
+  cohttp-lwt-unix
+  core
+  camlzip))
+
+(alias
+(name runtest)
+(deps (source_tree example) (source_tree example/src/.ignore-me)))

test/omega/example

@@ -0,0 +1 @@
+../example

test/omega/test_c_style_comments.ml

@@ -0,0 +1,222 @@
+(*open Core
+
+open Matchers
+open Rewriter
+
+let configuration = Configuration.create ~match_kind:Fuzzy ()
+
+let all ?(configuration = configuration) template source =
+  C.all ~configuration ~template ~source
+
+let print_matches matches =
+  List.map matches ~f:Match.to_yojson
+  |> (fun matches -> `List matches)
+  |> Yojson.Safe.pretty_to_string
+  |> print_string
+
+let%expect_test "rewrite_comments_1" =
+  let template = "replace this :[1] end" in
+  let source = "/* don't replace this () end */ do replace this () end" in
+  let rewrite_template = "X" in
+
+  all template source
+  |> (fun matches ->
+      Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact "/* don't replace this () end */ do X"]
+
+let%expect_test "rewrite_comments_2" =
+  let template =
+    {|
+      if (:[1]) { :[2] }
+    |}
+  in
+
+  let source =
+    {|
+      /* if (fake_condition_body_must_be_non_empty) { fake_body; } */
+      // if (fake_condition_body_must_be_non_empty) { fake_body; }
+      if (real_condition_body_must_be_empty) {
+        int i;
+        int j;
+      }
+    |}
+  in
+
+  let rewrite_template =
+    {|
+      if (:[1]) {}
+    |}
+  in
+
+  all template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact
+    {|
+      /* if (fake_condition_body_must_be_non_empty) { fake_body; } */
+      if (real_condition_body_must_be_empty) {}
+    |}]
+
+let%expect_test "capture_comments" =
+  let template = {|if (:[1]) { :[2] }|} in
+  let source = {|if (true) { /* some comment */ console.log(z); }|} in
+  let matches = all template source in
+  print_matches matches;
+  [%expect_exact {|[
+  {
+    "range": {
+      "start": { "offset": 0, "line": 1, "column": 1 },
+      "end": { "offset": 48, "line": 1, "column": 49 }
+    },
+    "environment": [
+      {
+        "variable": "1",
+        "value": "true",
+        "range": {
+          "start": { "offset": 4, "line": 1, "column": 5 },
+          "end": { "offset": 8, "line": 1, "column": 9 }
+        }
+      },
+      {
+        "variable": "2",
+        "value": "console.log(z);",
+        "range": {
+          "start": { "offset": 31, "line": 1, "column": 32 },
+          "end": { "offset": 46, "line": 1, "column": 47 }
+        }
+      }
+    ],
+    "matched": "if (true) { /* some comment */ console.log(z); }"
+  }
+]|}]
+
+let%expect_test "single_quote_in_comment" =
+  let template =
+    {| {:[1]} |}
+  in
+
+  let source =
+    {|
+       /*'*/
+       {test}
+    |}
+  in
+
+  let rewrite_template =
+    {|
+      {:[1]}
+    |}
+  in
+
+  all template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact
+    {|
+      {test}
+    |}]
+
+let%expect_test "single_quote_in_comment" =
+  let template =
+    {| {:[1]} |}
+  in
+
+  let source =
+    {|
+       {
+         a = 1;
+         /* Events with mask == AE_NONE are not set. So let's initiaize the
+          * vector with it. */
+         for (i = 0; i < setsize; i++)
+       }
+    |}
+  in
+
+  let rewrite_template =
+    {|
+      {:[1]}
+    |}
+  in
+
+  all template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact
+    {|
+      {
+         a = 1;
+         /* Events with mask == AE_NONE are not set. So let's initiaize the
+          * vector with it. */
+         for (i = 0; i < setsize; i++)
+       }
+    |}]
+
+let%expect_test "single_quote_in_comment" =
+  let template =
+    {| {:[1]} |}
+  in
+
+  let source =
+    {|
+       {
+         a = 1;
+         /* ' */
+         for (i = 0; i < setsize; i++)
+       }
+    |}
+  in
+
+  let rewrite_template =
+    {|
+      {:[1]}
+    |}
+  in
+
+  all template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact
+    {|
+      {
+         a = 1;
+         /* ' */
+         for (i = 0; i < setsize; i++)
+       }
+    |}]
+
+let%expect_test "give_back_the_comment_characters_for_newline_comments_too" =
+  let template =
+    {| {:[1]} |}
+  in
+
+  let source =
+    {|
+       {
+         // a comment
+       }
+    |}
+  in
+
+  let rewrite_template =
+    {|
+      {:[1]}
+    |}
+  in
+
+  all template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact
+    {|
+      {
+         // a comment
+       }
+    |}]
+  *)

test/omega/test_cli.ml

@@ -1,4 +1,4 @@
-open Core
+(*open Core
 
 module Time = Core_kernel.Time_ns.Span
 
@@ -942,3 +942,4 @@ let%expect_test "substitute_ok" =
   |> print_string;
   [%expect_exact {|hole_1 hole_2
 |}]
+  *)

test/omega/test_generic.ml

@@ -0,0 +1,434 @@
+open Core
+
+open Matchers
+open Rewriter
+
+let configuration = Configuration.create ~match_kind:Fuzzy ()
+
+let format s =
+  let s = String.chop_prefix_exn ~prefix:"\n" s in
+  let leading_indentation = Option.value_exn (String.lfindi s ~f:(fun _ c -> c <> ' ')) in
+  s
+  |> String.split ~on:'\n'
+  |> List.map ~f:(Fn.flip String.drop_prefix leading_indentation)
+  |> String.concat ~sep:"\n"
+  |> String.chop_suffix_exn ~suffix:"\n"
+
+let run ?(configuration = configuration) source match_template rewrite_template =
+  Generic.first ~configuration match_template source
+  |> function
+  | Ok result ->
+    Rewrite.all ~source ~rewrite_template [result]
+    |> (fun x -> Option.value_exn x)
+    |> (fun { rewritten_source; _ } -> rewritten_source)
+    |> print_string
+  | Error _ ->
+    (* this is too annoying to fix everytime the grammar changes. *)
+    print_string ""
+
+let run_all ?(configuration = configuration) source match_template rewrite_template =
+  Generic.all ~configuration ~template:match_template ~source
+  |> function
+  | [] -> print_string "No matches."
+  | results ->
+    Option.value_exn (Rewrite.all ~source ~rewrite_template results)
+    |> (fun { rewritten_source; _ } -> rewritten_source)
+    |> print_string
+
+
+let%expect_test "basic" =
+  let source = {|a b c d|} in
+  let match_template = {|:[1]|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|a b c d|}];
+
+  let source = {|a b c d|} in
+  let match_template = {|a :[1] c d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|b|}];
+
+  let source = {|a b c d|} in
+  let match_template = {|a :[1] d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|b c|}];
+
+  let source = {|a b c d|} in
+  let match_template = {|a :[1]|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|b c d|}];
+
+  let source = {|a b c d|} in
+  let match_template = {|:[1] c d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|a b|}];
+
+  let source = {|a b c d|} in
+  let match_template = {|:[1] :[2]|} in
+  let rewrite_template = {|(:[1]) (:[2])|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(a) (b c d)|}];
+
+  let source = {|a b c d|} in
+  let match_template = {|:[2] :[1]|} in
+  let rewrite_template = {|(:[2]) (:[1])|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(a) (b c d)|}];
+
+  let source = {|a b c d|} in
+  let match_template = {|a :[2] :[1] d|} in
+  let rewrite_template = {|(:[2]) (:[1])|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(b) (c)|}];
+
+  let source = {|a b c d|} in
+  let match_template = {|a :[2] :[1]|} in
+  let rewrite_template = {|(:[2]) (:[1])|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(b) (c d)|}];
+
+  let source = {|a b c d|} in
+  let match_template = {|a :[2] c :[1]|} in
+  let rewrite_template = {|(:[2]) (:[1])|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(b) (d)|}];
+
+  let source = {|x:|} in
+  let match_template = {|:[1]:|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|x|}]
+
+let%expect_test "basic_failures" =
+  let source = {|a x b bbq|} in
+  let match_template = {|a :[1] b c|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {||}];
+
+  let source = {|a b c d|} in
+  let match_template = {|a :[2] d :[1]|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact
+    {||}];
+
+  let source = {|a b c d|} in
+  let match_template = {|a :[2] b :[1]|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {||}]
+
+let%expect_test "delimiter_matching" =
+  let source = {|(a b c) d|} in
+  let match_template = {|(:[1]) d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|a b c|}];
+
+  let source = {|(a b c) d|} in
+  let match_template = {|(:[1] b :[2]) d|} in
+  let rewrite_template = {|(:[1]) (:[2])|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(a) (c)|}];
+
+  let source = {|q(a b c) d|} in
+  let match_template = {|q(:[1] b :[2]) d|} in
+  let rewrite_template = {|(:[1]) (:[2])|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(a) (c)|}];
+
+  let source = {|((a) b)|} in
+  let match_template = {|(:[1] b)|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(a)|}];
+
+  let source = {|((a b c)) d|} in
+  let match_template = {|(:[1]) d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(a b c)|}];
+
+  let source = {|((a b c)) d|} in
+  let match_template = {|(:[1]) d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(a b c)|}];
+
+  let source = {|((a b c) q) d|} in
+  let match_template = {|((:[1]) q) d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|a b c|}];
+
+  let source = {|((a b c) q) d|} in
+  let match_template = {|((:[1] c) q) d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|a b|}];
+
+  let source = {|((a b () c) q) d|} in
+  let match_template = {|((:[1] () c) q) d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|a b|}];
+
+  let source = {|((a ((x) d) b c)) d|} in
+  let match_template = {|((a :[1] :[2] c)) d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|((x) d)|}];
+
+  let source = {|((a ((x) d) b c)) d|} in
+  let match_template = {|((a (:[1]) :[2] c)) d|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(x) d b|}];
+
+  let source = {|(b (c) d)|} in
+  let match_template = {|(:[1])|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|b (c) d|}];
+
+  let source = {|(b (c) d.)|} in
+  let match_template = {|(:[1].)|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|b (c) d|}];
+
+  let source = {|(b (c.) d.)|} in
+  let match_template = {|(:[1].)|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|b (c.) d|}];
+
+  let source = {|(b. (c) d.)|} in
+  let match_template = {|(:[1].)|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|b. (c) d|}];
+
+  let source = {|(b (c) d.)|} in
+  let match_template = {|(b :[1] d.)|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(c)|}];
+
+  let source = {|outer(inner(dst,src),src)|} in
+  let match_template = {|outer(:[1],src)|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|inner(dst,src)|}];
+
+  let source = {|(b ((c)) d.)|} in
+  let match_template = {|(b :[1] d.)|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|((c))|}];
+
+  let source = {|a b c|} in
+  let match_template = {|a :[1] c|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|b|}];
+
+  let source = {|x = foo;|} in
+  let match_template = {|x = :[1];|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|foo|}];
+
+  let source = {|((a {{x} d} b c)) d|} in
+  let match_template = {|((a {:[1] d} :[2] c)) d|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|{x} b|}];
+
+  let source = {|((a {([{x}]) d} b c)) d|} in
+  let match_template = {|((a {:[1] d} :[2] c)) d|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|([{x}]) b|}];
+
+  let source = {|(((((x)))))|} in
+  let match_template = {|(((:[1])))|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|((x))|}];
+
+  let source = {|((((y(x)z))))|} in
+  let match_template = {|(((:[1])))|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(y(x)z)|}];
+
+  let source = {|((((y(x)z))))|} in
+  let match_template = {|(((:[1]):[2]))|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(y(x)z) |}];
+
+  let source = {|(((x)z))|} in
+  let match_template = {|(((:[1]):[2]))|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|x z|}];
+
+  let source = {|((((x))z))|} in
+  let match_template = {|(((:[1]):[2]))|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(x) z|}];
+
+  let source = {|lolwtfbbq|} in
+  let match_template = {|lol:[1]bbq|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|wtf|}];
+
+  let source = {|x = foo; x = bar;|} in
+  let match_template = {|x = :[1];|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|foo x = bar;|}];
+
+  let source = {|[ no match prefix ] x = foo; [ no match suffix ]|} in
+  let match_template = {|x = :[1];|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|[ no match prefix ] foo [ no match suffix ]|}];
+
+  let source = {|x = a; x = b; x = c|} in
+  let match_template = {|x = :[1];|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|a x = b; x = c|}];
+
+  let source = {|x = ( x = x; );|} in
+  let match_template = {|x = :[1];|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|( x = x; )|}];
+
+  let source = {|( x = x = x; )|} in
+  let match_template = {|x = :[1];|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|( x = x )|}];
+
+  let source = {|xxx a b d c 1 2 3 b d d blah|} in
+  let match_template = {|a :[1] c :[2] d|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|xxx b d 1 2 3 b d blah|}];
+
+  let source = {|howevenlolwtfbbqispossible|} in
+  let match_template = {|lol:[1]bbq|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|howevenwtfispossible|}];
+
+  let source = {|lolhowevenlolwtfbbqispossiblebbq|} in
+  let match_template = {|lol:[1]bbq|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|howevenlolwtfispossiblebbq|}];
+
+  let source = {|hello my name is bob the builder|} in
+  let match_template = {|:[alongidentifiername] :[2] :[3] :[xyz] :[5] :[6]|} in
+  let rewrite_template = {|:[alongidentifiername] :[2] :[3] :[xyz] :[5] :[6]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|hello my name is bob the builder|}];
+
+  let source = {|www.testdofooname.com/picsinsideit/stunningpictureofkays1381737242g8k4n-280x428.jpg|} in
+  let match_template = {|www.:[1]-:[2].jpg|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|testdofooname.com/picsinsideit/stunningpictureofkays1381737242g8k4n 280x428|}];
+
+  let source = {|https://api.github.com/repos/dmjacobsen/slurm/commits/716c1499695c68afcab848a1b49653574b4fc167|} in
+  let match_template = {|:[1]api.:[2]/repos/:[3]s/:[4]|} in
+  let rewrite_template = {|:[1] :[2] :[3] :[4]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|https:// github.com dmjacobsen/slurm/commit 716c1499695c68afcab848a1b49653574b4fc167|}];
+
+  let source =
+    {|
+      assert(stream->md_len + md_len -
+             si.foo_data_begin <= MAD_BUFFER_MDLEN);
+      memcpy(*stream->foo_data + stream->md_len,
+             mad_bit_nextbyte(&stream->ptr),
+             frame_used = md_len - si.foo_data_begin);
+      stream->md_len += frame_used;
+    |} |> format
+  in
+  let match_template = {|memcpy(:[1], :[2], :[3]);|} in
+  let rewrite_template = {|:[1], :[2], :[3]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|assert(stream->md_len + md_len -
+       si.foo_data_begin <= MAD_BUFFER_MDLEN);
+*stream->foo_data + stream->md_len, mad_bit_nextbyte(&stream->ptr), frame_used = md_len - si.foo_data_begin
+stream->md_len += frame_used;|}]
+
+let%expect_test "significant_whitespace" =
+  let configuration = Configuration.create ~match_kind:Fuzzy ~significant_whitespace:true () in
+  let run = run ~configuration in
+
+  let source = {|two  spaces|} in
+  let match_template = {|:[1]  :[2]|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|two spaces|}];
+
+  (* FIXME: this should fail. also test case where separators do or do not need
+     whitespace.  e.g., strict about strcpy(src,dst) matching a template
+     strcpy(:[1],:[2]) versus strcpy(:[1], :[2]) *)
+  let source = {|two  spaces|} in
+  let match_template = {|:[1] :[2]|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|two spaces|}]
+
+let%expect_test "contextual_matching" =
+  let run = run_all in
+
+  let source = {|memcpy(dst1, src1, 1); memcpy(dst2, src2, 2);|} in
+  let match_template = {|memcpy(:[1], :[2], :[3])|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|dst1; dst2;|}];
+
+  let source = {|memcpy(dst1, src1, 1); memcpy(dst2, src2, 2);|} in
+  let match_template = {|memcpy(:[1], :[2], :[3])|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|dst1; dst2;|}]
+
+let%expect_test "contextual_matching_with_short_hole_syntax" =
+  let run = run_all in
+
+  let source = {|memcpy(dst1, src1, 1); memcpy(dst2, src2, 2);|} in
+  let match_template = {|memcpy(:[[1]], :[2], :[3])|} in
+  let rewrite_template = {|:[[1]]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|dst1; dst2;|}]
+
+(*
+let%expect_test "trivial_empty_case" =
+  let source = "" in
+  let match_template = "" in
+  begin
+    Generic.all ~configuration ~template:match_template ~source
+    |> function
+    | [] -> print_string "No matches."
+    | hd :: _ ->
+      print_string (Yojson.Safe.to_string (Match.to_yojson hd))
+  end;
+  [%expect_exact {|{"range":{"start":{"offset":0,"line":1,"column":1},"end":{"offset":0,"line":1,"column":1}},"environment":[],"matched":""}|}]
+*)

test/omega/test_hole_extensions.ml

@@ -0,0 +1,144 @@
+(*open Core
+
+open Matchers
+open Rewriter
+
+let configuration = Configuration.create ~match_kind:Fuzzy ()
+
+let run_all ?(configuration = configuration) source match_template rewrite_template =
+  Generic.all ~configuration ~template:match_template ~source
+  |> function
+  | [] -> print_string "No matches."
+  | results ->
+    Option.value_exn (Rewrite.all ~source ~rewrite_template results)
+    |> (fun { rewritten_source; _ } -> rewritten_source)
+    |> print_string
+
+let%expect_test "non_space" =
+  let run = run_all in
+  let source = {|   foo.     foo.bar.quux derp|} in
+  let match_template = {|:[x.]|} in
+  let rewrite_template = {|{:[x]}|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|   {foo.}     {foo.bar.quux} {derp}|}]
+
+let%expect_test "only_space" =
+  let run = run_all in
+  let source = {|   foo.     foo.bar.quux derp|} in
+  let match_template = {|:[ x]|} in
+  let rewrite_template = {|{:[x]}|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|{   }foo.{     }foo.bar.quux{ }derp|}]
+
+let%expect_test "up_to_newline" =
+  let run = run_all in
+  let source =
+    {|
+foo.
+foo.bar.quux
+derp
+|} in
+  let match_template = {|:[x\n]|} in
+  let rewrite_template = {|{:[x]}|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|{
+}{foo.
+}{foo.bar.quux
+}{derp
+}|}]
+
+let%expect_test "match_empty_in_newline_hole" =
+  let run = run_all in
+  let source =
+    {|stuff
+after
+|} in
+  let match_template = {|stuff:[x\n]|} in
+  let rewrite_template = {|{->:[x]<-}|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|{->
+<-}after
+|}]
+
+let%expect_test "leading_indentation" =
+  let run = run_all in
+  let source =
+    {|
+       foo. bar bazz
+          foo.bar.quux
+  derp
+|} in
+  let match_template = {|:[ leading_indentation]:[rest\n]|} in
+  let rewrite_template = {|{:[leading_indentation]}:[rest]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|
+{       }foo. bar bazz
+{          }foo.bar.quux
+{  }derp
+|}]
+
+let%expect_test "non_space_partial_match" =
+  let run = run_all in
+  let source = {|   foo.     foo.bar.quux derp|} in
+  let match_template = {|foo.:[x.]ux|} in
+  let rewrite_template = {|{:[x]}|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|   foo.     {bar.qu} derp|}]
+
+let%expect_test "non_space_does_not_match_reserved_delimiters" =
+  let run = run_all in
+  let source = {|fo.o(x)|} in
+  let match_template = {|:[f.]|} in
+  let rewrite_template = {|{:[f]}|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|{fo.o}({x})|}]
+
+let%expect_test "alphanum_partial_match" =
+  let run = run_all in
+  let source = {|   foo.     foo.bar.quux derp|} in
+  let match_template = {|foo.b:[x]r.quux|} in
+  let rewrite_template = {|{:[x]}|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|   foo.     {a} derp|}]
+
+let%expect_test "newline_matcher_should_not_be_sat_on_space" =
+  let run = run_all in
+  let source =
+    {|a b c d
+ e f g h|} in
+  let match_template = {|:[line\n] |} in
+  let rewrite_template = {|{:[line]}|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|{a b c d
+}e f g h|}];
+
+  let run = run_all in
+  let source =
+    {|a b c d
+ e f g h|} in
+  let match_template = {|:[line\n]:[next]|} in
+  let rewrite_template = {|{:[line]}|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|{a b c d
+}|}];
+
+  let run = run_all in
+  let source =
+    {|a b c d
+e f g h
+|} in
+  let match_template = {|:[line1\n]:[next\n]|} in
+  let rewrite_template = {|{:[line1]|:[next]}|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|{a b c d
+|e f g h
+}|}]
+
+let%expect_test "implicit_equals" =
+  let run = run_all in
+  let source = {|a b a|} in
+  let match_template = {|:[[x]] :[[m]] :[[x]]|} in
+  let rewrite_template = {|:[m]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|b|}]
+  *)

test/omega/test_integration.ml

@@ -0,0 +1,323 @@
+(*open Core
+
+open Matchers
+open Rewriter
+
+let configuration = Configuration.create ~match_kind:Fuzzy ()
+
+let all ?(configuration = configuration) template source =
+  Generic.all ~configuration ~template ~source
+
+let print_matches matches =
+  List.map matches ~f:Match.to_yojson
+  |> (fun matches -> `List matches)
+  |> Yojson.Safe.pretty_to_string
+  |> print_string
+
+let%expect_test "dont_get_stuck" =
+  let template = "" in
+  let source = "a" in
+  let matches = all ~configuration template source in
+  print_matches matches;
+  [%expect_exact {|[]|}]
+
+let%expect_test "dont_get_stuck" =
+  let template = "a" in
+  let source = "a" in
+  let matches = all template source in
+  print_matches matches;
+  [%expect_exact {|[
+  {
+    "range": {
+      "start": { "offset": 0, "line": 1, "column": 1 },
+      "end": { "offset": 1, "line": 1, "column": 2 }
+    },
+    "environment": [],
+    "matched": "a"
+  }
+]|}]
+
+let%expect_test "dont_get_stuck" =
+  let template = "a" in
+  let source = "aaa" in
+  let matches = all template source in
+  print_matches matches;
+  [%expect_exact {|[
+  {
+    "range": {
+      "start": { "offset": 0, "line": 1, "column": 1 },
+      "end": { "offset": 1, "line": 1, "column": 2 }
+    },
+    "environment": [],
+    "matched": "a"
+  },
+  {
+    "range": {
+      "start": { "offset": 1, "line": 1, "column": 2 },
+      "end": { "offset": 2, "line": 1, "column": 3 }
+    },
+    "environment": [],
+    "matched": "a"
+  },
+  {
+    "range": {
+      "start": { "offset": 2, "line": 1, "column": 3 },
+      "end": { "offset": 3, "line": 1, "column": 4 }
+    },
+    "environment": [],
+    "matched": "a"
+  }
+]|}]
+
+let%expect_test "rewrite_awesome_1" =
+  let template = "replace this :[1] end" in
+  let source = "xreplace this () end" in
+  let rewrite_template = "X" in
+
+  all template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact "xX"]
+
+let%expect_test "rewrite_whole_template_matches" =
+  let template = {|rewrite :[1] <- this string|} in
+  let source = {|rewrite hello world <- this string|} in
+  let rewrite_template = "?" in
+
+  all template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact "?"]
+
+let%expect_test "single_token" =
+  let template = {|:[[1]] this|} in
+  let source = {|the problem is this|} in
+  let matches = all template source in
+  print_matches matches;
+  [%expect_exact {|[
+  {
+    "range": {
+      "start": { "offset": 12, "line": 1, "column": 13 },
+      "end": { "offset": 19, "line": 1, "column": 20 }
+    },
+    "environment": [
+      {
+        "variable": "1",
+        "value": "is",
+        "range": {
+          "start": { "offset": 12, "line": 1, "column": 13 },
+          "end": { "offset": 14, "line": 1, "column": 15 }
+        }
+      }
+    ],
+    "matched": "is this"
+  }
+]|}]
+
+
+let%expect_test "single_token_with_preceding_whitespace" =
+  let template = {| :[[1]] this|} in
+  let source = {|the problem is this|} in
+  let matches = all template source in
+  print_matches matches;
+  [%expect_exact {|[
+  {
+    "range": {
+      "start": { "offset": 11, "line": 1, "column": 12 },
+      "end": { "offset": 19, "line": 1, "column": 20 }
+    },
+    "environment": [
+      {
+        "variable": "1",
+        "value": "is",
+        "range": {
+          "start": { "offset": 12, "line": 1, "column": 13 },
+          "end": { "offset": 14, "line": 1, "column": 15 }
+        }
+      }
+    ],
+    "matched": " is this"
+  }
+]|}]
+
+let%expect_test "single_token_rewrite" =
+  let template = {| :[[1]] this|} in
+  let source = {|the problem is this|} in
+  let rewrite_template = ":[1]" in
+  all template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact "the problemis"]
+
+let%expect_test "single_token_match_inside_paren_no_succeeding_whitespace" =
+  let template = {|:[[1]](:[[2]])|} in
+  let source = {|foo(bar)|} in
+  let rewrite_template = ":[1] : :[2]" in
+  all template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact "foo : bar"]
+
+let%expect_test "whitespace_hole_rewrite" =
+  let template = {|:[ w]this|} in
+  let rewrite_template = "space:[ w]here" in
+  let source = {|      this|} in
+  all template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact "space      here"]
+
+let%expect_test "punctuation_hole_rewrite" =
+  let template = {|:[x.]|} in
+  let rewrite_template = "->:[x.]<-" in
+  let source = {|now.this. is,pod|racing|} in
+  all template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact "->now.this.<- ->is,pod|racing<-"]
+
+let%expect_test "newline_hole_rewrite" =
+  let template = {|:[x\n]|} in
+  let rewrite_template = {|->:[x\n]<-|} in
+  let source = {|now.this.
+is,pod|racing
+|} in
+  all template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact "->now.this.
+<-->is,pod|racing
+<-"]
+
+
+
+let%expect_test "shift_or_at_least_dont_get_stuck" =
+  let template = ":[1]" in
+  let source = "a" in
+  let matches = all template source in
+  print_matches matches;
+  [%expect_exact {|[
+  {
+    "range": {
+      "start": { "offset": 0, "line": 1, "column": 1 },
+      "end": { "offset": 1, "line": 1, "column": 2 }
+    },
+    "environment": [
+      {
+        "variable": "1",
+        "value": "a",
+        "range": {
+          "start": { "offset": 0, "line": 1, "column": 1 },
+          "end": { "offset": 1, "line": 1, "column": 2 }
+        }
+      }
+    ],
+    "matched": "a"
+  }
+]|}]
+
+let%expect_test "shift_or_at_least_dont_get_stuck" =
+  let template = ":[1]" in
+  let source = "aa" in
+  let matches = all template source in
+  print_matches matches;
+  [%expect_exact {|[
+  {
+    "range": {
+      "start": { "offset": 0, "line": 1, "column": 1 },
+      "end": { "offset": 2, "line": 1, "column": 3 }
+    },
+    "environment": [
+      {
+        "variable": "1",
+        "value": "aa",
+        "range": {
+          "start": { "offset": 0, "line": 1, "column": 1 },
+          "end": { "offset": 2, "line": 1, "column": 3 }
+        }
+      }
+    ],
+    "matched": "aa"
+  }
+]|}]
+
+let%expect_test "nested_rewrite1" =
+  let source =
+    {|
+      x x y strcpy(strcpy(dst1, src1), src2); blah blah XXX
+    |}
+  in
+
+  let template =
+    {|
+      strcpy(:[1], :[2])
+    |}
+  in
+
+  let matches = all template source in
+  print_matches matches;
+  [%expect_exact "[]"]
+
+(* FIXME(RVT) nested rewrites *)
+let%expect_test "nested_rewrite2" =
+  let template =
+    {|
+    if :[var_check] != nil {
+      for :[defines] := range :[var_use] {:[inner_body]}
+    }
+    |}
+  in
+  let source =
+    {|
+    if fields.List != nil {
+      for _, field := range fields.List {
+        if field.Names != nil {
+          for _, fieldName := range field.Names {
+            stuff with fields and things
+          }
+        }
+      }
+    }
+    |}
+  in
+  let rewrite_template = "for :[defines] := range :[var_use] {:[inner_body]}" in
+  all template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|for _, field := range fields.List {
+        if field.Names != nil {
+          for _, fieldName := range field.Names {
+            stuff with fields and things
+          }
+        }
+      }|}]
+
+let%expect_test "match_:[[1]]" =
+  let template =
+    {|
+    :[[1]].next()
+    |}
+  in
+  let source =
+    {|
+    col_names = reader.next()
+    }
+    |}
+  in
+  let rewrite_template = "next(:[1])" in
+  all template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|
+    col_names =next(reader)}
+    |}]
+  *)

test/omega/test_match_rule.ml

@@ -0,0 +1,698 @@
+(*open Core
+
+open Language
+open Matchers
+open Match
+
+let rule_parses rule =
+  match Rule.create rule with
+  | Ok _ -> "true"
+  | Error _ -> "false"
+
+let print_matches matches =
+  List.map matches ~f:Match.to_yojson
+  |> (fun matches -> `List matches)
+  |> Yojson.Safe.pretty_to_string
+  |> print_string
+
+let%expect_test "parse_rule" =
+  let rule = {| where :[1] == :[2], :[3] == "y" |} in
+  rule_parses rule |> print_string;
+  [%expect_exact {|true|}];
+
+  let rule = {| where :[1] == :[2], :[3] != "x" |} in
+  rule_parses rule |> print_string;
+  [%expect_exact {|true|}];
+
+  let rule =  {| where :[1] != :[3] |} in
+  rule_parses rule |> print_string;
+  [%expect_exact {|true|}]
+
+let%expect_test "parse_basic" =
+  Rule.create {|where "a" == "a"|}
+  |> Or_error.ok_exn
+  |> fun rule -> print_s [%message (rule : Ast.expression list)];
+  [%expect_exact {|(rule ((Equal (String a) (String a))))
+|}]
+
+let%expect_test "parse_match_one_case" =
+  Rule.create {|where match "match_me" { | "case_one" -> true }|}
+  |> Or_error.ok_exn
+  |> fun rule -> print_s [%message (rule : Ast.expression list)];
+  [%expect_exact "(rule ((Match (String match_me) (((String case_one) (True))))))
+"]
+
+let%expect_test "parse_match_multi_case" =
+  Rule.create
+    {| where
+       match "match_me" {
+       | "case_one" -> true
+       | "case_two" -> false
+       }
+    |}
+  |> Or_error.ok_exn
+  |> fun rule -> print_s [%message (rule : Ast.expression list)];
+  [%expect_exact "(rule
+ ((Match (String match_me)
+   (((String case_one) (True)) ((String case_two) (False))))))
+"]
+
+let sat ?(env = Environment.create ()) rule =
+  let rule = Rule.create rule |> Or_error.ok_exn in
+  Format.sprintf "%b" (Rule.(sat @@ apply rule env))
+
+let make_env bindings =
+  List.fold bindings
+    ~init:(Environment.create ())
+    ~f:(fun env (var, value) -> Environment.add env var value)
+
+let%expect_test "rule_sat" =
+  let rule = {| where "x" != "y" |} in
+  sat rule |> print_string;
+  [%expect_exact {|true|}];
+
+  let rule = {| where "x" != "x" |} in
+  sat rule |> print_string;
+  [%expect_exact {|false|}];
+
+  let rule = {| where "x" == "x" |} in
+  sat rule |> print_string;
+  [%expect_exact {|true|}];
+
+  let rule = {| where "x" == "y" |} in
+  sat rule |> print_string;
+  [%expect_exact {|false|}];
+
+  let rule = {| where :[x] == "y" |} in
+  sat rule |> print_string;
+  [%expect_exact {|false|}];
+
+  let rule = {| where :[x] == :[x] |} in
+  sat rule |> print_string;
+  [%expect_exact {|false|}]
+
+let%expect_test "rule_sat_with_env" =
+  let env = make_env ["1", "x"; "2", "y"; "3", "x"] in
+
+  let rule = {| where :[1] == :[3], :[1] != :[2] |} in
+  sat ~env rule |> print_string;
+  [%expect_exact {|true|}];
+
+  let rule = {| where :[1] == :[3], :[1] != "y" |} in
+  sat ~env rule |> print_string;
+  [%expect_exact {|true|}];
+
+  let rule = {| where :[1] == :[3], :[1] == "x" |} in
+  sat ~env rule |> print_string;
+  [%expect_exact {|true|}];
+
+  let rule = {| where :[1] == :[2], :[1] != :[2] |} in
+  sat ~env rule |> print_string;
+  [%expect_exact {|false|}]
+
+let configuration = Configuration.create ~match_kind:Fuzzy ()
+
+let format s =
+  let s = s |> String.chop_prefix_exn ~prefix:"\n" in
+  let leading_indentation =
+    Option.value_exn (String.lfindi s ~f:(fun _ c -> c <> ' ')) in
+  s
+  |> String.split ~on:'\n'
+  |> List.map ~f:(Fn.flip String.drop_prefix leading_indentation)
+  |> String.concat ~sep:"\n"
+  |> String.chop_suffix_exn ~suffix:"\n"
+
+
+let%expect_test "where_true" =
+  let template =
+    {|
+      (:[1]) => {}
+    |}
+    |> format
+  in
+
+  let source =
+    {|
+      (b,c) => {}
+    |}
+    |> format
+  in
+
+  let rule =
+    {| where true
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [
+      {
+        "range": {
+          "start": { "offset": 0, "line": 1, "column": 1 },
+          "end": { "offset": 11, "line": 1, "column": 12 }
+        },
+        "environment": [
+          {
+            "variable": "1",
+            "value": "b,c",
+            "range": {
+              "start": { "offset": 1, "line": 1, "column": 2 },
+              "end": { "offset": 4, "line": 1, "column": 5 }
+            }
+          }
+        ],
+        "matched": "(b,c) => {}"
+      }
+    ] |}]
+
+let%expect_test "match_sat" =
+  let template =
+    {|
+      (:[1]) => {}
+    |}
+    |> format
+  in
+
+  let source =
+    {|
+      (b,c) => {}
+    |}
+    |> format
+  in
+
+  let rule =
+    {| where
+       match :[1] {
+       | ":[_],:[_]" -> false
+       }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [] |}];
+
+  let rule =
+    {| where
+       match :[1] {
+       | ":[_],:[_]" -> true
+       }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [
+      {
+        "range": {
+          "start": { "offset": 0, "line": 1, "column": 1 },
+          "end": { "offset": 11, "line": 1, "column": 12 }
+        },
+        "environment": [
+          {
+            "variable": "1",
+            "value": "b,c",
+            "range": {
+              "start": { "offset": 1, "line": 1, "column": 2 },
+              "end": { "offset": 4, "line": 1, "column": 5 }
+            }
+          }
+        ],
+        "matched": "(b,c) => {}"
+      }
+    ] |}];
+
+  let source =
+    {|
+      (a) => {}
+      (b,c) => {}
+    |}
+    |> format
+  in
+
+  let rule =
+    {| where
+       match :[1] {
+       | ":[_],:[_]" -> false
+       | ":[_]" -> true
+       }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [
+      {
+        "range": {
+          "start": { "offset": 0, "line": 1, "column": 1 },
+          "end": { "offset": 9, "line": 1, "column": 10 }
+        },
+        "environment": [
+          {
+            "variable": "1",
+            "value": "a",
+            "range": {
+              "start": { "offset": 1, "line": 1, "column": 2 },
+              "end": { "offset": 2, "line": 1, "column": 3 }
+            }
+          }
+        ],
+        "matched": "(a) => {}"
+      }
+    ] |}];
+
+  let rule =
+    {|
+       where
+       match :[1] {
+       | ":[_],:[_]" -> false
+       | ":[_]" -> :[1] == "a"
+       }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [
+      {
+        "range": {
+          "start": { "offset": 0, "line": 1, "column": 1 },
+          "end": { "offset": 9, "line": 1, "column": 10 }
+        },
+        "environment": [
+          {
+            "variable": "1",
+            "value": "a",
+            "range": {
+              "start": { "offset": 1, "line": 1, "column": 2 },
+              "end": { "offset": 2, "line": 1, "column": 3 }
+            }
+          }
+        ],
+        "matched": "(a) => {}"
+      }
+    ] |}];
+
+  let rule =
+    {| where
+       match :[1] {
+       | ":[_],:[_]" -> false
+       | ":[_]" -> :[1] == "b"
+       }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [] |}]
+
+
+let%expect_test "match_s_suffix" =
+  let template = ":[1]s" in
+
+  let source = "names" in
+
+  let rule =
+    {| where true
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [
+      {
+        "range": {
+          "start": { "offset": 0, "line": 1, "column": 1 },
+          "end": { "offset": 5, "line": 1, "column": 6 }
+        },
+        "environment": [
+          {
+            "variable": "1",
+            "value": "name",
+            "range": {
+              "start": { "offset": 0, "line": 1, "column": 1 },
+              "end": { "offset": 4, "line": 1, "column": 5 }
+            }
+          }
+        ],
+        "matched": "names"
+      }
+    ] |}]
+
+let%expect_test "match_s_suffix" =
+  let template = ":[1]" in
+
+  let source = "names" in
+
+  let rule =
+    {| where true
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [
+      {
+        "range": {
+          "start": { "offset": 0, "line": 1, "column": 1 },
+          "end": { "offset": 5, "line": 1, "column": 6 }
+        },
+        "environment": [
+          {
+            "variable": "1",
+            "value": "names",
+            "range": {
+              "start": { "offset": 0, "line": 1, "column": 1 },
+              "end": { "offset": 5, "line": 1, "column": 6 }
+            }
+          }
+        ],
+        "matched": "names"
+      }
+    ] |}]
+
+let%expect_test "configuration_choice_based_on_case" =
+  let template = ":[1]" in
+
+  let source = "names" in
+
+  let rule =
+    {| where match :[1] {
+       | "ame" -> true
+       }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [] |}];
+
+  let template = ":[1]" in
+
+  let source = "names" in
+
+  let rule =
+    {| where match :[1] {
+       | "names" -> true
+       }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [
+      {
+        "range": {
+          "start": { "offset": 0, "line": 1, "column": 1 },
+          "end": { "offset": 5, "line": 1, "column": 6 }
+        },
+        "environment": [
+          {
+            "variable": "1",
+            "value": "names",
+            "range": {
+              "start": { "offset": 0, "line": 1, "column": 1 },
+              "end": { "offset": 5, "line": 1, "column": 6 }
+            }
+          }
+        ],
+        "matched": "names"
+      }
+    ] |}];
+
+  let template = ":[1]" in
+
+  let source = "namesXXXXX" in
+
+  let rule =
+    {| where match :[1] {
+       | "names" -> true
+       }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [] |}]
+
+
+
+
+let%expect_test "match_using_environment_merge" =
+  let template = "{:[1]}" in
+
+  let source = "{{ a : a } { a : a }}" in
+
+  let rule =
+    {| where match :[1] { | "{ :[x] : :[y] }" -> :[x] == :[y] }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [
+      {
+        "range": {
+          "start": { "offset": 0, "line": 1, "column": 1 },
+          "end": { "offset": 21, "line": 1, "column": 22 }
+        },
+        "environment": [
+          {
+            "variable": "1",
+            "value": "{ a : a } { a : a }",
+            "range": {
+              "start": { "offset": 1, "line": 1, "column": 2 },
+              "end": { "offset": 20, "line": 1, "column": 21 }
+            }
+          }
+        ],
+        "matched": "{{ a : a } { a : a }}"
+      }
+    ] |}];
+
+  let template = "{:[1]}" in
+
+  let source = "{{ a : a } { a : b }}" in
+
+  let rule =
+    {| where match :[1] { | "{ :[x] : :[y] }" -> :[x] == :[y] }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [] |}]
+
+
+let%expect_test "nested_matches" =
+  let template = ":[1]" in
+
+  let source = "{ { foo : { bar : { baz : qux } } } }" in
+
+  let rule =
+    {| where match :[1] {
+       | "{ :[foo] : :[tail1] }" -> match :[tail1] {
+                     | "{ :[bar] : :[tail2] }" -> match :[tail2] {
+                                                 | "{ baz : :[qux] }" -> :[qux] == "qux", :[bar] == "bar"
+                                                 }
+                     }
+       }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [
+      {
+        "range": {
+          "start": { "offset": 0, "line": 1, "column": 1 },
+          "end": { "offset": 37, "line": 1, "column": 38 }
+        },
+        "environment": [
+          {
+            "variable": "1",
+            "value": "{ { foo : { bar : { baz : qux } } } }",
+            "range": {
+              "start": { "offset": 0, "line": 1, "column": 1 },
+              "end": { "offset": 37, "line": 1, "column": 38 }
+            }
+          }
+        ],
+        "matched": "{ { foo : { bar : { baz : qux } } } }"
+      }
+    ] |}];
+
+  let template = ":[1]" in
+
+  let source = "{ { foo : { bar : { baz : qux } } } }" in
+
+  let rule =
+    {| where match :[1] {
+       | "{ :[foo] : :[tail1] }" -> match :[tail1] {
+                     | "{ :[bar] : :[tail2] }" -> match :[tail2] {
+                                                 | "{ baz : :[qux] }" -> :[qux] == "fail"
+                                                 }
+                     }
+       }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [] |}]
+
+
+let%expect_test "match_on_template" =
+  let template = ":[1]" in
+
+  let source = "oodles" in
+
+  let rule =
+    {| where match "p:[1]" {
+       | "poodles" -> true
+       }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [
+      {
+        "range": {
+          "start": { "offset": 0, "line": 1, "column": 1 },
+          "end": { "offset": 6, "line": 1, "column": 7 }
+        },
+        "environment": [
+          {
+            "variable": "1",
+            "value": "oodles",
+            "range": {
+              "start": { "offset": 0, "line": 1, "column": 1 },
+              "end": { "offset": 6, "line": 1, "column": 7 }
+            }
+          }
+        ],
+        "matched": "oodles"
+      }
+    ] |}];
+
+  let template = ":[1]" in
+
+  let source = "poodle" in
+
+  let rule =
+    {| where match ":[1]s" {
+       | "poodles" -> true
+       }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [
+      {
+        "range": {
+          "start": { "offset": 0, "line": 1, "column": 1 },
+          "end": { "offset": 6, "line": 1, "column": 7 }
+        },
+        "environment": [
+          {
+            "variable": "1",
+            "value": "poodle",
+            "range": {
+              "start": { "offset": 0, "line": 1, "column": 1 },
+              "end": { "offset": 6, "line": 1, "column": 7 }
+            }
+          }
+        ],
+        "matched": "poodle"
+      }
+    ] |}];
+
+  let template = ":[1]" in
+
+  let source = "poodle" in
+
+  let rule =
+    {| where match ":[1]," {
+       | "poodle" -> true
+       }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [] |}];
+    *)

test/omega/test_python_string_literals.ml

@@ -0,0 +1,55 @@
+(*open Core
+
+open Matchers
+
+let configuration = Configuration.create ~match_kind:Fuzzy ()
+
+let print_matches matches =
+  List.map matches ~f:(fun matched -> `String matched.Match.matched)
+  |> (fun matches -> `List matches)
+  |> Yojson.Safe.pretty_to_string
+  |> print_string
+
+
+let%expect_test "matched_contains_raw_literal_quotes" =
+  let source = {|"""blah"""|} in
+  let template = {|""":[[1]]"""|} in
+  let matches = Python.all ~configuration ~template ~source in
+  print_matches matches;
+  [%expect_exact {|[ "\"\"\"blah\"\"\"" ]|}]
+
+let%expect_test "interpreted_string_does_not_match_raw_literal" =
+  let source = {|"""blah""" "blah"|} in
+  let template = {|":[[1]]"|} in
+  let matches = Python.all ~configuration ~template ~source in
+  print_matches matches;
+  [%expect_exact {|[ "\"blah\"" ]|}]
+
+let%expect_test "interpreted_string_does_not_match_raw_literal_containing_quote" =
+  let source = {|"""blah""" """bl"ah""" "blah"|} in
+  let template = {|":[[1]]"|} in
+  let matches = Python.all ~configuration ~template ~source in
+  print_matches matches;
+  [%expect_exact {|[ "\"blah\"" ]|}]
+
+let%expect_test "raw_string_matches_string_containing_quote" =
+  let source = {|"""bl"ah"""|} in
+  let template = {|""":[1]"""|} in
+  let matches = Python.all ~configuration ~template ~source in
+  print_matches matches;
+  [%expect_exact {|[ "\"\"\"bl\"ah\"\"\"" ]|}]
+
+let%expect_test "invalid_raw_string_in_python_but_matches_because_ignores_after" =
+  let source = {|"""""""|} in
+  let template = {|""":[1]"""|} in
+  let matches = Python.all ~configuration ~template ~source in
+  print_matches matches;
+  [%expect_exact {|[ "\"\"\"\"\"\"" ]|}]
+
+let%expect_test "raw_string_captures_escape_sequences" =
+  let source = {|"""\""""|} in
+  let template = {|""":[1]"""|} in
+  let matches = Python.all ~configuration ~template ~source in
+  print_matches matches;
+  [%expect_exact {|[ "\"\"\"\\\"\"\"" ]|}]
+  *)

test/omega/test_rewrite_rule.ml

@@ -0,0 +1,127 @@
+(*open Core
+
+open Language
+open Matchers
+open Match
+open Rewriter
+
+let configuration = Configuration.create ~match_kind:Fuzzy ()
+
+let format s =
+  let s = String.chop_prefix_exn ~prefix:"\n" s in
+  let leading_indentation = Option.value_exn (String.lfindi s ~f:(fun _ c -> c <> ' ')) in
+  s
+  |> String.split ~on:'\n'
+  |> List.map ~f:(Fn.flip String.drop_prefix leading_indentation)
+  |> String.concat ~sep:"\n"
+  |> String.chop_suffix_exn ~suffix:"\n"
+
+let run_rule source match_template rewrite_template rule =
+  Generic.first ~configuration match_template source
+  |> function
+  | Error _ -> print_string "bad"
+  | Ok result ->
+    match result with
+    | ({ environment; _ } as m) ->
+      let e = Rule.(result_env @@ apply rule environment) in
+      match e with
+      | None -> print_string "bad bad"
+      | Some e ->
+        { m with environment = e }
+        |> List.return
+        |> Rewrite.all ~source ~rewrite_template
+        |> (fun x -> Option.value_exn x)
+        |> (fun { rewritten_source; _ } -> rewritten_source)
+        |> print_string
+
+let%expect_test "rewrite_rule" =
+  let source = {|int|} in
+  let match_template = {|:[1]|} in
+  let rewrite_template = {|:[1]|} in
+
+  let rule =
+    {|
+      where rewrite :[1] { "int" -> "expect" }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  run_rule source match_template rewrite_template rule;
+  [%expect_exact {|expect|}]
+
+let%expect_test "sequenced_rewrite_rule" =
+  let source = {|{ { a : { b : { c : d } } } }|} in
+  let match_template = {|{ :[a] : :[rest] }|} in
+  let rewrite_template = {|{ :[a] : :[rest] }|} in
+
+  let rule =
+    {|
+      where
+      rewrite :[a] { "a" -> "qqq" },
+      rewrite :[rest] { "{ b : { :[other] } }" -> "{ :[other] }" }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  run_rule source match_template rewrite_template rule;
+  [%expect_exact {|{ { qqq : { c : d } } }|}]
+
+let%expect_test "rewrite_rule_for_list" =
+  let source = {|[1, 2, 3, 4,]|} in
+  let match_template = {|[:[contents]]|} in
+  let rewrite_template = {|[:[contents]]|} in
+
+  let rule =
+    {|
+      where rewrite :[contents] { ":[[x]]," -> ":[[x]];" }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  run_rule source match_template rewrite_template rule;
+  [%expect_exact {|[1; 2; 3; 4;]|}]
+
+let%expect_test "rewrite_rule_for_list_strip_last" =
+  let source = {|[1, 2, 3, 4]|} in
+  let match_template = {|[:[contents]]|} in
+  let rewrite_template = {|[:[contents]]|} in
+
+  let rule =
+    {|
+      where rewrite :[contents] { ":[x], " -> ":[x]; " }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  run_rule source match_template rewrite_template rule;
+  [%expect_exact {|[1; 2; 3; 4]|}]
+
+let%expect_test "haskell_example" =
+  let source = {|
+     (concat
+     [ "blah blah blah"
+     , "blah"
+     ])
+|} in
+  let match_template = {|(concat [:[contents]])|} in
+  let rewrite_template = {|(:[contents])|} in
+
+  let rule =
+    {|
+      where rewrite :[contents] { "," -> "++" }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  run_rule source match_template rewrite_template rule;
+  [%expect_exact {|
+     ( "blah blah blah"
+     ++ "blah"
+     )
+|}]
+*)

test/omega/test_server.ml

@@ -1,4 +1,4 @@
-open Core
+(*open Core
 
 open Lwt.Infix
 
@@ -202,3 +202,4 @@ let%expect_test "post_substitute" =
   [%expect {| { "result": "oh hi there", "id": 0 } |}]
 
 let () = kill ()
+*)

test/omega/test_special_matcher_cases.ml

@@ -0,0 +1,459 @@
+(*open Core
+
+open Matchers
+open Rewriter
+
+let configuration = Configuration.create ~match_kind:Fuzzy ()
+
+
+let run ?(configuration = configuration) (module M : Matchers.Matcher) source match_template rewrite_template =
+  M.all ~configuration ~template:match_template ~source
+  |> function
+  | [] -> print_string "No matches."
+  | results ->
+    Option.value_exn (Rewrite.all ~source ~rewrite_template results)
+    |> (fun { rewritten_source; _ } -> rewritten_source)
+    |> print_string
+
+let%expect_test "parse_rust_apostrophe_ok" =
+  let source = {|width="1280"|} in
+  let match_template = {|width=":[1]"|} in
+  let rewrite_template = {|:[1]|} in
+
+  run (module Matchers.Generic) source match_template rewrite_template;
+  [%expect_exact {|1280|}]
+
+let%expect_test "parse_rust_apostrophe_ok" =
+  let source = {|pub struct GlobBuilder<'a> {}|} in
+  let match_template = {|{}|} in
+  let rewrite_template = {|{} // success|} in
+
+  run (module Matchers.Rust) source match_template rewrite_template;
+  [%expect_exact {|pub struct GlobBuilder<'a> {} // success|}]
+
+let%expect_test "parse_ocaml_apostrophe_ok" =
+  let source = {|type 'a t = Poly of 'a | Int of int |} in
+  let match_template = {|type :[v] t = :[_] Int of :[i]|} in
+  let rewrite_template = {|:[v], :[i]|} in
+
+  run (module Matchers.OCaml) source match_template rewrite_template;
+  [%expect_exact {|'a, int |}]
+
+let%expect_test "strict_nested_matching" =
+  let source = {|({})|} in
+  let match_template = {|(:[1])|} in
+  let rewrite_template = {|:[1]|} in
+
+  run (module Matchers.Dyck) source match_template rewrite_template;
+  [%expect_exact {|{}|}]
+
+let%expect_test "strict_nested_matching" =
+  let source = {|(})|} in
+  let match_template = {|(:[1])|} in
+  let rewrite_template = {|:[1]|} in
+
+  run (module Matchers.Dyck) source match_template rewrite_template;
+  [%expect_exact {|No matches.|}]
+
+
+
+let%expect_test "ocaml_blocks" =
+  let source = {|
+    module M : sig
+        type t
+    end = struct
+       type t = int
+
+       module Nested_M = struct
+         type r = int
+       end
+    end
+|}
+  in
+  let match_template = {|struct :[1] end|} in
+  let rewrite_template = {|struct <deleted> end|} in
+
+  run (module Matchers.OCaml) source match_template rewrite_template;
+  [%expect_exact {|
+    module M : sig
+        type t
+    end = struct <deleted> end
+|}]
+
+let%expect_test "ocaml_complex_blocks_with_same_end" =
+  let source = {|
+    begin
+    match x with
+    | _ ->
+        let module M = struct type t end
+        begin
+        begin
+        match y with
+        | _ -> ()
+        end
+        end
+    end
+|}
+  in
+  let match_template = {|begin :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.OCaml) source match_template rewrite_template;
+  [%expect_exact {|
+    <1>match x with
+    | _ ->
+        let module M = struct type t end
+        begin
+        begin
+        match y with
+        | _ -> ()
+        end
+        end</1>
+|}]
+
+let%expect_test "ruby_blocks" =
+  let source = {|
+class ActionController::Base
+  before_filter :generate_css_from_less
+
+  def generate_css_from_less
+    Less::More.generate_all
+  end
+end
+|}
+  in
+  let match_template = {|class :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|
+<1>ActionController::Base
+  before_filter :generate_css_from_less
+
+  def generate_css_from_less
+    Less::More.generate_all
+  end</1>
+|}]
+
+let%expect_test "erlang_blocks" =
+  let source = {|Big =  fun(X) -> if X > 10 -> true; true -> false end end.|} in
+  let match_template = {|fun(:[1]) :[rest] end|} in
+  let rewrite_template = {|<rest>:[rest]</rest>|} in
+
+  run (module Matchers.Erlang) source match_template rewrite_template;
+  [%expect_exact {|Big =  <rest>-> if X > 10 -> true; true -> false end</rest>.|}]
+
+let%expect_test "ruby_blocks" =
+  let source = {|class x end|} in
+  let match_template = {|class :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|<1>x</1>|}]
+
+let%expect_test "ruby_blocks_1" =
+  let source = {|class class def body1 end def body2 end end end|} in
+  let match_template = {|class :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|<1>class def body1 end def body2 end end</1>|}]
+
+let%expect_test "ruby_blocks_2" =
+  let source = {|class class (def body1 end) (def body2 end) end end|} in
+  let match_template = {|class :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|<1>class (def body1 end) (def body2 end) end</1>|}]
+
+let%expect_test "ruby_blocks_3" =
+  let source = {| def (def b end)(def b end) end |} in
+  let match_template = {|def :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {| <1>(def b end)(def b end)</1> |}]
+
+let%expect_test "ruby_blocks_4" =
+  let source = {| def (def a end) f (def b end)end |} in
+  let match_template = {|def :[1]end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {| <1>(def a end) f (def b end)</1> |}]
+
+(* this is correct: there's no space in the source 'end', so it matches the inner def/end blocks *)
+let%expect_test "ruby_blocks_5" =
+  let source = {| def (def a end) f (def b end) end |} in
+  let match_template = {|def :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {| <1>(def a end) f (def b end)</1> |}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {| def(df b ed) (df b ed)end |} in
+  let match_template = {|def:[1]end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {| <1>(df b ed) (df b ed)</1> |}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|class class ((def (x) end) f (def body end)) end end|} in
+  let match_template = {|class :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|<1>class ((def (x) end) f (def body end)) end</1>|}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|class class (def body1 end) (def body2 end) end end|} in
+  let match_template = {|def :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|class class (<1>body1</1>) (<1>body2</1>) end end|}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|class class (def body1 end) (def body2 end) end end|} in
+  let match_template = {|class :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|<1>class (def body1 end) (def body2 end) end</1>|}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|class class def (def body1 end) (def body2 end) end end end|} in
+  let match_template = {|def :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|class class <1>(def body1 end) (def body2 end)</1> end end|}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|class class def () (def body2 end) end end end|} in
+  let match_template = {|def :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|class class <1>() (def body2 end)</1> end end|}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|def (def end) (def end) end|} in
+  let match_template = {|def :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|<1>(def end) (def end)</1>|}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|def (def end) a (def end) end|} in
+  let match_template = {|def :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|<1>(def end) a (def end)</1>|}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|def (defa aend) (adef aend) end|} in
+  let match_template = {|def :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|<1>(defa aend) (adef aend)</1>|}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|def (defa aend) a (adef aend) end|} in
+  let match_template = {|def :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|<1>(defa aend) a (adef aend)</1>|}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|def (adef a endq) end|} in
+  let match_template = {|def :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|<1>(adef a endq)</1>|}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|def adef a endq end|} in
+  let match_template = {|def :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|<1>adef a endq</1>|}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|def def foo end end|} in
+  let match_template = {|def :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|<1>def foo end</1>|}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|def def end endq|} in
+  let match_template = {|def :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|No matches.|}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|def adef a endq end |} in
+  let match_template = {|def :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|<1>adef a endq</1> |}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|def fadef a qendq end|} in
+  let match_template = {|def :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|<1>fadef a qendq</1>|}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|def defa aend end|} in
+  let match_template = {|def :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|<1>defa aend</1>|}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|(adef a endq)|} in
+  let match_template = {|def :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|No matches.|}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|def adef a endq end|} in
+  let match_template = {|def :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|<1>adef a endq</1>|}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|def (adef a endq) end|} in
+  let match_template = {|def :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|<1>(adef a endq)</1>|}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|class (def ( body )end) end|} in
+  let match_template = {|class :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|<1>(def ( body )end)</1>|}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|class def ( body )end end|} in
+  let match_template = {|class :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|<1>def ( body )end</1>|}]
+
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|class def( body ) end end|} in
+  let match_template = {|class :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|<1>def( body ) end</1>|}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|class def() end end|} in
+  let match_template = {|class :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|<1>def() end</1>|}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|class def () end end|} in
+  let match_template = {|class :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|<1>def () end</1>|}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|class def( body )end end|} in
+  let match_template = {|class :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|<1>def( body )end</1>|}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|class ( def( body )end) end|} in
+  let match_template = {|class :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|<1>( def( body )end)</1>|}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|class (def( body )end) end|} in
+  let match_template = {|class :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|<1>(def( body )end)</1>|}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|(def a endq)|} in
+  let match_template = {|def :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|No matches.|}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|def  end|} in
+  let match_template = {|def :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|No matches.|}]
+
+let%expect_test "ruby_blocks_5" =
+  let source = {|(def foo end)|} in
+  let match_template = {|(def :[1] end)|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.Ruby) source match_template rewrite_template;
+  [%expect_exact {|<1>foo</1>|}]
+
+let%expect_test "ocaml_struct_end" =
+  let source = {|= struct Something end|} in
+  let match_template = {|= struct :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.OCaml) source match_template rewrite_template;
+  [%expect_exact {|<1>Something</1>|}]
+
+let%expect_test "ocaml_struct_end_2" =
+  let source = {|= struct include Something end|} in
+  let match_template = {|= struct :[1] end|} in
+  let rewrite_template = {|<1>:[1]</1>|} in
+
+  run (module Matchers.OCaml) source match_template rewrite_template;
+  [%expect_exact {|<1>include Something</1>|}]
+  *)

test/omega/test_statistics.ml

@@ -0,0 +1,88 @@
+(*open Core
+
+open Language
+open Matchers
+open Match
+
+let configuration = Configuration.create ~match_kind:Fuzzy ()
+
+let format s =
+  let s = s |> String.chop_prefix_exn ~prefix:"\n" in
+  let leading_indentation =
+    Option.value_exn (String.lfindi s ~f:(fun _ c -> c <> ' ')) in
+  s
+  |> String.split ~on:'\n'
+  |> List.map ~f:(Fn.flip String.drop_prefix leading_indentation)
+  |> String.concat ~sep:"\n"
+  |> String.chop_suffix_exn ~suffix:"\n"
+
+let %expect_test "statistics" =
+  let template =
+    {|
+      def :[fn_name](:[fn_params])
+    |}
+    |> format
+  in
+
+  let source =
+    {|
+      def foo(bar):
+        pass
+
+      def bar(bazz):
+        pass
+    |}
+    |> format
+  in
+
+  let rule =
+    {| where true
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+  Go.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } ->
+      Rule.(sat @@ apply rule environment))
+  |> fun matches ->
+  let statistics =
+    Statistics.
+      { number_of_files = 1
+      ; lines_of_code = 5
+      ; number_of_matches = List.length matches
+      ; total_time = 0.0
+      }
+  in
+  statistics
+  |> Statistics.to_yojson
+  |> Yojson.Safe.pretty_to_string
+  |> print_string;
+  [%expect {|
+    {
+      "number_of_files": 1,
+      "lines_of_code": 5,
+      "number_of_matches": 2,
+      "total_time": 0.0
+    } |}];
+
+  let statistics' =
+    Statistics.merge
+      { number_of_files = 1
+      ; lines_of_code = 10
+      ; number_of_matches = 1
+      ; total_time = 1.5
+      }
+      statistics
+  in
+  statistics'
+  |> Statistics.to_yojson
+  |> Yojson.Safe.pretty_to_string
+  |> print_string;
+  [%expect {|
+    {
+      "number_of_files": 2,
+      "lines_of_code": 15,
+      "number_of_matches": 3,
+      "total_time": 1.5
+    } |}]
+    *)

test/omega/test_string_literals.ml

@@ -0,0 +1,413 @@
+(*open Core
+
+open Matchers
+open Rewriter
+
+let format s =
+  let s = String.chop_prefix_exn ~prefix:"\n" s in
+  let leading_indentation = Option.value_exn (String.lfindi s ~f:(fun _ c -> c <> ' ')) in
+  s
+  |> String.split ~on:'\n'
+  |> List.map ~f:(Fn.flip String.drop_prefix leading_indentation)
+  |> String.concat ~sep:"\n"
+  |> String.chop_suffix_exn ~suffix:"\n"
+
+let configuration = Configuration.create ~match_kind:Fuzzy ()
+
+let all ?(configuration = configuration) template source =
+  C.all ~configuration ~template ~source
+
+let print_matches matches =
+  List.map matches ~f:Match.to_yojson
+  |> (fun matches -> `List matches)
+  |> Yojson.Safe.pretty_to_string
+  |> print_string
+
+let%expect_test "comments_in_string_literals_should_not_be_treated_as_comments_by_fuzzy" =
+  let source = {|"/*"(x)|} in
+  let template = {|(:[1])|} in
+  let rewrite_template = {|:[1]|} in
+  all template source
+  |> Rewrite.all ~source ~rewrite_template
+  |> (function
+      | Some { rewritten_source; _ } -> print_string rewritten_source
+      | None -> print_string "BROKEN EXPECT");
+  [%expect_exact {|"/*"x|}]
+
+let%expect_test "comments_in_string_literals_should_not_be_treated_as_comments_by_fuzzy_go_raw" =
+  let source = {|`//`(x)|} in
+  let template = {|(:[1])|} in
+  let rewrite_template = {|:[1]|} in
+  Go.all ~configuration ~template ~source
+  |> Rewrite.all ~source ~rewrite_template
+  |> (function
+      | Some { rewritten_source; _ } -> print_string rewritten_source
+      | None -> print_string "BROKEN EXPECT");
+  [%expect_exact {|`//`x|}]
+
+let%expect_test "tolerate_unbalanced_stuff_in_string_literals" =
+  let template = {|"("|} in
+  let source = {|"("|} in
+  let matches = all ~configuration template source in
+  print_matches matches;
+  [%expect_exact {|[
+  {
+    "range": {
+      "start": { "offset": 0, "line": 1, "column": 1 },
+      "end": { "offset": 3, "line": 1, "column": 4 }
+    },
+    "environment": [],
+    "matched": "\"(\""
+  }
+]|}]
+
+let%expect_test "base_literal_matching" =
+  let source = {|"hello"|} in
+  let match_template = {|":[1]"|} in
+  let rewrite_template = {|:[1]|} in
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|hello|}]
+
+let%expect_test "base_literal_matching" =
+  let source = {|rewrite ("hello") this string|} in
+  let match_template = {|rewrite (":[1]") this string|} in
+  let rewrite_template = {|:[1]|} in
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|hello|}]
+
+let%expect_test "match_string_literals" =
+  let source = {|rewrite (".") this string|} in
+  let match_template = {|rewrite (":[1]") this string|} in
+  let rewrite_template = {|:[1]|} in
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|.|}]
+
+let%expect_test "match_string_literals" =
+  let source = {|rewrite ("") this string|} in
+  let match_template = {|rewrite (":[1]") this string|} in
+  let rewrite_template = {|:[1]|} in
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {||}]
+
+let%expect_test "match_string_literals" =
+  let source = {|"(" match "a""a" this "(" |} in
+  let match_template = {|match :[1] this|} in
+  let rewrite_template = {|:[1]|} in
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|"(" "a""a" "(" |}]
+
+(* this tests special functionality in non-literal hole parser
+   but which must still ignore unbalanced delims within strings *)
+let%expect_test "match_string_literals" =
+  let source = {|"(" match "(""(" this "(" |} in
+  let match_template = {|match :[1] this|} in
+  let rewrite_template = {|:[1]|} in
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|"(" "(""(" "(" |}]
+
+let%expect_test "match_string_literals" =
+  let source = {|rewrite ("") this string|} in
+  let match_template = {|rewrite (:[1]) this string|} in
+  let rewrite_template = {|:[1]|} in
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|""|}]
+
+let%expect_test "base_literal_matching" =
+  let source = {|"("|} in
+  let match_template = {|":[1]"|} in
+  let rewrite_template = {|:[1]|} in
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|(|}]
+
+let%expect_test "base_literal_matching" =
+  let source = {|"(""("|} in
+  let match_template = {|":[1]"|} in
+  let rewrite_template = {|:[1]|} in
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|((|}]
+
+let%expect_test "base_literal_matching" =
+  let source = {|"(""("|} in
+  let match_template = {|":[1]"|} in
+  let rewrite_template = {|:[1]|} in
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|((|}]
+
+let%expect_test "base_literal_matching" =
+  let source = {|"hello world"|} in
+  let match_template = {|":[x] :[y]"|} in
+  let rewrite_template = {|:[x] :[y]|} in
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|hello world|}]
+
+(* complex test: basically, we are checking that the inside of this literal is only matched by the val b part *)
+let%expect_test "base_literal_matching" =
+  let source = {|val a = "class = ${String::class}" val b = "not $a"|} in
+  let match_template = {|":[x]$:[[y]]"|} in
+  let rewrite_template = {|(rewritten part: (:[x]) ([y]))|} in
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|val a = "class = ${String::class}" val b = (rewritten part: (not ) ([y]))|}]
+
+let%expect_test "base_literal_matching" =
+  let source = {|get("type") rekt ("enabled", True)|} in
+  let match_template = {|(":[1]", :[[3]])|} in
+  let rewrite_template = {|(rewritten part: (:[1]) (:[3]))|} in
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|get("type") rekt (rewritten part: (enabled) (True))|}]
+
+let%expect_test "rewrite_string_literals_8" =
+  let source = {|match "\"" this|} in
+  let match_template = {|match "\"" this|} in
+  let rewrite_template = "" in
+
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {||}]
+
+let%expect_test "rewrite_string_literals_8" =
+  let source = {|match "\"" this|} in
+  let match_template = {|match :[1] this|} in
+  let rewrite_template = ":[1]" in
+
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|"\""|}]
+
+let%expect_test "rewrite_string_literals_8" =
+  let source = {|match "\"\"" this|} in
+  let match_template = {|match :[1] this|} in
+  let rewrite_template = ":[1]" in
+
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|"\"\""|}]
+
+let%expect_test "rewrite_string_literals_8" =
+  let source = {|match "\"(\"" "(\"" this|} in
+  let match_template = {|match :[1] this|} in
+  let rewrite_template = ":[1]" in
+
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|"\"(\"" "(\""|}]
+
+let%expect_test "rewrite_string_literals_8" =
+  let source = {|match "\"(\"" "(\"" this|} in
+  let match_template = {|match ":[1]" ":[2]" this|} in
+  let rewrite_template = {|:[1] :[2]|} in
+
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|\"(\" (\"|}]
+
+let%expect_test "rewrite_string_literals_8" =
+  let source = {|match 'sin(gle' 'quo(tes' this|} in
+  let match_template = {|:[1]|} in
+  let rewrite_template = {|:[1]|} in
+
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|match 'sin(gle' 'quo(tes' this|}]
+
+let%expect_test "rewrite_string_literals_8" =
+  let source = {|match '\''|} in
+  let match_template = {|:[1]|} in
+  let rewrite_template = {|:[1]|} in
+
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|match '\''|}]
+
+let%expect_test "rewrite_string_literals_8" =
+  let source = {|match 'asdf'|} in
+  let match_template = {|':[1]'|} in
+  let rewrite_template = {|:[1]|} in
+
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|match asdf|}]
+
+let%expect_test "rewrite_string_literals_8" =
+  let source = {|match '\''|} in
+  let match_template = {|':[1]'|} in
+  let rewrite_template = {|:[1]|} in
+
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|match \'|}]
+
+
+let%expect_test "go_raw_string_literals" =
+  let source =
+    {|
+       x = x
+       y = `multi-line
+            raw str(ing literal`
+       z = `other multi-line
+            raw stri(ng literal`
+    |}
+  in
+  let match_template = {|`:[1]`|} in
+  let rewrite_template = {|:[1]|} in
+
+  Go.all ~configuration ~source ~template:match_template
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|
+       x = x
+       y = multi-line
+            raw str(ing literal
+       z = other multi-line
+            raw stri(ng literal
+    |}]
+
+let%expect_test "go_raw_string_literals" =
+  let source = {|blah `(` quux|} in
+  let match_template = {|:[1]|} in
+  let rewrite_template = {|:[1]|} in
+
+  Go.all ~configuration ~source ~template:match_template
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|blah `(` quux|}]
+
+let%expect_test "match_string_literals" =
+  let source = {|`(` match `(``(` this `(` |} in
+  let match_template = {|match :[1] this|} in
+  let rewrite_template = {|:[1]|} in
+  Go.all ~configuration ~template:match_template ~source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|`(` `(``(` `(` |}]
+
+
+let%expect_test "go_raw_string_literals" =
+  let source =
+    {|
+       x = x
+       y = `multi-line
+            raw "str"(ing literal`
+       z = `other multi-line
+            raw '"'\"\\s\\\\\tr\ni(ng literal`
+    |}
+  in
+  let match_template = {|`:[1]`|} in
+  let rewrite_template = {|:[1]|} in
+
+  Go.all ~configuration ~source ~template:match_template
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|
+       x = x
+       y = multi-line
+            raw "str"(ing literal
+       z = other multi-line
+            raw '"'\"\\s\\\\\tr\ni(ng literal
+    |}]
+
+let%expect_test "regression_matching_kubernetes" =
+  let source = {|"\n" y = 5|} in
+  let template = {|y = :[1]|} in
+  let rewrite_template = {|:[1]|} in
+  Go.all ~configuration ~template ~source
+  |> Rewrite.all ~source ~rewrite_template
+  |> (function
+      | Some { rewritten_source; _ } -> print_string rewritten_source
+      | None -> print_string "BROKEN EXPECT");
+  [%expect_exact {|"\n" 5|}]
+
+
+let%expect_test "match_escaped_any_char" =
+  let source = {|printf("hello world\n");|} in
+  let template = {|printf(":[1]");|} in
+  let rewrite_template = {|:[1]|} in
+  Go.all ~configuration ~template ~source
+  |> Rewrite.all ~source ~rewrite_template
+  |> (function
+      | Some { rewritten_source; _ } -> print_string rewritten_source
+      | None -> print_string "BROKEN EXPECT");
+  [%expect_exact {|hello world\n|}]
+
+let%expect_test "match_escaped_escaped" =
+  let source = {|printf("hello world\n\\");|} in
+  let template = {|printf(":[1]");|} in
+  let rewrite_template = {|:[1]|} in
+  Go.all ~configuration ~template ~source
+  |> Rewrite.all ~source ~rewrite_template
+  |> (function
+      | Some { rewritten_source; _ } -> print_string rewritten_source
+      | None -> print_string "BROKEN EXPECT");
+  [%expect_exact {|hello world\n\\|}]
+
+let%expect_test "match_escaped_escaped" =
+  let source = {|printf("hello world\n\");|} in
+  let template = {|printf(":[1]");|} in
+  let rewrite_template = {|:[1]|} in
+  Go.all ~configuration ~template ~source
+  |> Rewrite.all ~source ~rewrite_template
+  |> (function
+      | Some { rewritten_source; _ } -> print_string rewritten_source
+      | None -> print_string "EXPECT SUCCESS");
+  [%expect_exact {|EXPECT SUCCESS|}]
+  *)

test/omega/test_substring_disabled.ml

@@ -0,0 +1,433 @@
+(*open Core
+
+open Matchers
+open Rewriter
+
+let configuration = Configuration.create ~disable_substring_matching:true ~match_kind:Fuzzy ()
+
+let format s =
+  let s = String.chop_prefix_exn ~prefix:"\n" s in
+  let leading_indentation = Option.value_exn (String.lfindi s ~f:(fun _ c -> c <> ' ')) in
+  s
+  |> String.split ~on:'\n'
+  |> List.map ~f:(Fn.flip String.drop_prefix leading_indentation)
+  |> String.concat ~sep:"\n"
+  |> String.chop_suffix_exn ~suffix:"\n"
+
+let run ?(configuration = configuration) source match_template rewrite_template =
+  Generic.first ~configuration match_template source
+  |> function
+  | Ok result ->
+    Rewrite.all ~source ~rewrite_template [result]
+    |> (fun x -> Option.value_exn x)
+    |> (fun { rewritten_source; _ } -> rewritten_source)
+    |> print_string
+  | Error _ ->
+    (* this is too annoying to fix everytime the grammar changes. *)
+    print_string ""
+
+let run_all ?(configuration = configuration) source match_template rewrite_template =
+  Generic.all ~configuration ~template:match_template ~source
+  |> function
+  | [] -> print_string "No matches."
+  | results ->
+    Option.value_exn (Rewrite.all ~source ~rewrite_template results)
+    |> (fun { rewritten_source; _ } -> rewritten_source)
+    |> print_string
+
+
+let%expect_test "basic" =
+  let source = {|a b c d|} in
+  let match_template = {|:[1]|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|a b c d|}];
+
+  let source = {|a b c d|} in
+  let match_template = {|a :[1] c d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|b|}];
+
+  let source = {|a b c d|} in
+  let match_template = {|a :[1] d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|b c|}];
+
+  let source = {|a b c d|} in
+  let match_template = {|a :[1]|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|b c d|}];
+
+  let source = {|a b c d|} in
+  let match_template = {|:[1] c d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|a b|}];
+
+  let source = {|a b c d|} in
+  let match_template = {|:[1] :[2]|} in
+  let rewrite_template = {|(:[1]) (:[2])|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(a) (b c d)|}];
+
+  let source = {|a b c d|} in
+  let match_template = {|:[2] :[1]|} in
+  let rewrite_template = {|(:[2]) (:[1])|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(a) (b c d)|}];
+
+  let source = {|a b c d|} in
+  let match_template = {|a :[2] :[1] d|} in
+  let rewrite_template = {|(:[2]) (:[1])|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(b) (c)|}];
+
+  let source = {|a b c d|} in
+  let match_template = {|a :[2] :[1]|} in
+  let rewrite_template = {|(:[2]) (:[1])|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(b) (c d)|}];
+
+  let source = {|a b c d|} in
+  let match_template = {|a :[2] c :[1]|} in
+  let rewrite_template = {|(:[2]) (:[1])|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(b) (d)|}];
+
+  let source = {|x:|} in
+  let match_template = {|:[1]:|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|x|}]
+
+let%expect_test "basic_failures" =
+  let source = {|a x b bbq|} in
+  let match_template = {|a :[1] b c|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {||}];
+
+  let source = {|a b c d|} in
+  let match_template = {|a :[2] d :[1]|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact
+    {||}];
+
+  let source = {|a b c d|} in
+  let match_template = {|a :[2] b :[1]|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {||}]
+
+let%expect_test "delimiter_matching" =
+  let source = {|(a b c) d|} in
+  let match_template = {|(:[1]) d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|a b c|}];
+
+  let source = {|(a b c) d|} in
+  let match_template = {|(:[1] b :[2]) d|} in
+  let rewrite_template = {|(:[1]) (:[2])|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(a) (c)|}];
+
+  let source = {|q(a b c) d|} in
+  let match_template = {|q(:[1] b :[2]) d|} in
+  let rewrite_template = {|(:[1]) (:[2])|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(a) (c)|}];
+
+  let source = {|((a) b)|} in
+  let match_template = {|(:[1] b)|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(a)|}];
+
+  let source = {|((a b c)) d|} in
+  let match_template = {|(:[1]) d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(a b c)|}];
+
+  let source = {|((a b c)) d|} in
+  let match_template = {|(:[1]) d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(a b c)|}];
+
+  let source = {|((a b c) q) d|} in
+  let match_template = {|((:[1]) q) d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|a b c|}];
+
+  let source = {|((a b c) q) d|} in
+  let match_template = {|((:[1] c) q) d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|a b|}];
+
+  let source = {|((a b () c) q) d|} in
+  let match_template = {|((:[1] () c) q) d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|a b|}];
+
+  let source = {|((a ((x) d) b c)) d|} in
+  let match_template = {|((a :[1] :[2] c)) d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|((x) d)|}];
+
+  let source = {|((a ((x) d) b c)) d|} in
+  let match_template = {|((a (:[1]) :[2] c)) d|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(x) d b|}];
+
+  let source = {|(b (c) d)|} in
+  let match_template = {|(:[1])|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|b (c) d|}];
+
+  let source = {|(b (c) d.)|} in
+  let match_template = {|(:[1].)|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|b (c) d|}];
+
+  let source = {|(b (c.) d.)|} in
+  let match_template = {|(:[1].)|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|b (c.) d|}];
+
+  let source = {|(b. (c) d.)|} in
+  let match_template = {|(:[1].)|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|b. (c) d|}];
+
+  let source = {|(b (c) d.)|} in
+  let match_template = {|(b :[1] d.)|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(c)|}];
+
+  let source = {|outer(inner(dst,src),src)|} in
+  let match_template = {|outer(:[1],src)|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|inner(dst,src)|}];
+
+  let source = {|(b ((c)) d.)|} in
+  let match_template = {|(b :[1] d.)|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|((c))|}];
+
+  let source = {|a b c|} in
+  let match_template = {|a :[1] c|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|b|}];
+
+  let source = {|x = foo;|} in
+  let match_template = {|x = :[1];|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|foo|}];
+
+  let source = {|((a {{x} d} b c)) d|} in
+  let match_template = {|((a {:[1] d} :[2] c)) d|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|{x} b|}];
+
+  let source = {|((a {([{x}]) d} b c)) d|} in
+  let match_template = {|((a {:[1] d} :[2] c)) d|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|([{x}]) b|}];
+
+  let source = {|(((((x)))))|} in
+  let match_template = {|(((:[1])))|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|((x))|}];
+
+  let source = {|((((y(x)z))))|} in
+  let match_template = {|(((:[1])))|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(y(x)z)|}];
+
+  let source = {|((((y(x)z))))|} in
+  let match_template = {|(((:[1]):[2]))|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(y(x)z) |}];
+
+  let source = {|(((x)z))|} in
+  let match_template = {|(((:[1]):[2]))|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|x z|}];
+
+  let source = {|((((x))z))|} in
+  let match_template = {|(((:[1]):[2]))|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(x) z|}];
+
+  let source = {|lolwtfbbq|} in
+  let match_template = {|lol:[1]bbq|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {||}];
+
+  let source = {|x = foo; x = bar;|} in
+  let match_template = {|x = :[1];|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|foo x = bar;|}];
+
+  let source = {|[ no match prefix ] x = foo; [ no match suffix ]|} in
+  let match_template = {|x = :[1];|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|[ no match prefix ] foo [ no match suffix ]|}];
+
+  let source = {|x = a; x = b; x = c|} in
+  let match_template = {|x = :[1];|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|a x = b; x = c|}];
+
+  let source = {|x = ( x = x; );|} in
+  let match_template = {|x = :[1];|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|( x = x; )|}];
+
+  let source = {|( x = x = x; )|} in
+  let match_template = {|x = :[1];|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|( x = x )|}];
+
+  let source = {|xxx a b d c 1 2 3 b d d blah|} in
+  let match_template = {|a :[1] c :[2] d|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|xxx b d 1 2 3 b d blah|}];
+
+  let source = {|howevenlolwtfbbqispossible|} in
+  let match_template = {|lol:[1]bbq|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {||}];
+
+  let source = {|lolhowevenlolwtfbbqispossiblebbq|} in
+  let match_template = {|lol:[1]bbq|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {||}];
+
+  let source = {|hello my name is bob the builder|} in
+  let match_template = {|:[alongidentifiername] :[2] :[3] :[xyz] :[5] :[6]|} in
+  let rewrite_template = {|:[alongidentifiername] :[2] :[3] :[xyz] :[5] :[6]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|hello my name is bob the builder|}];
+
+  let source = {|www.testdofooname.com/picsinsideit/stunningpictureofkays1381737242g8k4n-280x428.jpg|} in
+  let match_template = {|www.:[1]-:[2].jpg|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|testdofooname.com/picsinsideit/stunningpictureofkays1381737242g8k4n 280x428|}];
+
+  let source = {|https://api.github.com/repos/dmjacobsen/slurm/commits/716c1499695c68afcab848a1b49653574b4fc167|} in
+  let match_template = {|:[1]api.:[2]/repos/:[3]s/:[4]|} in
+  let rewrite_template = {|:[1] :[2] :[3] :[4]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {||}];
+
+  let source =
+    {|
+      assert(stream->md_len + md_len -
+             si.foo_data_begin <= MAD_BUFFER_MDLEN);
+      memcpy(*stream->foo_data + stream->md_len,
+             mad_bit_nextbyte(&stream->ptr),
+             frame_used = md_len - si.foo_data_begin);
+      stream->md_len += frame_used;
+    |} |> format
+  in
+  let match_template = {|memcpy(:[1], :[2], :[3]);|} in
+  let rewrite_template = {|:[1], :[2], :[3]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|assert(stream->md_len + md_len -
+       si.foo_data_begin <= MAD_BUFFER_MDLEN);
+*stream->foo_data + stream->md_len, mad_bit_nextbyte(&stream->ptr), frame_used = md_len - si.foo_data_begin
+stream->md_len += frame_used;|}]
+
+let%expect_test "significant_whitespace" =
+  let configuration = Configuration.create ~match_kind:Fuzzy ~significant_whitespace:true () in
+  let run = run ~configuration in
+
+  let source = {|two  spaces|} in
+  let match_template = {|:[1]  :[2]|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|two spaces|}];
+
+  (* FIXME: this should fail. also test case where separators do or do not need
+     whitespace.  e.g., strict about strcpy(src,dst) matching a template
+     strcpy(:[1],:[2]) versus strcpy(:[1], :[2]) *)
+  let source = {|two  spaces|} in
+  let match_template = {|:[1] :[2]|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|two spaces|}]
+
+let%expect_test "contextual_matching" =
+  let run = run_all in
+
+  let source = {|memcpy(dst1, src1, 1); memcpy(dst2, src2, 2);|} in
+  let match_template = {|memcpy(:[1], :[2], :[3])|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|dst1; dst2;|}];
+
+  let source = {|memcpy(dst1, src1, 1); memcpy(dst2, src2, 2);|} in
+  let match_template = {|memcpy(:[1], :[2], :[3])|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|dst1; dst2;|}]
+
+let%expect_test "contextual_matching_with_short_hole_syntax" =
+  let run = run_all in
+
+  let source = {|memcpy(dst1, src1, 1); memcpy(dst2, src2, 2);|} in
+  let match_template = {|memcpy(:[[1]], :[2], :[3])|} in
+  let rewrite_template = {|:[[1]]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|dst1; dst2;|}]
+
+let%expect_test "trivial_empty_case" =
+  let source = "" in
+  let match_template = "" in
+  begin
+    Generic.all ~configuration ~template:match_template ~source
+    |> function
+    | [] -> print_string "No matches."
+    | hd :: _ ->
+      print_string (Yojson.Safe.to_string (Match.to_yojson hd))
+  end;
+  [%expect_exact {|{"range":{"start":{"offset":0,"line":1,"column":1},"end":{"offset":0,"line":1,"column":1}},"environment":[],"matched":""}|}]
+  *)