Initial code commit

.gitignore

@@ -0,0 +1,3 @@
+.merlin
+_build
+*.install

LICENSE

@@ -41,7 +41,7 @@
       form, that is based on (or derived from) the Work and for which the
       editorial revisions, annotations, elaborations, or other modifications
       represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
+      of this License, Derivative Works shall not include works that refoo
       separable from, or merely link (or bind by name) to the interfaces of,
       the Work and Derivative Works thereof.
 

Makefile

@@ -0,0 +1,27 @@
+all: build comby
+
+build:
+	@dune build
+
+comby comby-server:
+	@ln -s _build/install/default/bin/$@ ./$@
+
+install:
+	@dune install
+
+doc:
+	@dune build @doc
+
+test:
+	@dune runtest
+
+clean:
+	@dune clean
+
+uninstall:
+	@dune uninstall
+
+promote:
+	@dune promote
+
+.PHONY: all build install test clean promote

README.md

@@ -1 +1,171 @@
-# comby
+# comby
+
+## Build
+
+- Install [opam](https://opam.ocaml.org/doc/Install.html)
+
+- Create a new switch if you don't have OCaml installed
+
+```
+opam init
+opam switch create 4.05.0 4.05.0 
+```
+
+- Install dependencies
+
+### Linux
+
+```
+sudo apt-get install pkg-config pcre
+```
+
+### Mac
+
+```
+brew install pkg-config pcre
+```
+
+- Install opam libraries
+
+```
+opam install ppx_deriving_yojson
+opam install core
+opam install ppxlib
+opam install ppx_deriving
+opam install angstrom
+opam install hack_parallel
+opam install opium
+opam install pcre
+opam install oasis
+```
+
+- Install mparser
+
+```
+git clone https://github.com/comby-tools/mparser
+oasis setup
+ocaml setup.ml -configure --enable-pcre --enable-re
+ocaml setup.ml -build
+ocaml setup.ml -install
+```
+
+- Build and test
+
+```
+make
+make test
+```
+
+## Running
+
+`./comby MATCH_TEMPLATE REWRITE_TEMPLATE` by default rewrites all files
+in-place in the current directory and all subdirectories. 
+
+- Adding `-f .ml` will rewrite all files ending in `.ml`. Adding `-f README.md` will rewrite all files
+ending in `README.md`
+
+- Input can also be fed via `-stdin` instead of rewriting files:
+
+```
+./comby -stdin 'printf(":[1] :[2]")' 'printf("comby, :[1]")' << EOF 2> /dev/null
+int main(void) {
+  printf("hello world");
+}
+EOF
+```
+
+Outputs:
+
+```
+int main(void) {
+  printf("comby, hello");
+}
+```
+- Adding a `-json` flag will output JSON content of the rewrite:
+
+```
+[
+  {
+    "range": {
+      "start": { "offset": 19, "line": -1, "column": -1 },
+      "end": { "offset": 41, "line": -1, "column": -1 }
+    },
+    "replacement_content": "printf(\"comby, hello\")",
+    "environment": [
+      [
+        "1",
+        {
+          "value": "hello",
+          "range": {
+            "start": { "offset": 15, "line": -1, "column": -1 },
+            "end": { "offset": 20, "line": -1, "column": -1 }
+          }
+        }
+      ]
+    ]
+  }
+]
+```
+
+- Adding a `-match-only` flag will output JSON content of all matches:
+
+```
+[
+  {
+    "range": {
+      "start": { "offset": 19, "line": 1, "column": 20 },
+      "end": { "offset": 40, "line": 1, "column": 41 }
+    },
+    "environment": [
+      [
+        "1",
+        {
+          "value": "hello",
+          "range": {
+            "start": { "offset": 27, "line": 1, "column": 28 },
+            "end": { "offset": 32, "line": 1, "column": 33 }
+          }
+        }
+      ],
+      [
+        "2",
+        {
+          "value": "world",
+          "range": {
+            "start": { "offset": 33, "line": 1, "column": 34 },
+            "end": { "offset": 38, "line": 1, "column": 39 }
+          }
+        }
+      ]
+    ],
+    "matched": "printf(\"hello world\")"
+  }
+]
+```
+
+See other flags for more options with `./comby -h`:
+
+```
+Run a rewrite pass.
+
+  comby [MATCH_TEMPLATE REWRITE_TEMPLATE]
+
+=== flags ===
+
+  [-directory path]     Run on files in a directory. Default is current
+                        directory: /Users/rvt/comby
+  [-filter extensions]  CSV of extensions to include
+  [-jobs n]             Number of worker processes
+  [-json]               Output JSON format for matches or rewrite text to stdout
+  [-match-only]         Only perform matching (ignore rewrite templates)
+  [-rule rule]          Apply rules to matches. Respects -f
+  [-sequential]         Run sequentially
+  [-stdin]              Read source from stdin
+  [-templates path]     CSV of directories containing templates
+  [-timeout seconds]    Set match timeout on a source. Default: 3
+  [-verbose]            Log to /tmp/comby.out
+  [-build-info]         print info about this build and exit
+  [-version]            print the version of this build and exit
+  [-help]               print this help text and exit
+                        (alias: -?)
+```

comby

@@ -0,0 +1 @@
+_build/install/default/bin/comby

comby.opam

@@ -0,0 +1,13 @@
+opam-version: "2.0"
+version: "0.1.0"
+maintainer: "rvantonder@gmail.com"
+authors: ["Rijnard van Tonder"]
+homepage: "https://github.com/comby-tools/comby"
+bug-reports: "https://github.com/comby-tools/comby/issues"
+dev-repo: "git+https://github.com/comby-tools/comby.git"
+license: "Apache-2.0"
+build: [["jbuilder" "build" "-p" name "-j" jobs "@install"]]
+depends: [
+  "mparser"
+  "core"
+]

dune

@@ -0,0 +1,3 @@
+(env
+ (dev
+  (flags (:standard -w A-3-4-32-34-39-40-41-42-44-45-48-49-50-57))))

dune-project

@@ -0,0 +1,2 @@
+(lang dune 1.8)
+(name comby)

lib/comby.ml

@@ -0,0 +1,5 @@
+module Language = Language
+module Matchers = Matchers
+module Match = Match
+module Rewriter = Rewriter
+module Statistics = Statistics

lib/comby.mli

@@ -0,0 +1,5 @@
+module Language = Language
+module Matchers = Matchers
+module Match = Match
+module Rewriter = Rewriter
+module Statistics = Statistics

lib/dune

@@ -0,0 +1,14 @@
+(library
+  (name comby)
+  (public_name comby)
+  (preprocess (pps ppx_deriving.show ppx_deriving.eq ppx_sexp_conv))
+  (libraries 
+   ppxlib 
+   core 
+   mparser 
+   mparser.pcre 
+   comby.matchers
+   comby.rewriter
+   comby.match
+   comby.language
+   comby.statistics))

lib/language/ast.ml

@@ -0,0 +1,30 @@
+open Core
+
+type atom =
+  | Variable of string
+  | String of string
+[@@deriving sexp]
+
+type antecedent = atom
+[@@deriving sexp]
+
+type expression =
+  | True
+  | False
+  | Equal of atom * atom
+  | Not_equal of atom * atom
+  | Match of atom * (antecedent * consequent) list
+  | RewriteTemplate of string
+  | Rewrite of atom * (antecedent * consequent) list
+  | Rewrite_old of atom * ((atom * atom * expression list) list)
+and consequent = expression list
+[@@deriving sexp]
+
+let (=) left right = Equal (left, right)
+
+let (<>) left right = Not_equal (left, right)
+
+(* Semantics for rewrite_rule with a list more than one expression is
+   undefined *)
+type t = expression list
+[@@deriving sexp]

lib/language/dune

@@ -0,0 +1,5 @@
+(library
+  (name language)
+  (public_name comby.language)
+  (preprocess (pps ppx_deriving.show ppx_sexp_conv ppx_sexp_message ppx_deriving_yojson))
+  (libraries comby.parsers comby.match comby.rewriter comby.matchers ppxlib core mparser mparser.pcre yojson ppx_deriving_yojson ppx_deriving_yojson.runtime))

lib/language/parser.ml

@@ -0,0 +1,25 @@
+open Core
+
+open MParser
+open MParser_PCRE.Tokens
+
+open Ast
+
+let variable_parser s =
+  (string Syntax.variable_left_delimiter
+   >> (many (alphanum <|> char '_') |>> String.of_char_list)
+      << string Syntax.variable_right_delimiter) s
+
+let value_parser s =
+  string_literal s
+
+let operator_parser s =
+  ((string Syntax.equal)
+   <|> (string Syntax.not_equal)) s
+
+let atom_parser s =
+  ((variable_parser >>= fun variable -> return (Variable variable))
+   <|> (value_parser >>= fun value -> return (String value))) s
+
+let rewrite_template_parser s =
+  (value_parser >>= fun value -> return (RewriteTemplate value)) s

lib/language/rule.ml

@@ -0,0 +1,233 @@
+open Core
+open MParser
+open MParser_PCRE.Tokens
+
+open Match
+open Matchers
+
+open Rewriter
+
+open Ast
+open Parser
+
+type result = bool * environment option
+
+let sat = fst
+
+let result_env = snd
+
+let match_configuration_of_syntax template =
+  (* decide match configuration based on whether there are holes *)
+  let antecedent_contains_hole_syntax case =
+    String.is_substring case ~substring:Syntax.variable_left_delimiter
+  in
+  if antecedent_contains_hole_syntax template then
+    Configuration.create ~match_kind:Fuzzy ()
+  else
+    Configuration.create ~match_kind:Exact ()
+
+let merge_match_environments matches environment' =
+  List.map matches ~f:(fun { environment; _ } ->
+      Environment.merge environment environment')
+
+type rewrite_context =
+  { variable : string }
+
+let rec apply ?(matcher = (module Matchers.Generic : Matchers.Matcher)) predicates env =
+  let open Option in
+  let module Matcher = (val matcher : Matchers.Matcher) in
+
+  let equal_in_environment var value env =
+    match Environment.lookup env var with
+    | None -> false, Some env
+    | Some var_value -> String.equal var_value value, Some env
+  in
+  (* accepts only one expression *)
+  let rec rule_match ?(rewrite_context : rewrite_context option) env =
+    function
+    | True -> true, Some env
+    | False -> false, Some env
+    | Equal (Variable var, String value)
+    | Equal (String value, Variable var) ->
+      equal_in_environment var value env
+    | Equal (String left, String right) ->
+      String.equal left right, Some env
+    | Equal (Variable left, Variable right) ->
+      let result =
+        Environment.lookup env left >>= fun left ->
+        Environment.lookup env right >>= fun right ->
+        return (String.equal left right)
+      in
+      Option.value result ~default:false, Some env
+    | Not_equal (left, right) ->
+      let sat, env = rule_match env (Equal (left, right)) in
+      not sat, env
+    | Match (Variable variable, cases) ->
+      let result =
+        Environment.lookup env variable >>= fun source ->
+        List.find_map cases ~f:(fun (template, case_expression) ->
+            match template with
+            | String template ->
+              begin
+                let configuration = match_configuration_of_syntax template in
+                Matcher.all ~configuration ~template ~source |> function
+                | [] -> None
+                | matches ->
+                  (* merge environments. overwrite behavior is undefined *)
+                  let fold_matches (sat, out) { environment; _ } =
+                    let fold_cases (sat, out) predicate =
+                      if sat then
+                        let env' = Environment.merge env environment in
+                        rule_match ?rewrite_context env' predicate
+                      else
+                        (sat, out)
+                    in
+                    List.fold case_expression ~init:(sat, out) ~f:fold_cases
+                  in
+                  List.fold matches ~init:(true, None) ~f:fold_matches
+                  |> Option.some
+              end
+            | Variable _ ->
+              failwith "| :[hole] is invalid. Maybe you meant to put quotes")
+      in
+      Option.value_map result ~f:ident ~default:(false, Some env)
+    | Match (String template, cases) ->
+      let source, _ = Rewriter.Rewrite_template.substitute template env in
+      let fresh_var = Uuid.(Fn.compose to_string create ()) in
+      let env = Environment.add env fresh_var source in
+      rule_match env (Match (Variable fresh_var, cases))
+    | RewriteTemplate rewrite_template ->
+      begin
+        match rewrite_context with
+        | None -> false, None
+        | Some { variable; _ } ->
+          (* FIXME(RVT) assumes only contextual rewrite for now. *)
+          let env =
+            Rewrite_template.substitute rewrite_template env
+            |> fst
+            |> fun replacement' ->
+            Environment.update env variable replacement'
+            |> Option.some
+          in
+          true, env
+      end
+    | Rewrite (Variable variable, cases) ->
+      let result =
+        Environment.lookup env variable >>= fun source ->
+        List.find_map cases ~f:(fun (template, case_expression) ->
+            match template with
+            | String template ->
+              begin
+                let configuration = match_configuration_of_syntax template in
+                let matches = Matcher.all ~configuration ~template ~source in
+                if List.is_empty matches then
+                  None
+                else
+                  let fold_cases (sat, out) predicate =
+                    if sat then
+                      let env =
+                        match out with
+                        | Some out -> Environment.merge out env
+                        | None -> env
+                      in
+                      match matches with
+                      | { environment; _ } :: _ ->
+                        let env = Environment.merge env environment in
+                        rule_match ~rewrite_context:{ variable } env predicate
+                      | _ ->
+                        sat, out
+                    else
+                      (sat, out)
+                  in
+                  List.fold case_expression ~init:(true, None) ~f:fold_cases
+                  |> Option.some
+              end
+            | Variable _ ->
+              failwith "| :[hole] is invalid. Maybe you meant to put quotes")
+      in
+      Option.value_map result ~f:ident ~default:(false, Some env)
+    | Rewrite _ -> failwith "TODO"
+    | Rewrite_old _ -> failwith "Deprecated"
+  in
+  List.fold predicates ~init:(true, None) ~f:(fun (sat, out) predicate ->
+      if sat then
+        let env =
+          Option.value_map out
+            ~f:(fun out -> Environment.merge out env)
+            ~default:env
+        in
+        rule_match env predicate
+      else
+        (sat, out))
+
+
+let make_equality_expression operator left right =
+  if String.equal operator Syntax.equal then
+    return (Equal (left, right))
+  else if
+    String.equal operator Syntax.not_equal then
+    return (Not_equal (left, right))
+  else
+    let message =
+      Format.sprintf
+        "Unhandled operator %s. Did you mean %s or %s?"
+        operator
+        Syntax.equal
+        Syntax.not_equal in
+    fail message
+
+let create rule =
+  let operator_parser =
+    spaces >> atom_parser >>= fun left ->
+    spaces >> operator_parser >>= fun operator ->
+    spaces >> atom_parser << spaces >>= fun right ->
+    make_equality_expression operator left right << spaces
+  in
+  let true' = spaces >> string Syntax.true' << spaces |>> fun _ -> True in
+  let false' = spaces >> string Syntax.false' << spaces |>> fun _ -> False in
+  let rec expression_parser s =
+    choice
+      [ pattern_parser
+      (* string literals are ambiguous, so attempt to parse operator first *)
+      ; attempt operator_parser
+      ; rewrite_template_parser
+      ; true'
+      ; false'
+      ]
+      s
+  and pattern_parser s =
+    let case_parser : (atom * expression list, unit) parser =
+      spaces >> string Syntax.pipe_operator >>
+      spaces >> atom_parser << spaces << string Syntax.arrow << spaces >>= fun antecedent ->
+      spaces >> comma_sep expression_parser << spaces |>> fun consequent ->
+      antecedent, consequent
+    in
+    let pattern keyword =
+      string keyword << spaces >> atom_parser << spaces << char '{' << spaces
+      >>= fun atom ->
+      many1 case_parser
+      << char '}' << spaces
+      >>= fun cases -> return (atom, cases)
+    in
+    let match_pattern =
+      pattern Syntax.start_match_pattern |>> fun (atom, cases) ->
+      (Match (atom, cases))
+    in
+    let rewrite_pattern =
+      pattern Syntax.start_rewrite_pattern |>> fun (atom, cases) ->
+      (Rewrite (atom, cases))
+    in
+    choice [ match_pattern; rewrite_pattern ]
+      s
+  in
+  let rule_parser s =
+    (spaces
+     >> string Syntax.rule_prefix
+     >> spaces
+     >> comma_sep expression_parser
+        << eof)
+      s
+  in
+  match parse_string rule_parser rule () with
+  | Success rule -> Ok rule
+  | Failed (msg, _) -> Or_error.error_string msg

lib/language/rule.mli

@@ -0,0 +1,20 @@
+open Core
+
+open Matchers
+open Match
+
+open Ast
+
+type result = bool * environment option
+
+val sat : result -> bool
+
+val result_env : result -> environment option
+
+val create : string -> expression list Or_error.t
+
+val apply
+  :  ?matcher:(module Matcher)
+  -> t
+  -> environment
+  -> result

lib/language/syntax.ml

@@ -0,0 +1,11 @@
+let rule_prefix = "where"
+let start_match_pattern = "match"
+let start_rewrite_pattern = "rewrite"
+let equal = "=="
+let not_equal = "!="
+let variable_left_delimiter = ":["
+let variable_right_delimiter = "]"
+let true' = "true"
+let false' = "false"
+let pipe_operator = "|"
+let arrow = "->"

lib/match/dune

@@ -0,0 +1,5 @@
+(library
+  (name match)
+  (public_name comby.match)
+  (preprocess (pps ppx_deriving.show ppx_deriving.eq ppx_sexp_conv ppx_sexp_message ppx_deriving_yojson))
+  (libraries comby.parsers ppxlib core mparser mparser.pcre yojson ppx_deriving_yojson ppx_deriving_yojson.runtime))

lib/match/environment.ml

@@ -0,0 +1,74 @@
+open Core
+
+module Data = struct
+  type t =
+    { value : string
+    ; range : Range.t
+    }
+  [@@deriving yojson, eq, sexp]
+end
+
+open Data
+type data = Data.t
+[@@deriving yojson, eq, sexp]
+
+type t = data Core.String.Map.t
+
+type t_alist = (string * data) list
+[@@deriving yojson, eq]
+
+let create () : t =
+  String.Map.empty
+
+let vars (env : t) : string list =
+  Map.keys env
+
+let add ?(range = Range.default) (env : t) (var : string) (value : string) : t =
+  Map.add env ~key:var ~data:{ value; range }
+  |> function
+  | `Duplicate -> env
+  | `Ok env -> env
+
+let lookup (env : t) (var : string) : string option =
+  Map.find env var
+  |> Option.map ~f:(fun { value; _ } -> value)
+
+let lookup_range (env : t) (var : string) : Range.t option =
+  Map.find env var
+  |> Option.map ~f:(fun { range; _ } -> range)
+
+let fold (env : t) =
+  Map.fold env
+
+let update env var value =
+  Map.change env var ~f:(Option.map ~f:(fun result -> { result with value }))
+
+let update_range env var range =
+  Map.change env var ~f:(Option.map ~f:(fun result -> { result with range }))
+
+let to_string env =
+  Map.fold env ~init:"" ~f:(fun ~key:variable ~data:{ value; _ } acc ->
+      Format.sprintf "%s |-> %s\n%s" variable value acc)
+
+let furthest_match env =
+  Map.fold
+    env
+    ~init:0
+    ~f:(fun ~key:_ ~data:{ range = { match_start = { offset; _ }; _ }; _ } max ->
+        Int.max offset max)
+
+let equal env1 env2 =
+  Map.equal Data.equal env1 env2
+
+let merge env1 env2 =
+  Map.merge_skewed env1 env2 ~combine:(fun ~key:_ v1 _ -> v1)
+
+let copy env =
+  fold env ~init:(create ()) ~f:(fun ~key ~data:{ value; range } env' ->
+      add ~range env' key value)
+
+let to_yojson env =
+  Map.to_alist env
+  |> t_alist_to_yojson
+
+let of_yojson _ = assert false

lib/match/environment.mli

@@ -0,0 +1,26 @@
+type t
+[@@deriving yojson]
+
+val create : unit -> t
+
+val vars : t -> string list
+
+val add : ?range:Range.t -> t -> string -> string -> t
+
+val lookup : t -> string -> string option
+
+val update : t -> string -> string -> t
+
+val lookup_range : t -> string -> Range.t option
+
+val update_range : t -> string -> Range.t -> t
+
+val furthest_match : t -> int
+
+val equal : t -> t -> bool
+
+val merge : t -> t -> t
+
+val copy : t -> t
+
+val to_string : t -> string

lib/match/location.ml

@@ -0,0 +1,14 @@
+open Core
+
+type t =
+  { offset : int
+  ; line : int
+  ; column : int
+  }
+[@@deriving yojson, eq, sexp]
+
+let default =
+  { offset = -1
+  ; line = -1
+  ; column = -1
+  }

lib/match/match.ml

@@ -0,0 +1,6 @@
+module Location = Location
+module Range = Range
+module Environment = Environment
+
+include Types
+include Match_context

lib/match/match.mli

@@ -0,0 +1,67 @@
+module Location : sig
+  type t =
+    { offset : int
+    ; line : int
+    ; column : int
+    }
+  [@@deriving yojson, eq, sexp]
+
+  val default : t
+end
+
+type location = Location.t
+[@@deriving yojson, eq, sexp]
+
+module Range : sig
+  type t =
+    { match_start : location [@key "start"]
+    ; match_end : location [@key "end"]
+    }
+  [@@deriving yojson, eq, sexp]
+
+  val default : t
+end
+
+type range = Range.t
+[@@deriving yojson, eq, sexp]
+
+module Environment : sig
+  type t
+  [@@deriving yojson, eq]
+
+  val create : unit -> t
+
+  val vars : t -> string list
+
+  val add : ?range:range -> t -> string -> string -> t
+
+  val lookup : t -> string -> string option
+
+  val update : t -> string -> string -> t
+
+  val lookup_range : t -> string -> range option
+
+  val update_range : t -> string -> range -> t
+
+  val furthest_match : t -> int
+
+  val equal : t -> t -> bool
+
+  val copy : t -> t
+
+  val merge : t -> t -> t
+
+  val to_string : t -> string
+end
+
+type environment = Environment.t
+[@@deriving yojson]
+
+type t =
+  { range : range
+  ; environment : environment
+  ; matched : string
+  }
+[@@deriving yojson]
+
+val create : unit -> t

lib/match/match_context.ml

@@ -0,0 +1,12 @@
+type t =
+  { range : Range.t
+  ; environment : Environment.t
+  ; matched : string
+  }
+[@@deriving yojson]
+
+let create () =
+  { range = Range.default
+  ; environment = Environment.create ()
+  ; matched = ""
+  }

lib/match/range.ml

@@ -0,0 +1,10 @@
+type t =
+  { match_start : Location.t [@key "start"]
+  ; match_end : Location.t [@key "end"]
+  }
+[@@deriving yojson, eq, sexp]
+
+let default =
+  { match_start = Location.default
+  ; match_end = Location.default
+  }

lib/match/types.ml

@@ -0,0 +1,8 @@
+type location = Location.t
+[@@deriving yojson, eq, sexp]
+
+type range = Range.t
+[@@deriving yojson, eq, sexp]
+
+type environment = Environment.t
+[@@deriving yojson]

lib/matchers/bash.ml

@@ -0,0 +1,28 @@
+open MParser
+
+module Syntax = struct
+  include Generic.Syntax
+
+  let user_defined_delimiters =
+    [ ("if", "fi")
+    ; ("case", "esac")
+    ]
+    @ Generic.Syntax.user_defined_delimiters
+
+  let escapable_string_literals =
+    [ {|"|}
+    ; {|'|}
+    ]
+
+  let escape_char =
+    '\\'
+
+  let raw_string_literals =
+    []
+
+  let comment_parser s =
+    (Parsers.Comments.c_multiline
+     <|> Parsers.Comments.c_newline) s
+end
+
+include Matcher.Make(Syntax)

lib/matchers/c.ml

@@ -0,0 +1,22 @@
+open MParser
+
+module Syntax = struct
+  include Generic.Syntax
+
+  let escapable_string_literals =
+    [ {|"|}
+    ; {|'|}
+    ]
+
+  let escape_char =
+    '\\'
+
+  let raw_string_literals =
+    []
+
+  let comment_parser s =
+    (Parsers.Comments.c_multiline
+     <|> Parsers.Comments.c_newline) s
+end
+
+include Matcher.Make(Syntax)

lib/matchers/c.mli

@@ -0,0 +1,4 @@
+open Types
+
+module Syntax : Syntax.S
+include Matcher.S

lib/matchers/configuration.ml

@@ -0,0 +1,13 @@
+type match_kind =
+  | Exact
+  | Fuzzy
+
+type t =
+  { match_kind : match_kind
+  ; significant_whitespace: bool
+  }
+
+let create ?(match_kind = Fuzzy) ?(significant_whitespace = false) () =
+  { match_kind
+  ; significant_whitespace
+  }

lib/matchers/dune

@@ -0,0 +1,5 @@
+(library
+  (name matchers)
+  (public_name comby.matchers)
+  (preprocess (pps ppx_deriving.show ppx_sexp_conv ppx_sexp_message))
+  (libraries comby.parsers comby.match ppxlib core mparser mparser.pcre))

lib/matchers/generic.ml

@@ -0,0 +1,20 @@
+module Syntax = struct
+  (** these are nestable. strings, on the other hand, are not
+      nestable without escapes *)
+  let user_defined_delimiters =
+    [ ("(", ")")
+    ; ("{", "}")
+    ; ("[", "]")
+    ]
+
+  let escapable_string_literals = []
+
+  let escape_char =
+    '\\'
+
+  let raw_string_literals = []
+
+  let comment_parser = MParser.zero
+end
+
+include Matcher.Make(Syntax)

lib/matchers/generic.mli

@@ -0,0 +1,4 @@
+open Types
+
+module Syntax : Syntax.S
+include Matcher.S

lib/matchers/go.ml

@@ -0,0 +1,10 @@
+module Syntax = struct
+  include C.Syntax
+
+  let raw_string_literals =
+    [ ({|`|}, {|`|})
+    ]
+
+end
+
+include Matcher.Make(Syntax)

lib/matchers/go.mli

@@ -0,0 +1,4 @@
+open Types
+
+module Syntax : Syntax.S
+include Matcher.S

lib/matchers/html.ml

@@ -0,0 +1,17 @@
+module Syntax = struct
+  include Generic.Syntax
+
+  let user_defined_delimiters =
+    Generic.Syntax.user_defined_delimiters @
+    [ ("<", ">")
+    ]
+
+
+  let escapable_string_literals =
+    [ {|"|}
+    ; {|'|}
+    ]
+
+end
+
+include Matcher.Make(Syntax)

lib/matchers/matcher.ml

@@ -0,0 +1,401 @@
+open Core
+open MParser
+
+open Configuration
+open Match
+open Range
+open Location
+open Types
+
+let configuration_ref = ref (Configuration.create ())
+
+let debug = false
+
+let f _ = return Unit
+
+let extract_matched_text source { offset = match_start; _ } { offset = match_end; _ } =
+  String.slice source match_start match_end
+
+let is_not p s =
+  if is_ok (p s) then
+    Empty_failed (unknown_error s)
+  else
+    match read_char s with
+    | Some c -> Consumed_ok (c, advance_state s 1, No_error)
+    | None -> Empty_failed (unknown_error s)
+
+type 'a literal_parser_callback = contents:string -> left_delimiter:string -> right_delimiter:string -> 'a
+type 'a nested_delimiter_callback = left_delimiter:string -> right_delimiter:string -> 'a
+
+module Make (Syntax : Syntax.S) = struct
+  let escapable_string_literal_parser (f : 'a literal_parser_callback) =
+    List.map Syntax.escapable_string_literals ~f:(fun delimiter ->
+        let module M =
+          Parsers.String_literals.Escapable.Make(struct
+            let delimiter = delimiter
+            let escape = Syntax.escape_char
+          end)
+        in
+        M.base_string_literal >>= fun contents ->
+        return (f ~contents ~left_delimiter:delimiter ~right_delimiter:delimiter))
+    |> choice
+
+
+  let raw_string_literal_parser (f : 'a literal_parser_callback) =
+    List.map Syntax.raw_string_literals ~f:(fun (left_delimiter, right_delimiter) ->
+        let module M =
+          Parsers.String_literals.Raw.Make(struct
+            let left_delimiter = left_delimiter
+            let right_delimiter = right_delimiter
+          end)
+        in
+        M.base_string_literal >>= fun contents ->
+        return (f ~contents ~left_delimiter ~right_delimiter))
+    |> choice
+
+  let escapable_literal_grammar ~right_delimiter =
+    (attempt
+       (char Syntax.escape_char
+        >> string right_delimiter
+        >>= fun s -> return (Format.sprintf "%c%s" Syntax.escape_char s))
+    )
+    <|>
+    (attempt
+       (char Syntax.escape_char
+        >> char Syntax.escape_char
+        >> return (Format.sprintf "%c%c" Syntax.escape_char Syntax.escape_char))
+    )
+    <|> (is_not (string right_delimiter) |>> String.of_char)
+
+  let raw_literal_grammar ~right_delimiter =
+    is_not (string right_delimiter) |>> String.of_char
+
+  (** a parser that understands the single hole matching is alphanum and _ *)
+  let generate_single_hole_parser () =
+    (alphanum <|> char '_') |>> String.of_char
+
+  let generate_spaces_parser () =
+    (* at least a space followed by comments and spaces *)
+    (spaces1
+     >> many Syntax.comment_parser << spaces
+     >>= fun result -> f result)
+    <|>
+    (* This case not covered by tests, may not be needed *)
+    (many1 Syntax.comment_parser << spaces >>= fun result -> f result)
+
+  let sequence_chain (plist : ('c, Match.t) parser sexp_list) : ('c, Match.t) parser =
+    List.fold plist ~init:(return Unit) ~f:(>>)
+
+  let nested_delimiters_parser (f : 'a nested_delimiter_callback) =
+    Syntax.user_defined_delimiters
+    |> List.map ~f:(fun (left_delimiter, right_delimiter) ->
+        Parsers.Delimiters.between
+          (f ~left_delimiter ~right_delimiter)
+          left_delimiter right_delimiter)
+    |> choice
+
+  (** All code can have comments interpolated *)
+  let generate_string_token_parser str : ('c, _) parser =
+    many Syntax.comment_parser
+    >> string str
+    >> many Syntax.comment_parser
+    >>= fun result -> f result
+
+  let reserved_delimiters =
+    List.concat_map Syntax.user_defined_delimiters ~f:(fun (from, until) -> [from; until])
+    |> List.append [":["; "]"] (* lazy hole *)
+    |> List.append [":[["; "]]"] (* single token hole *)
+    |> List.map ~f:string
+    |> choice
+
+  let reserved =
+    reserved_delimiters
+    <|> (space |>> Char.to_string)
+
+  let greedy_hole_parser _s =
+    string ":[" >> (many (alphanum <|> char '_') |>> String.of_char_list) << string "]"
+
+  let single_hole_parser _s =
+    string ":[[" >> (many (alphanum <|> char '_') |>> String.of_char_list) << string "]]"
+
+  let until_of_from from =
+    Syntax.user_defined_delimiters
+    |> List.find_map ~f:(fun (from', until) -> if from = from' then Some until else None)
+    |> function
+    | Some until -> until
+    | None -> assert false
+
+  let record_matches identifier p : ('c, Match.t) parser =
+    get_pos >>= fun (pre_index, pre_line, pre_column) ->
+    p >>= fun matched ->
+    get_pos >>= fun (post_index, post_line, post_column) ->
+    update_user_state
+      (fun ({ Match.environment; _ } as result) ->
+         if debug then begin
+           Format.printf "Updating user state:@.";
+           Format.printf "%s |-> %s@." identifier (String.concat matched);
+           Format.printf "ID %s: %d:%d:%d - %d:%d:%d@."
+             identifier
+             pre_index pre_line pre_column
+             post_index post_line post_column;
+         end;
+         let pre_location : Location.t =
+           Location.
+             { offset = pre_index
+             ; line = pre_line
+             ; column = pre_column
+             }
+         in
+         let post_location : Location.t =
+           Location.
+             { offset = post_index
+             ; line = post_line
+             ; column = post_column
+             }
+         in
+         let range = { match_start = pre_location; match_end = post_location } in
+         let environment = Environment.add ~range environment identifier (String.concat matched) in
+         { result with environment })
+    >>= fun () -> f matched
+
+  let generate_hole_parser ?priority_left_delimiter:left_delimiter ?priority_right_delimiter:right_delimiter =
+    let between_nested_delims p from =
+      let until = until_of_from from in
+      between (string from) (string until) p
+      |>> fun result -> (String.concat @@ [from] @ result @ [until])
+    in
+    let between_nested_delims p =
+      (match left_delimiter, right_delimiter with
+       | Some left_delimiter, Some right_delimiter -> [ (left_delimiter, right_delimiter) ]
+       | _ -> Syntax.user_defined_delimiters)
+      |> List.map ~f:fst
+      |> List.map ~f:(between_nested_delims p)
+      |> choice
+    in
+    (* applies looser delimiter constraints for matching *)
+    let reserved =
+      (match left_delimiter, right_delimiter with
+       | Some left_delimiter, Some right_delimiter -> [ (left_delimiter, right_delimiter) ]
+       | _ -> Syntax.user_defined_delimiters)
+      |> List.concat_map ~f:(fun (from, until) -> [from; until])
+      |> List.map ~f:string
+      |> choice
+    in
+    (* a parser that understands the hole matching cut off points happen at
+       delimiters *)
+    let rec nested_grammar s =
+      (Syntax.comment_parser
+       <|> escapable_string_literal_parser (fun ~contents ~left_delimiter:_ ~right_delimiter:_ -> contents)
+       <|> raw_string_literal_parser (fun ~contents ~left_delimiter:_ ~right_delimiter:_ -> contents)
+       <|> delimsx
+       <|> (is_not reserved |>> String.of_char))
+        s
+    and delimsx s = (between_nested_delims (many nested_grammar)) s
+    in
+    nested_grammar
+
+  let turn_holes_into_matchers_for_this_level ?left_delimiter ?right_delimiter p_list =
+    List.fold_right p_list ~init:[] ~f:(fun p acc ->
+        let process_hole =
+          match parse_string p "_signal_hole" (Match.create ()) with
+          | Failed _ -> p
+          | Success result ->
+            match result with
+            | Hole (Lazy (identifier, dimension)) ->
+              let matcher =
+                match dimension with
+                | Code ->
+                  generate_hole_parser
+                    ?priority_left_delimiter:left_delimiter
+                    ?priority_right_delimiter:right_delimiter
+                | Escapable_string_literal ->
+                  let right_delimiter = Option.value_exn right_delimiter in
+                  escapable_literal_grammar ~right_delimiter
+                | Raw_string_literal ->
+                  let right_delimiter = Option.value_exn right_delimiter in
+                  raw_literal_grammar ~right_delimiter
+                | Comment -> failwith "Unimplemented"
+              in
+              let rest =
+                match acc with
+                | [] -> eof >>= fun () -> f [""]
+                | _ -> sequence_chain acc
+              in
+              (* continue until rest, but don't consume rest. *)
+              let hole_semantics = many (not_followed_by rest "" >> matcher) in
+              record_matches identifier hole_semantics
+
+            | Hole (Single (identifier, _)) ->
+              let hole_semantics = many1 (generate_single_hole_parser ()) in
+              record_matches identifier hole_semantics
+
+            | _ -> failwith "Hole expected"
+        in
+        process_hole::acc)
+
+  let hole_parser sort dimension =
+    let skip_signal result =
+      skip (string "_signal_hole") |>> fun () -> result
+    in
+    match sort with
+    | `Single ->
+      single_hole_parser () |>> fun id ->
+      skip_signal (Hole (Single (id, dimension)))
+    | `Lazy ->
+      greedy_hole_parser () |>> fun id ->
+      skip_signal (Hole (Lazy (id, dimension)))
+
+  let generate_hole_for_literal sort ~contents ~left_delimiter ~right_delimiter s =
+    let p =
+      many (hole_parser `Single sort
+            <|> (hole_parser `Lazy sort)
+            <|> ((many1 (is_not (string ":[" <|> string ":[["))
+                  |>> String.of_char_list) |>> generate_string_token_parser))
+    in
+    match parse_string p contents "" with
+    | Success p ->
+      begin
+        match sort with
+        | Escapable_string_literal
+        | Raw_string_literal ->
+          (turn_holes_into_matchers_for_this_level ~left_delimiter ~right_delimiter p
+           |> sequence_chain) s
+        | _ -> assert false
+      end
+    | Failed (_msg, _) ->
+      failwith "literal parser did not succeed"
+
+  let rec generate_parsers s =
+    many (common s)
+
+  and common _s =
+    (hole_parser `Single Code)
+    <|> (hole_parser `Lazy Code)
+    (* string literals are handled specially because match semantics change inside string delimiters *)
+    <|> (escapable_string_literal_parser (generate_hole_for_literal Escapable_string_literal))
+    <|> (raw_string_literal_parser (generate_hole_for_literal Raw_string_literal))
+    (* whitespace is handled specially because we may change whether they are significant for matching *)
+    <|> (spaces1 |>> generate_spaces_parser)
+    (* nested delimiters are handled specially for nestedness *)
+    <|> (nested_delimiters_parser generate_outer_delimiter_parsers)
+    (* everything else *)
+    <|> ((many1 (is_not reserved) |>> String.of_char_list) |>> generate_string_token_parser)
+
+  and generate_outer_delimiter_parsers ~left_delimiter ~right_delimiter s =
+    (generate_parsers s >>= fun p_list ->
+     (turn_holes_into_matchers_for_this_level ~left_delimiter ~right_delimiter
+        ([ string left_delimiter
+           >>= fun _ -> f [left_delimiter]]
+         @ p_list
+         @ [ string right_delimiter
+             >>= fun _ -> f [right_delimiter]])
+      |> sequence_chain)
+     |> return
+    ) s
+
+  let general_parser_generator s =
+    let outer_p =
+      generate_parsers s >>= fun p_list ->
+      (* eof of template is here *)
+      eof >> (* result is unit so ignore *)
+      (* customize the inner parser *)
+      let inner_p =
+        let matcher : ('a, Match.t) parser =
+          turn_holes_into_matchers_for_this_level p_list
+          |> sequence_chain
+        in
+        let matcher : ('a, Match.t) parser =
+          let with_positions (matcher : ('a, Match.t) parser) : ('a, Match.t) parser =
+            get_pos >>= fun (pre_offset, pre_line, pre_column) ->
+            matcher >>= fun _last_production ->
+            get_pos >>= fun (post_offset, post_line, post_column) ->
+            let match_start =
+              { offset = pre_offset
+              ; line = pre_line
+              ; column = pre_column
+              } in
+            let match_end =
+              { offset = post_offset
+              ; line = post_line
+              ; column = post_column
+              }
+            in
+            let range = { match_start; match_end } in
+            update_user_state (fun result -> { result with range })
+            >> return Unit
+          in
+          with_positions matcher
+        in
+        match !configuration_ref.match_kind with
+        | Exact -> matcher << eof
+        | Fuzzy ->
+          many
+            (not_followed_by matcher "" >>
+             (
+               (* respect grammar but ignore contents up to a match *)
+               skip Syntax.comment_parser
+               <|> skip (escapable_string_literal_parser (fun ~contents:_ ~left_delimiter:_ ~right_delimiter:_ -> ()))
+               <|> skip (raw_string_literal_parser (fun ~contents:_ ~left_delimiter:_ ~right_delimiter:_ -> ()))
+               <|> skip any_char)
+            )
+          >> matcher
+      in
+      return inner_p
+    in
+    outer_p s
+
+  let to_template template : ('a, Match.t) MParser.t Or_error.t =
+    match parse_string general_parser_generator template 0 with
+    | Success p -> Ok p
+    | Failed (msg, _) -> Or_error.error_string msg
+
+  (** shift: start the scan in the source at an offset *)
+  let first' shift p source : Match.t Or_error.t =
+    let set_start_pos p = fun s -> p (advance_state s shift) in
+    let p = set_start_pos p in
+    match parse_string' p source (Match.create ()) with
+    | Success (_, result) -> Ok result
+    | Failed (msg, _) -> Or_error.error_string msg
+
+  let first ?configuration ?shift template source =
+    let open Or_error in
+    configuration_ref := Option.value configuration ~default:!configuration_ref;
+    to_template template >>= fun p ->
+    let shift =
+      match shift with
+      | Some s -> s
+      | None -> 0
+    in
+    first' shift p source
+
+  let all ?configuration ~template ~source:original_source : Match.t list =
+    let open Or_error in
+    configuration_ref := Option.value configuration ~default:!configuration_ref;
+    let make_result = function
+      | Ok ok -> ok
+      | Error _ -> []
+    in
+    make_result @@ begin
+      to_template template >>= fun p ->
+      if original_source = "" || template = "" then
+        return []
+      else
+        let rec aux acc shift =
+          match first' shift p original_source with
+          | Ok ({range = { match_start; match_end; _ }; _} as result) ->
+            let shift = match_end.offset in
+            let matched = extract_matched_text original_source match_start match_end in
+            let result = { result with matched } in
+            if shift >= String.length original_source then
+              result :: acc
+            else
+              aux (result :: acc) shift
+          | Error _ -> acc
+        in
+        let matches = aux [] 0 |> List.rev in
+        (* TODO(RVT): reintroduce nested matches *)
+        let compute_nested_matches matches = matches in
+        let matches = compute_nested_matches matches in
+        return matches
+    end
+end

lib/matchers/matcher.mli

@@ -0,0 +1,3 @@
+open Types
+
+module Make (Syntax: Syntax.S) : Matcher.S

lib/matchers/matchers.ml

@@ -0,0 +1,10 @@
+module Generic = Generic
+module C = C
+module Go = Go
+module Python = Python
+module Bash = Bash
+module Html = Html
+
+module Configuration = Configuration
+
+module type Matcher = Types.Matcher.S

lib/matchers/matchers.mli

@@ -0,0 +1,10 @@
+module Generic = Generic
+module C = C
+module Go = Go
+module Python = Python
+module Bash = Bash
+module Html = Html
+
+module Configuration = Configuration
+
+module type Matcher = Types.Matcher.S

lib/matchers/python.ml

@@ -0,0 +1,20 @@
+module Python = struct
+  include Generic.Syntax
+
+  let escapable_string_literals =
+    [ {|"|}
+    ; {|'|}
+    ]
+
+  let escape_char =
+    '\\'
+
+  let raw_string_literals =
+    [ ({|"""|}, {|"""|})
+    ]
+
+  let comment_parser s =
+    Parsers.Comments.python_newline s
+end
+
+include Matcher.Make(Python)

lib/matchers/types.ml

@@ -0,0 +1,44 @@
+open Core
+
+module Syntax = struct
+  module type S = sig
+    val user_defined_delimiters : (string * string) list
+    val escapable_string_literals : string list
+    val escape_char : char
+    val raw_string_literals : (string * string) list
+    val comment_parser : (string, _) MParser.t
+  end
+end
+
+type dimension =
+  | Code
+  | Escapable_string_literal
+  | Raw_string_literal
+  | Comment
+
+type hole =
+  | Lazy of (string * dimension)
+  | Single of (string * dimension)
+
+type production =
+  | Unit
+  | String of string
+  | Hole of hole
+  | Match of (int * string * string)
+
+module Matcher = struct
+  module type S = sig
+    val first
+      :  ?configuration:Configuration.t
+      -> ?shift:int
+      -> string
+      -> string
+      -> Match.t Or_error.t
+
+    val all
+      :  ?configuration:Configuration.t
+      -> template:string
+      -> source:string
+      -> Match.t list
+  end
+end

lib/parsers/comments.ml

@@ -0,0 +1,69 @@
+open Core
+open MParser
+
+let to_string from until between : string =
+  from ^ (String.of_char_list between) ^ until
+
+let anything_including_newlines ~until =
+  (many
+     (not_followed_by (string until) ""
+      >>= fun () -> any_char_or_nl))
+
+let anything_excluding_newlines ~until =
+  (many
+     (not_followed_by (string until) ""
+      >>= fun () -> any_char))
+
+(** a parser for comments with delimiters [from] and [until] that do not nest *)
+let non_nested_comment_delimiters from until s =
+  (between
+     (string from)
+     (string until)
+     (anything_including_newlines ~until)
+   |>> to_string from until
+  ) s
+
+(** a parser for /* ... */ style block comments. *)
+let c_multiline s =
+  non_nested_comment_delimiters "/*" "*/" s
+
+let c_newline s =
+  (string "//" >> anything_excluding_newlines ~until:"\n"
+   |>> fun l -> "//"^(String.of_char_list l)) s
+
+let python_newline s =
+  (string "#" >> anything_excluding_newlines ~until:"\n"
+   |>> fun l -> ("#"^String.of_char_list l)) s
+
+let any_newline comment_string s =
+  (string comment_string >> anything_excluding_newlines ~until:"\n" |>> fun l -> (comment_string^String.of_char_list l)) s
+
+let is_not p s =
+  if is_ok (p s) then
+    Empty_failed (unknown_error s)
+  else
+    match read_char s with
+    | Some c ->
+      Consumed_ok (c, advance_state s 1, No_error)
+    | None ->
+      Empty_failed (unknown_error s)
+
+(** A nested comment parser *)
+let skip_nested_comments_inner from until s =
+  let reserved = skip ((string from) <|> (string until)) in
+  let rec grammar s =
+    ((comment_delimiters >>= fun string -> return string)
+     <|>
+     (is_not reserved >>= fun c -> return (Char.to_string c)))
+      s
+
+  and comment_delimiters s =
+    (between
+       (string from)
+       (string until)
+       ((many grammar) >>= fun result ->
+        return (String.concat result)))
+      s
+  in
+  (comment_delimiters >>= fun _ ->
+   return ()) s

lib/parsers/comments.mli

@@ -0,0 +1,11 @@
+(** C-style /* */ block comment parser *)
+val c_multiline : (string, _) MParser.t
+
+(** C++-style // line comment parser *)
+val c_newline : (string, _) MParser.t
+
+(** Python-style # line comment parser *)
+val python_newline : (string, _) MParser.t
+
+(** Anything until newline *)
+val any_newline : string -> (string, _) MParser.t

lib/parsers/delimiters.ml

@@ -0,0 +1,18 @@
+open MParser
+
+(** Significant, potentially nested delimiters. *)
+
+let parens' p =
+  char '(' >> p << char ')'
+
+let braces' p =
+  char '{' >> p << char '}'
+
+let brackets' p =
+  char '<' >> p << char '>'
+
+let squares' p =
+  char '[' >> p << char ']'
+
+let between p from until =
+  string from >> p << string until

lib/parsers/dune

@@ -0,0 +1,5 @@
+(library
+  (name parsers)
+  (public_name comby.parsers)
+  (preprocess (pps ppx_deriving.show ppx_sexp_conv))
+  (libraries ppxlib core mparser mparser.pcre))

lib/parsers/string_literals.ml

@@ -0,0 +1,57 @@
+open Core
+open MParser
+
+(** Assumes the left and right delimiter are the same, and that these can be
+    escaped. Does not parse a string body containing newlines (as usual when
+    escaping with \n) *)
+module Escapable = struct
+  module type S = sig
+    val delimiter : string
+    val escape : char
+  end
+
+  module Make (M : S) = struct
+    (* delimiters can be escaped and parsing continues within the string body *)
+    let escaped_char_s s =
+      any_char s
+
+    let char_token_s s =
+      ((char M.escape >> escaped_char_s >>= fun c -> return (Format.sprintf {|%c%c|} M.escape c))
+       <|> (any_char |>> String.of_char)
+      )
+        s
+
+    let base_string_literal s =
+      ((string M.delimiter >> (many_until char_token_s (string M.delimiter))
+        |>> String.concat)
+       >>= fun result ->
+       return (Format.sprintf {|%s%s%s|} M.delimiter result M.delimiter)
+      )
+        s
+  end
+end
+
+(** Quoted or raw strings. Allows different left and right delimiters, and
+    disallows any sort of escaping. Does not support raw strings with identifiers
+    yet, e.g., {blah|<string body>|blah} (OCaml) or delim`<string body>`delim
+    syntax (Go) *)
+module Raw = struct
+  module type S = sig
+    val left_delimiter : string
+    val right_delimiter : string
+  end
+
+  module Make (M : S) = struct
+    let char_token_s s =
+      (any_char_or_nl |>> String.of_char) s
+
+    let base_string_literal s =
+      ((
+        string M.left_delimiter >> (many_until char_token_s (string M.right_delimiter))
+        |>> String.concat <?> "raw string literal body")
+       >>= fun result ->
+       return (Format.sprintf {|%s%s%s|} M.left_delimiter result M.right_delimiter)
+      )
+        s
+  end
+end

lib/rewriter/dune

@@ -0,0 +1,5 @@
+(library
+  (name rewriter)
+  (public_name comby.rewriter)
+  (preprocess (pps ppx_deriving.show ppx_sexp_conv ppx_deriving_yojson))
+  (libraries comby.matchers ppxlib core))

lib/rewriter/rewrite.ml

@@ -0,0 +1,112 @@
+open Core
+
+open Match
+
+type match_context_replacement =
+  { range : range
+  ; replacement_content : string
+  ; environment : environment
+  }
+[@@deriving yojson]
+
+type result =
+  { rewritten_source : string
+  ; contextual_substitutions : match_context_replacement list
+  }
+[@@deriving yojson]
+
+let empty_result =
+  { rewritten_source = ""
+  ; contextual_substitutions = []
+  }
+[@@deriving yojson]
+
+let substitute_match_contexts (matches: Match.t list) source replacements =
+  let rewrite_template, environment =
+    List.fold2_exn
+      matches replacements
+      ~init:(source, Environment.create ())
+      ~f:(fun
+           (rewrite_template, accumulator_environment)
+           ({ environment = _match_environment; _ } as match_)
+           { replacement_content; _ } ->
+           (* create a hole in the rewrite template based on this match context *)
+           let hole_id, rewrite_template = Rewrite_template.of_match_context match_ ~source:rewrite_template in
+           (* add this match context replacement to the environment *)
+           let accumulator_environment = Environment.add accumulator_environment hole_id replacement_content in
+           (* update match context replacements offset *)
+           rewrite_template, accumulator_environment)
+  in
+  let rewritten_source = Rewrite_template.substitute rewrite_template environment |> fst in
+  let offsets = Rewrite_template.get_offsets_for_holes rewrite_template (Environment.vars environment) in
+  let offsets = Rewrite_template.get_offsets_after_substitution offsets environment in
+  let contextual_substitutions =
+    List.map2_exn replacements offsets ~f:(fun replacement (_uid, offset) ->
+        let match_start = { Location.default with offset } in
+        let offset = offset + String.length replacement.replacement_content in
+        let match_end = { Location.default with offset } in
+        let range = Range.{ match_start; match_end } in
+        { replacement with range })
+  in
+  { rewritten_source
+  ; contextual_substitutions
+  }
+
+  (*
+   store range information for this match_context replacement:
+   (a) its offset in the original source
+   (b) its replacement context (to calculate the range)
+   (c) an environment of values that are updated to reflect their relative offset in the rewrite template
+   *)
+let substitute_in_rewrite_template rewrite_template ({ environment; _ } : Match.t) =
+  let replacement_content, vars_substituted_for = Rewrite_template.substitute rewrite_template environment in
+  let offsets = Rewrite_template.get_offsets_for_holes rewrite_template (Environment.vars environment) in
+  let offsets = Rewrite_template.get_offsets_after_substitution offsets environment in
+  let environment =
+    List.fold offsets ~init:(Environment.create ()) ~f:(fun acc (var, relative_offset) ->
+        if List.mem vars_substituted_for var ~equal:String.equal then
+          let value = Option.value_exn (Environment.lookup environment var) in
+          (* FIXME(RVT): Location does not update row/column here *)
+          let start_location =
+            Location.{ default with offset = relative_offset }
+          in
+          let end_location =
+            let offset = relative_offset + String.length value in
+            Location.{ default with offset }
+          in
+          let range =
+            Range.
+              { match_start = start_location
+              ; match_end = end_location
+              }
+          in
+          Environment.add ~range acc var value
+        else
+          acc)
+  in
+  { replacement_content
+  ; environment
+  ; range =
+      { match_start = { Location.default with offset = 0 }
+      ; match_end = Location.default
+      }
+  }
+
+let all ?source ~rewrite_template matches : result option =
+  if matches = [] then None else
+    match source with
+    (* in-place substitution *)
+    | Some source ->
+      let matches : Match.t list = List.rev matches in
+      matches
+      |> List.map ~f:(substitute_in_rewrite_template rewrite_template)
+      |> substitute_match_contexts matches source
+      |> Option.some
+    (* no-inplace substitution, emit result separated by newlines *)
+    | None ->
+      matches
+      |> List.map ~f:(substitute_in_rewrite_template rewrite_template)
+      |> List.map ~f:(fun { replacement_content; _ } -> replacement_content)
+      |> String.concat ~sep:"\n"
+      |> (fun rewritten_source -> { rewritten_source; contextual_substitutions = [] })
+      |> Option.some

lib/rewriter/rewrite.mli

@@ -0,0 +1,22 @@
+open Match
+
+type match_context_replacement =
+  { range : range
+  ; replacement_content : string
+  ; environment : environment
+  }
+[@@deriving yojson]
+
+type result =
+  { rewritten_source : string
+  ; contextual_substitutions : match_context_replacement list
+  }
+[@@deriving yojson]
+
+(** if [source] is given, substitute in-place. If not,
+    emit result separated by newlines *)
+val all
+  :  ?source:string
+  -> rewrite_template:string
+  -> Match.t list
+  -> result option

lib/rewriter/rewrite_template.ml

@@ -0,0 +1,63 @@
+open Core
+
+open Match
+
+let substitute template env =
+  Environment.vars env
+  |> List.fold ~init:(template, []) ~f:(fun (acc, vars) variable ->
+      match Environment.lookup env variable with
+      | Some value ->
+        if Option.is_some (String.substr_index template  ~pattern:(":["^variable^"]")) then
+          (String.substr_replace_all acc ~pattern:(":["^variable^"]") ~with_:value, variable::vars)
+        else
+          acc, vars
+      | None -> acc, vars)
+
+let of_match_context
+    { range =
+        { match_start = { offset = start_index; _ }
+        ; match_end = { offset = end_index; _ } }
+    ; _
+    }
+    ~source =
+  let before_part =
+    if start_index = 0 then
+      ""
+    else
+      String.slice source 0 start_index
+  in
+  let after_part = String.slice source end_index (String.length source) in
+  let hole_id = Uuid.(Fn.compose to_string create ()) in
+  let rewrite_template = String.concat [before_part; ":["; hole_id;  "]"; after_part] in
+  hole_id, rewrite_template
+
+(* return the offset for holes (specified by variables) in a given match template *)
+let get_offsets_for_holes rewrite_template variables =
+  let sorted_variables =
+    List.fold variables ~init:[] ~f:(fun acc variable ->
+        match String.substr_index rewrite_template ~pattern:(":["^variable^"]") with
+        | Some index ->
+          (variable, index)::acc
+        | None -> acc)
+    |> List.sort ~compare:(fun (_, i1) (_, i2) -> i1 - i2)
+    |> List.map ~f:fst
+  in
+  List.fold sorted_variables ~init:(rewrite_template, []) ~f:(fun (rewrite_template, acc) variable ->
+      match String.substr_index rewrite_template ~pattern:(":["^variable^"]") with
+      | Some index ->
+        let rewrite_template =
+          String.substr_replace_all rewrite_template ~pattern:(":["^variable^"]") ~with_:"" in
+        rewrite_template, (variable, index)::acc
+      | None -> rewrite_template, acc)
+  |> snd
+
+(* pretend we substituted vars in offsets with environment. return what the offsets are after *)
+let get_offsets_after_substitution offsets environment =
+  List.fold_right offsets ~init:([],0) ~f:(fun (var, offset) (acc, shift)  ->
+      match Environment.lookup environment var with
+      | None -> failwith "Expected var"
+      | Some s ->
+        let offset' = offset + shift in
+        let shift = shift + String.length s in
+        ((var, offset')::acc), shift)
+  |> fst

lib/rewriter/rewrite_template.mli

@@ -0,0 +1,10 @@
+open Match
+
+(** substitute returns the result and variables substituted for *)
+val substitute : string -> Environment.t -> (string * string list)
+
+val of_match_context : Match.t -> source:string -> (string * string)
+
+val get_offsets_for_holes : string -> string list -> (string * int) list
+
+val get_offsets_after_substitution : (string * int) list -> Environment.t -> (string * int) list

lib/statistics/dune

@@ -0,0 +1,5 @@
+(library
+  (name statistics)
+  (public_name comby.statistics)
+  (preprocess (pps ppx_deriving.show ppx_sexp_conv ppx_sexp_message ppx_deriving_yojson))
+  (libraries ppxlib core yojson ppx_deriving_yojson ppx_deriving_yojson.runtime))

lib/statistics/statistics.ml

@@ -0,0 +1,34 @@
+module Time = Time
+module Timer = Timer
+
+type t =
+  { number_of_files : int
+  ; lines_of_code : int
+  ; number_of_matches : int
+  ; total_time : float
+  }
+[@@deriving yojson]
+
+let empty =
+  { number_of_files = 0
+  ; lines_of_code = 0
+  ; number_of_matches = 0
+  ; total_time = 0.0
+  }
+
+let merge
+    { number_of_files
+    ; lines_of_code
+    ; number_of_matches
+    ; total_time
+    }
+    { number_of_files = number_of_files'
+    ; lines_of_code = lines_of_code'
+    ; number_of_matches = number_of_matches'
+    ; total_time = total_time'
+    } =
+  { number_of_files = number_of_files + number_of_files'
+  ; lines_of_code = lines_of_code + lines_of_code'
+  ; number_of_matches = number_of_matches + number_of_matches'
+  ; total_time = total_time +. total_time'
+  }

lib/statistics/statistics.mli

@@ -0,0 +1,14 @@
+module Time = Time
+module Timer = Timer
+
+type t =
+  { number_of_files : int
+  ; lines_of_code : int
+  ; number_of_matches : int
+  ; total_time : float
+  }
+[@@deriving yojson]
+
+val empty : t
+
+val merge : t -> t -> t

lib/statistics/time.ml

@@ -0,0 +1,25 @@
+open Core
+
+let start () = Unix.gettimeofday ()
+
+let stop start =
+  (Unix.gettimeofday () -. start) *. 1000.0
+
+exception Time_out
+
+let time_out ~after f args =
+  let behavior =
+    Signal.(Expert.signal alrm (`Handle (fun _ -> raise Time_out)))
+  in
+  let cancel_alarm () =
+    Unix.alarm 0 |> ignore;
+    Signal.(Expert.set alrm behavior)
+  in
+  Unix.alarm after |> ignore;
+  match f args with
+  | result ->
+    cancel_alarm ();
+    result
+  | exception exc ->
+    cancel_alarm ();
+    raise exc

src/dune

@@ -0,0 +1,12 @@
+(executables
+ (libraries comby core ppx_deriving_yojson ppx_deriving_yojson.runtime hack_parallel)
+ (preprocess (pps ppx_deriving_yojson ppx_let ppx_deriving.show))
+ (names main))
+
+(alias
+ (name DEFAULT)
+ (deps main.exe))
+
+(install
+ (section bin)
+ (files (main.exe as comby)))

src/main.ml

@@ -0,0 +1,385 @@
+open Core
+open Command.Let_syntax
+
+open Hack_parallel
+
+open Matchers
+open Match
+open Language
+open Rewriter
+open Statistics
+
+type json_result =
+  { matches : Match.t list
+  ; source : string
+  }
+[@@deriving yojson]
+
+type input_kind =
+  | Paths of string list
+  | Path of string
+  | String of string
+[@@deriving show]
+
+type processed_source_result =
+  | Matches of (Match.t list * int)
+  | Rewritten of (Rewrite.match_context_replacement list * string * int)
+  | Nothing
+
+let read = Fn.compose String.rstrip In_channel.read_all
+
+let read_template =
+  Fn.compose
+    String.chop_suffix_exn ~suffix:"\n"
+    In_channel.read_all
+
+let verbose_out_file = "/tmp/comby.out"
+
+let get_matches (module Matcher : Matchers.Matcher) configuration match_template match_rule source =
+  let rule = Rule.create match_rule |> Or_error.ok_exn in
+  Matcher.all ~configuration ~template:match_template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule ~matcher:(module Matcher) environment))
+
+let apply_rewrite_rule matcher rewrite_rule matches =
+  let open Option in
+  match rewrite_rule with
+  | "" -> matches
+  | rewrite_rule ->
+    begin
+      match Rule.create rewrite_rule with
+      | Ok rule ->
+        List.filter_map matches ~f:(fun ({ environment; _ } as match_) ->
+            let sat, env = Rule.apply rule ~matcher environment in
+            (if sat then env else None)
+            >>| fun environment -> { match_ with environment })
+      | Error _ -> []
+    end
+
+let rewrite rewrite_template _rewrite_rule source matches =
+  Rewrite.all ~source ~rewrite_template matches
+
+let process_single_source matcher verbose configuration source specification match_timeout =
+  let open Specification in
+  try
+    let input_text =
+      match source with
+      | String input_text -> input_text
+      | Path path ->
+        if verbose then
+          Out_channel.with_file ~append:true verbose_out_file ~f:(fun out_channel ->
+              Out_channel.output_lines out_channel [Format.sprintf "Processing %s%!" path]);
+        In_channel.read_all path
+      | _ -> failwith "Don't send multiple paths to process_single_source"
+    in
+    match specification with
+    | { match_specification = { match_template; match_rule }
+      ; rewrite_specification = None
+      } ->
+      let matches =
+        try
+          let f () = get_matches matcher configuration match_template match_rule input_text in
+          Statistics.Time.time_out ~after:match_timeout f ();
+        with Statistics.Time.Time_out ->
+          Out_channel.with_file ~append:true verbose_out_file ~f:(fun out_channel ->
+              Out_channel.output_lines out_channel [Format.sprintf "TIMEOUT: %s@." (show_input_kind source) ]);
+          []
+      in
+      Matches (matches, List.length matches)
+    | { match_specification = { match_template; match_rule }
+      ; rewrite_specification = Some { rewrite_template; rewrite_rule }
+      } ->
+      let result =
+        try
+          let f () =
+            get_matches matcher configuration match_template match_rule input_text
+            |> fun matches ->
+            (* TODO(RVT): merge match and rewrite rule application. *)
+            apply_rewrite_rule matcher rewrite_rule matches
+            |> fun matches ->
+            if matches = [] then
+              (* If there are no matches, return the original source (for editor support). *)
+              Some (Some (Rewrite.{ rewritten_source = input_text; contextual_substitutions = [] }), [])
+            else
+              Some (rewrite rewrite_template rewrite_rule input_text matches, matches)
+          in
+          Statistics.Time.time_out ~after:match_timeout f ();
+        with Statistics.Time.Time_out ->
+          Out_channel.with_file ~append:true verbose_out_file ~f:(fun out_channel ->
+              Out_channel.output_lines out_channel [Format.sprintf "TIMEOUT: FOR %s@." (show_input_kind source) ]);
+          None
+      in
+      result
+      |> function
+      | Some (Some { rewritten_source; contextual_substitutions }, matches) ->
+        Rewritten (contextual_substitutions, rewritten_source, List.length matches)
+      | Some (None, _)
+      | None -> Nothing
+  with
+  | _ -> Nothing
+
+let output_result stdin spec_number json source_path result =
+  match result with
+  | Nothing -> ()
+  | Matches (matches, _) ->
+    if json then
+      let json_matches = `List (List.map ~f:Match.to_yojson matches) in
+      Format.printf "%s%!" @@ Yojson.Safe.pretty_to_string json_matches
+    else
+      let with_file =
+        match source_path with
+        | Some path -> Format.sprintf " in %s " path
+        | None -> " "
+      in
+      Format.printf
+        "%d matches%sfor spec %d (add -json for json format)@."
+        (List.length matches)
+        with_file
+        (spec_number + 1)
+  | Rewritten (replacements, result, _) ->
+    match source_path, json, stdin with
+    (* default: rewrite in place *)
+    | Some path, false, false -> Out_channel.write_all path ~data:result
+    (* stdin, not JSON *)
+    | _, false, true -> Format.printf "%s%!" result
+    (* stdin, JSON with path *)
+    | Some path, true, _ ->
+      let json_rewrites =
+        let value = `List (List.map ~f:Rewrite.match_context_replacement_to_yojson replacements) in
+        `Assoc [(path, value)]
+      in
+      Format.printf "%s%!" @@ Yojson.Safe.pretty_to_string json_rewrites
+    (* JSON, no path *)
+    | None, true, _ ->
+      let json_rewrites =
+        `List (List.map ~f:Rewrite.match_context_replacement_to_yojson replacements) in
+      Format.printf "%s%!" @@ Yojson.Safe.pretty_to_string json_rewrites
+    | _ -> Format.printf "%s%!" result
+
+let write_statistics number_of_matches paths total_time =
+  let total_time = Statistics.Time.stop total_time in
+  let lines_of_code =
+    List.fold paths ~init:0 ~f:(fun acc paths ->
+        In_channel.read_lines paths
+        |> List.length
+        |> (+) acc)
+  in
+  let statistics =
+    { number_of_files = List.length paths
+    ; lines_of_code
+    ; number_of_matches
+    ; total_time = total_time
+    }
+  in
+  Format.eprintf "%s%!"
+  @@ Yojson.Safe.pretty_to_string
+  @@ Statistics.to_yojson statistics
+
+let paths_with_file_size paths =
+  List.map paths ~f:(fun path ->
+      let length =
+        In_channel.create path
+        |> fun channel ->
+        In_channel.length channel
+        |> Int64.to_int
+        |> (fun value -> Option.value_exn value)
+        |> (fun value -> In_channel.close channel; value)
+      in
+      (path, length))
+
+let run
+    matcher
+    (sources : input_kind)
+    (specifications : Specification.t list)
+    sequential
+    number_of_workers
+    stdin
+    json
+    verbose
+    match_timeout =
+  let number_of_workers = if sequential then 0 else number_of_workers in
+  let scheduler = Scheduler.create ~number_of_workers () in
+  let configuration = Configuration.create ~match_kind:Fuzzy () in
+  let total_time = Statistics.Time.start () in
+
+  let run_on_specifications input output_file =
+    let result, count =
+      List.fold specifications ~init:(Nothing,0) ~f:(fun (result, count) specification ->
+          let input =
+            match result with
+            | Nothing | Matches _ -> input
+            | Rewritten (_, content, _) -> String content
+          in
+          process_single_source matcher verbose configuration input specification match_timeout
+          |> function
+          | Nothing -> Nothing, count
+          | Matches (x, number_of_matches) ->
+            Matches (x, number_of_matches), count + number_of_matches
+          | Rewritten (x, content, number_of_matches) ->
+            Rewritten (x, content, number_of_matches),
+            count + number_of_matches)
+    in
+    output_result stdin 0 json output_file result;
+    count
+  in
+
+  match sources with
+  | String source ->
+    let number_of_matches = run_on_specifications (String source) None in
+    (* FIXME(RVT): statistics for single source text doesn't output LOC *)
+    write_statistics number_of_matches [] total_time
+  | Paths paths ->
+    if sequential then
+      let number_of_matches =
+        List.fold ~init:0 paths ~f:(fun acc path ->
+            let matches = run_on_specifications (Path path) (Some path) in
+            acc + matches)
+      in
+      write_statistics number_of_matches paths total_time
+    else
+      let map init paths =
+        List.fold
+          paths
+          ~init
+          ~f:(fun count path -> count + run_on_specifications (Path path) (Some path))
+      in
+      let number_of_matches =
+        try Scheduler.map_reduce scheduler ~init:0 ~map ~reduce:(+) paths
+        with End_of_file -> 0
+      in
+      begin
+        try Scheduler.destroy scheduler
+        with Unix.Unix_error (_,"kill",_) ->
+          (* No kill command on Mac OS X *)
+          ()
+      end;
+      write_statistics number_of_matches paths total_time
+  | _ -> failwith "No single path handled here"
+
+let parse_source_directories ?(file_extensions = []) target_directory =
+  let rec ls_rec path =
+    if Sys.is_file path = `Yes then
+      match file_extensions with
+      | [] -> [path]
+      | suffixes when List.exists suffixes ~f:(fun suffix -> String.is_suffix ~suffix path) ->
+        [path]
+      | _ -> []
+    else
+      try
+        Sys.ls_dir path
+        |> List.map ~f:(fun sub -> ls_rec (Filename.concat path sub))
+        |> List.concat
+      with
+      | _ -> []
+  in
+  ls_rec target_directory
+
+let parse_specification_directories match_only specification_directory_paths =
+  let parse_directory path =
+    let match_template =
+      let filename = path ^/ "match" in
+      try read_template filename
+      with _ -> failwith (Format.sprintf "Could not read required match file %s" filename)
+    in
+    let match_rule =
+      let filename = path ^/ "match_rule" in
+      try Some (read filename)
+      with _ -> None
+    in
+    let rewrite_template =
+      let filename = path ^/ "rewrite" in
+      if match_only then
+        None
+      else
+        try Some (read_template filename)
+        with _ -> None
+    in
+    let rewrite_rule =
+      let filename = path ^/ "rewrite_rule" in
+      if match_only then
+        None
+      else
+        try Some (read filename)
+        with _ -> None
+    in
+    Specification.create ~match_template ?match_rule ?rewrite_template ?rewrite_rule ()
+  in
+  List.map specification_directory_paths ~f:parse_directory
+
+let base_command_parameters : (unit -> 'result) Command.Param.t =
+  [%map_open
+     (* flags. *)
+    let sequential = flag "sequential" no_arg ~doc:"Run sequentially"
+    and match_only = flag "match-only" no_arg ~doc:"Only perform matching (ignore rewrite templates)"
+    and verbose = flag "verbose" no_arg ~doc:(Format.sprintf "Log to %s" verbose_out_file)
+    and rule = flag "rule" (optional_with_default "where true" string) ~doc:"rule Apply rules to matches. Respects -f"
+    and match_timeout = flag "timeout" (optional_with_default 3 int) ~doc:"seconds Set match timeout on a source. Default: 3"
+    and target_directory = flag "directory" (optional_with_default "." string) ~doc:(Format.sprintf "path Run on files in a directory. Default is current directory: %s" @@ Sys.getcwd ())
+    and specification_directories = flag "templates" (optional (Arg_type.comma_separated string)) ~doc:"path CSV of directories containing templates"
+    and file_extensions = flag "filter" (optional (Arg_type.comma_separated string)) ~doc:"extensions CSV of extensions to include"
+    and json = flag "json" no_arg ~doc:"Output JSON format for matches or rewrite text to stdout"
+    and number_of_workers = flag "jobs" (optional_with_default 4 int) ~doc:"n Number of worker processes. Default: 4"
+    and stdin = flag "stdin" no_arg ~doc:"Read source from stdin"
+    and anonymous_arguments =
+      anon (maybe (t2
+                     ("MATCH_TEMPLATE" %: string)
+                     ("REWRITE_TEMPLATE" %: string)))
+    in
+    fun () ->
+      let () =
+        match Rule.create rule with
+        | Ok _ -> ()
+        | Error error ->
+          let message = Error.to_string_hum error in
+          Format.printf "Match rule parse error: %s@." message;
+          exit 1
+      in
+      let specifications =
+        match specification_directories, anonymous_arguments with
+        | None, None
+        | Some [], None ->
+          Format.eprintf
+            "Please either specify templates on the command line or using \
+             -templates [dir] for templates in directory [dir].@.";
+          exit 1
+        | None, Some (match_template, rewrite_template) ->
+          if match_only then
+            [Specification.create ~match_template ~match_rule:rule ()]
+          else
+            [Specification.create ~match_template ~rewrite_template ~match_rule:rule ~rewrite_rule:rule ()]
+        | Some specification_directories, None ->
+          parse_specification_directories match_only specification_directories
+        | Some specification_directories, Some _ ->
+          Format.eprintf
+            "Warning: ignoring match and rewrite templates and rules on \
+             commandline and using those in directories instead@.";
+          parse_specification_directories match_only specification_directories
+      in
+      let sources =
+        match stdin with
+        | false -> Paths (parse_source_directories ?file_extensions target_directory)
+        | true -> String (In_channel.input_all In_channel.stdin)
+      in
+
+      let matcher =
+        let default = (module Matchers.C : Matchers.Matcher) in
+        match file_extensions with
+        | None | Some [] -> default
+        | Some (hd::_) ->
+          match hd with
+          | ".c" | ".h" | ".cc" | ".cpp" | ".hpp" -> (module Matchers.C : Matchers.Matcher)
+          | ".py" -> (module Matchers.Python : Matchers.Matcher)
+          | ".go" -> (module Matchers.Go : Matchers.Matcher)
+          | ".sh" -> (module Matchers.Bash : Matchers.Matcher)
+          | ".html" -> (module Matchers.Html : Matchers.Matcher)
+          | _ -> default
+      in
+      run matcher sources specifications sequential number_of_workers stdin json verbose match_timeout
+  ]
+
+let default_command =
+  Command.basic ~summary:"Run a rewrite pass." base_command_parameters
+
+let () =
+  Scheduler.Daemon.check_entry_point ();
+  default_command
+  |> Command.run

src/specification.ml

@@ -0,0 +1,32 @@
+open Core
+
+type match_specification =
+  { match_template : string
+  ; match_rule : string
+  }
+[@@deriving show]
+
+type rewrite_specification =
+  { rewrite_template : string
+  ; rewrite_rule : string
+  }
+[@@deriving show]
+
+type t =
+  { match_specification : match_specification
+  ; rewrite_specification : rewrite_specification option
+  }
+[@@deriving show]
+
+let create
+    ?rewrite_template
+    ?(match_rule = "where true")
+    ?(rewrite_rule = "")
+    ~match_template
+    () =
+  let match_specification = { match_template; match_rule } in
+  let rewrite_specification =
+    Option.map rewrite_template ~f:(fun rewrite_template ->
+        { rewrite_template; rewrite_rule})
+  in
+  { match_specification; rewrite_specification }

test/dune

@@ -0,0 +1,21 @@
+(library
+ (name test_integration)
+ (modules
+  test_match_rule
+  test_integration
+  test_statistics
+  test_c
+  test_cli
+  test_bash
+  test_go
+  test_c_style_comments
+  test_c_separators
+  test_string_literals
+  test_generic
+  test_rewrite_parts
+  test_rewrite_rule)
+ (inline_tests)
+ (preprocess (pps ppx_expect ppx_sexp_message))
+ (libraries
+  comby
+  core))

test/test_bash.ml

@@ -0,0 +1,77 @@
+open Core
+
+open Matchers
+open Rewriter
+
+let configuration = Configuration.create ~match_kind:Fuzzy ()
+
+let run_bash source match_template rewrite_template =
+  Bash.first ~configuration match_template source
+  |> function
+  | Ok result ->
+    Rewrite.all ~source ~rewrite_template [result]
+    |> (fun x -> Option.value_exn x)
+    |> (fun { rewritten_source; _ } -> rewritten_source)
+    |> print_string
+  | Error _ ->
+    print_string rewrite_template
+
+let run_go source match_template rewrite_template =
+  Go.first ~configuration match_template source
+  |> function
+  | Ok result ->
+    Rewrite.all ~source ~rewrite_template [result]
+    |> (fun x -> Option.value_exn x)
+    |> (fun { rewritten_source; _ } -> rewritten_source)
+    |> print_string
+  | Error _ ->
+    print_string rewrite_template
+
+let%expect_test "custom_long_delimiters" =
+  let source =
+    {|
+      case
+        case
+          block 1
+        esac
+
+        case
+          block 2
+        esac
+      esac
+    |}
+  in
+  let match_template = {|case :[1] esac|} in
+  let rewrite_template = {|case nuked blocks esac|} in
+
+  run_bash source match_template rewrite_template;
+  [%expect_exact {|
+      case nuked blocks esac
+    |}]
+
+let%expect_test "custom_long_delimiters_doesn't_work_in_go" =
+  let source =
+    {|
+      case
+        case
+          block 1
+        esac
+
+        case
+          block 2
+        esac
+      esac
+    |}
+  in
+  let match_template = {|case :[1] esac|} in
+  let rewrite_template = {|case nuked blocks esac|} in
+
+  run_go source match_template rewrite_template;
+  [%expect_exact {|
+      case nuked blocks esac
+
+        case
+          block 2
+        esac
+      esac
+    |}]

test/test_c.ml

@@ -0,0 +1,105 @@
+open Core
+
+open Matchers
+open Rewriter
+
+let configuration = Configuration.create ~match_kind:Fuzzy ()
+
+let run source match_template rewrite_template =
+  C.first ~configuration match_template source
+  |> function
+  | Ok result ->
+    Rewrite.all ~source ~rewrite_template [result]
+    |> (fun x -> Option.value_exn x)
+    |> (fun { rewritten_source; _ } -> rewritten_source)
+    |> print_string
+  | Error _ ->
+    print_string rewrite_template
+
+let%expect_test "comments_1" =
+  let source = {|match this /**/ expect end|} in
+  let match_template = {|match this :[1] end|} in
+  let rewrite_template = {|:[1]|} in
+
+  run source match_template rewrite_template;
+  [%expect_exact {|expect|}]
+
+let%expect_test "comments_2" =
+  let source = {|match this /* */ expect end|} in
+  let match_template = {|match this :[1] end|} in
+  let rewrite_template = {|:[1]|} in
+
+  run source match_template rewrite_template;
+  [%expect_exact {|expect|}]
+
+let%expect_test "comments_3" =
+  let source = {|match this /* blah blah */ expect /**/ end|} in
+  let match_template = {|match this :[1] end|} in
+  let rewrite_template = {|:[1]|} in
+
+  run source match_template rewrite_template;
+  [%expect_exact {|expect|}]
+
+let%expect_test "comments_4" =
+  let source = {|match this expect/**/end|} in
+  let match_template = {|match this :[1]end|} in
+  let rewrite_template = {|:[1]|} in
+
+  run source match_template rewrite_template;
+  [%expect_exact {|expect|}]
+
+let%expect_test "comments_5" =
+  let source = {|match this expect /**/end|} in
+  let match_template = {|match this :[1] end|} in
+  let rewrite_template = {|:[1]|} in
+
+  run source match_template rewrite_template;
+  [%expect_exact {|expect|}]
+
+let%expect_test "comments_6" =
+  let source = {|/* don't match this (a) end */|} in
+  let match_template = {|match this :[1] end|} in
+  let rewrite_template = {|nothing matches|} in
+
+  run source match_template rewrite_template;
+  [%expect_exact {|nothing matches|}]
+
+let%expect_test "comments_7" =
+  let source = {|/* don't match /**/ this (a) end */|} in
+  let match_template = {|match this :[1] end|} in
+  let rewrite_template = {|nothing matches|} in
+
+  run source match_template rewrite_template;
+  [%expect_exact {|nothing matches|}]
+
+let%expect_test "comments_8" =
+  let source = {|(/* don't match this (a) end */)|} in
+  let match_template = {|match this :[1] end|} in
+  let rewrite_template = {|nothing matches|} in
+
+  run source match_template rewrite_template;
+  [%expect_exact {|nothing matches|}]
+
+let%expect_test "comments_9" =
+  let source = {|/* don't match this (a) end */ do match this (b) end|} in
+  let match_template = {|match this :[1] end|} in
+  let rewrite_template = {|:[1]|} in
+
+  run source match_template rewrite_template;
+  [%expect_exact {|/* don't match this (a) end */ do (b)|}]
+
+let%expect_test "comments_10" =
+  let source = {|/* don't match this (a) end */ do match this () end|} in
+  let match_template = {|match this :[1] end|} in
+  let rewrite_template = {|:[1]|} in
+
+  run source match_template rewrite_template;
+  [%expect_exact {|/* don't match this (a) end */ do ()|}]
+
+let%expect_test "comments_11" =
+  let source = {|do match this (b) end /* don't match this (a) end */|} in
+  let match_template = {|match this :[1] end|} in
+  let rewrite_template = {|:[1]|} in
+
+  run source match_template rewrite_template;
+  [%expect_exact {|do (b) /* don't match this (a) end */|}]

test/test_c_separators.ml

@@ -0,0 +1,36 @@
+open Core
+
+open Matchers
+open Rewriter
+
+let configuration = Configuration.create ~match_kind:Fuzzy ()
+
+let run source match_template rewrite_template =
+  C.first ~configuration match_template source
+  |> function
+  | Ok result ->
+    Rewrite.all ~source ~rewrite_template [result]
+    |> (fun x -> Option.value_exn x)
+    |> (fun { rewritten_source; _ } -> rewritten_source)
+    |> print_string
+  | Error _ ->
+    print_string rewrite_template
+
+let%expect_test "whitespace_should_not_matter_between_separators" =
+  let source = {|*p|} in
+  let match_template = {|*:[1]|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|p|}];
+
+  let source = {|*          p|} in
+  let match_template = {|*:[1]|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|          p|}];
+
+  let source = {|*          p|} in
+  let match_template = {|* :[1]|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|p|}]

test/test_c_style_comments.ml

@@ -0,0 +1,225 @@
+open Core
+
+open Matchers
+open Rewriter
+
+let configuration = Configuration.create ~match_kind:Fuzzy ()
+
+let all ?(configuration = configuration) template source =
+  C.all ~configuration ~template ~source
+
+let print_matches matches =
+  List.map matches ~f:Match.to_yojson
+  |> (fun matches -> `List matches)
+  |> Yojson.Safe.pretty_to_string
+  |> print_string
+
+let%expect_test "rewrite_comments_1" =
+  let template = "replace this :[1] end" in
+  let source = "/* don't replace this () end */ do replace this () end" in
+  let rewrite_template = "X" in
+
+  all template source
+  |> (fun matches ->
+      Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact "/* don't replace this () end */ do X"]
+
+let%expect_test "rewrite_comments_2" =
+  let template =
+    {|
+      if (:[1]) { :[2] }
+    |}
+  in
+
+  let source =
+    {|
+      /* if (fake_condition_body_must_be_non_empty) { fake_body; } */
+      // if (fake_condition_body_must_be_non_empty) { fake_body; }
+      if (real_condition_body_must_be_empty) {
+        int i;
+        int j;
+      }
+    |}
+  in
+
+  let rewrite_template =
+    {|
+      if (:[1]) {}
+    |}
+  in
+
+  all template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact
+    {|
+      /* if (fake_condition_body_must_be_non_empty) { fake_body; } */
+      if (real_condition_body_must_be_empty) {}
+    |}]
+
+let%expect_test "capture_comments" =
+  let template = {|if (:[1]) { :[2] }|} in
+  let source = {|if (true) { /* some comment */ console.log(z); }|} in
+  let matches = all template source in
+  print_matches matches;
+  [%expect_exact {|[
+  {
+    "range": {
+      "start": { "offset": 0, "line": 1, "column": 1 },
+      "end": { "offset": 48, "line": 1, "column": 49 }
+    },
+    "environment": [
+      [
+        "1",
+        {
+          "value": "true",
+          "range": {
+            "start": { "offset": 4, "line": 1, "column": 5 },
+            "end": { "offset": 8, "line": 1, "column": 9 }
+          }
+        }
+      ],
+      [
+        "2",
+        {
+          "value": "console.log(z);",
+          "range": {
+            "start": { "offset": 31, "line": 1, "column": 32 },
+            "end": { "offset": 46, "line": 1, "column": 47 }
+          }
+        }
+      ]
+    ],
+    "matched": "if (true) { /* some comment */ console.log(z); }"
+  }
+]|}]
+
+let%expect_test "single_quote_in_comment" =
+  let template =
+    {| {:[1]} |}
+  in
+
+  let source =
+    {|
+       /*'*/
+       {test}
+    |}
+  in
+
+  let rewrite_template =
+    {|
+      {:[1]}
+    |}
+  in
+
+  all template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact
+    {|
+      {test}
+    |}]
+
+let%expect_test "single_quote_in_comment" =
+  let template =
+    {| {:[1]} |}
+  in
+
+  let source =
+    {|
+       {
+         a = 1;
+         /* Events with mask == AE_NONE are not set. So let's initiaize the
+          * vector with it. */
+         for (i = 0; i < setsize; i++)
+       }
+    |}
+  in
+
+  let rewrite_template =
+    {|
+      {:[1]}
+    |}
+  in
+
+  all template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact
+    {|
+      {
+         a = 1;
+         /* Events with mask == AE_NONE are not set. So let's initiaize the
+          * vector with it. */
+         for (i = 0; i < setsize; i++)
+       }
+    |}]
+
+let%expect_test "single_quote_in_comment" =
+  let template =
+    {| {:[1]} |}
+  in
+
+  let source =
+    {|
+       {
+         a = 1;
+         /* ' */
+         for (i = 0; i < setsize; i++)
+       }
+    |}
+  in
+
+  let rewrite_template =
+    {|
+      {:[1]}
+    |}
+  in
+
+  all template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact
+    {|
+      {
+         a = 1;
+         /* ' */
+         for (i = 0; i < setsize; i++)
+       }
+    |}]
+
+let%expect_test "give_back_the_comment_characters_for_newline_comments_too" =
+  let template =
+    {| {:[1]} |}
+  in
+
+  let source =
+    {|
+       {
+         // a comment
+       }
+    |}
+  in
+
+  let rewrite_template =
+    {|
+      {:[1]}
+    |}
+  in
+
+  all template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact
+    {|
+      {
+         // a comment
+       }
+    |}]

test/test_cli.ml

@@ -0,0 +1,178 @@
+open Core
+
+module Time = Core_kernel.Time_ns.Span
+
+let binary_path = "../../../comby"
+
+let read_with_timeout read_from_channel =
+  let read_from_fd = Unix.descr_of_in_channel read_from_channel in
+  let read_from_channel =
+    Unix.select ~read:[read_from_fd] ~write:[] ~except:[] ~timeout:(`After (Time.of_int_sec 5)) ()
+    |> (fun { Unix.Select_fds.read; _ } -> List.hd_exn read)
+    |> Unix.in_channel_of_descr
+  in
+  In_channel.input_all read_from_channel
+
+let read_source_from_stdin command source =
+  let open Unix.Process_channels in
+  let { stdin; stdout ; stderr = _ } = Unix.open_process_full ~env:[||] command in
+  Out_channel.output_string stdin source;
+  Out_channel.flush stdin;
+  Out_channel.close stdin;
+  read_with_timeout stdout
+
+let%expect_test "stdin_command" =
+  let source = "hello world" in
+  let match_template = "hello :[1]" in
+  let rewrite_template = ":[1]" in
+  let command_args =
+    Format.sprintf "-stdin '%s' '%s' -f .c" match_template rewrite_template
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  read_source_from_stdin command source
+  |> print_string;
+  [%expect_exact {|world|}]
+
+let%expect_test "with_match_rule" =
+  let source = "hello world" in
+  let match_template = "hello :[1]" in
+  let rewrite_template = ":[1]" in
+  let rule = {|where :[1] == "world"|} in
+  let command_args =
+    Format.sprintf "-stdin '%s' '%s' -rule '%s' -f .c "
+      match_template rewrite_template rule
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  read_source_from_stdin command source
+  |> print_string;
+  [%expect_exact {|world|}];
+
+  let source = "hello world" in
+  let match_template = "hello :[1]" in
+  let rewrite_template = ":[1]" in
+  let rule = {|where :[1] != "world"|} in
+  let command_args =
+    Format.sprintf "-stdin '%s' '%s' -rule '%s' -f .c "
+      match_template rewrite_template rule
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  read_source_from_stdin command source
+  |> print_string;
+  [%expect_exact {|hello world|}]
+
+let%expect_test "with_rewrite_rule" =
+  let source = "hello world" in
+  let match_template = ":[2] :[1]" in
+  let rewrite_template = ":[1]" in
+  let rule = {|where rewrite :[1] { | ":[_]" -> ":[2]" }|} in
+  let command_args =
+    Format.sprintf "-stdin '%s' '%s' -rule '%s' -f .c "
+      match_template rewrite_template rule
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  read_source_from_stdin command source
+  |> print_string;
+  [%expect_exact {|hello|}]
+
+let%expect_test "json_output_option" =
+  let source = "a X c a Y c" in
+  let match_template = "a :[1] c" in
+  let rewrite_template = "c :[1] a" in
+  let command_args =
+    Format.sprintf "-stdin -json '%s' '%s' -f .c "
+      match_template rewrite_template
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  read_source_from_stdin command source
+  |> print_string;
+  [%expect_exact {|[
+  {
+    "range": {
+      "start": { "offset": 6, "line": -1, "column": -1 },
+      "end": { "offset": 11, "line": -1, "column": -1 }
+    },
+    "replacement_content": "c Y a",
+    "environment": [
+      [
+        "1",
+        {
+          "value": "Y",
+          "range": {
+            "start": { "offset": 2, "line": -1, "column": -1 },
+            "end": { "offset": 3, "line": -1, "column": -1 }
+          }
+        }
+      ]
+    ]
+  },
+  {
+    "range": {
+      "start": { "offset": 0, "line": -1, "column": -1 },
+      "end": { "offset": 5, "line": -1, "column": -1 }
+    },
+    "replacement_content": "c X a",
+    "environment": [
+      [
+        "1",
+        {
+          "value": "X",
+          "range": {
+            "start": { "offset": 2, "line": -1, "column": -1 },
+            "end": { "offset": 3, "line": -1, "column": -1 }
+          }
+        }
+      ]
+    ]
+  }
+]|}];
+
+  let source = "a X c a Y c" in
+  let match_template = "a :[1] c" in
+  let rewrite_template = "c :[1] a" in
+  let command_args =
+    Format.sprintf "-stdin -json -match-only '%s' '%s' -f .c "
+      match_template rewrite_template
+  in
+  let command = Format.sprintf "%s %s" binary_path command_args in
+  read_source_from_stdin command source
+  |> print_string;
+  [%expect_exact {|[
+  {
+    "range": {
+      "start": { "offset": 0, "line": 1, "column": 1 },
+      "end": { "offset": 5, "line": 1, "column": 6 }
+    },
+    "environment": [
+      [
+        "1",
+        {
+          "value": "X",
+          "range": {
+            "start": { "offset": 2, "line": 1, "column": 3 },
+            "end": { "offset": 3, "line": 1, "column": 4 }
+          }
+        }
+      ]
+    ],
+    "matched": "a X c"
+  },
+  {
+    "range": {
+      "start": { "offset": 6, "line": 1, "column": 7 },
+      "end": { "offset": 11, "line": 1, "column": 12 }
+    },
+    "environment": [
+      [
+        "1",
+        {
+          "value": "Y",
+          "range": {
+            "start": { "offset": 8, "line": 1, "column": 9 },
+            "end": { "offset": 9, "line": 1, "column": 10 }
+          }
+        }
+      ]
+    ],
+    "matched": "a Y c"
+  }
+]|}]

test/test_generic.ml

@@ -0,0 +1,408 @@
+open Core
+
+open Matchers
+open Rewriter
+
+let configuration = Configuration.create ~match_kind:Fuzzy ()
+
+let format s =
+  let s = String.chop_prefix_exn ~prefix:"\n" s in
+  let leading_indentation = Option.value_exn (String.lfindi s ~f:(fun _ c -> c <> ' ')) in
+  s
+  |> String.split ~on:'\n'
+  |> List.map ~f:(Fn.flip String.drop_prefix leading_indentation)
+  |> String.concat ~sep:"\n"
+  |> String.chop_suffix_exn ~suffix:"\n"
+
+let run ?(configuration = configuration) source match_template rewrite_template =
+  Generic.first ~configuration match_template source
+  |> function
+  | Ok result ->
+    Rewrite.all ~source ~rewrite_template [result]
+    |> (fun x -> Option.value_exn x)
+    |> (fun { rewritten_source; _ } -> rewritten_source)
+    |> print_string
+  | Error _ ->
+    (* this is too annoying to fix everytime the grammar changes. *)
+    print_string ""
+
+let run_all ?(configuration = configuration) source match_template rewrite_template =
+  Generic.all ~configuration ~template:match_template ~source
+  |> (fun results -> Option.value_exn (Rewrite.all ~source ~rewrite_template results))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string
+
+
+let%expect_test "basic" =
+  let source = {|a b c d|} in
+  let match_template = {|:[1]|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|a b c d|}];
+
+  let source = {|a b c d|} in
+  let match_template = {|a :[1] c d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|b|}];
+
+  let source = {|a b c d|} in
+  let match_template = {|a :[1] d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|b c|}];
+
+  let source = {|a b c d|} in
+  let match_template = {|a :[1]|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|b c d|}];
+
+  let source = {|a b c d|} in
+  let match_template = {|:[1] c d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|a b|}];
+
+  let source = {|a b c d|} in
+  let match_template = {|:[1] :[2]|} in
+  let rewrite_template = {|(:[1]) (:[2])|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(a) (b c d)|}];
+
+  let source = {|a b c d|} in
+  let match_template = {|:[2] :[1]|} in
+  let rewrite_template = {|(:[2]) (:[1])|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(a) (b c d)|}];
+
+  let source = {|a b c d|} in
+  let match_template = {|a :[2] :[1] d|} in
+  let rewrite_template = {|(:[2]) (:[1])|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(b) (c)|}];
+
+  let source = {|a b c d|} in
+  let match_template = {|a :[2] :[1]|} in
+  let rewrite_template = {|(:[2]) (:[1])|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(b) (c d)|}];
+
+  let source = {|a b c d|} in
+  let match_template = {|a :[2] c :[1]|} in
+  let rewrite_template = {|(:[2]) (:[1])|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(b) (d)|}];
+
+  let source = {|x:|} in
+  let match_template = {|:[1]:|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|x|}]
+
+let%expect_test "basic_failures" =
+  let source = {|a x b bbq|} in
+  let match_template = {|a :[1] b c|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {||}];
+
+  let source = {|a b c d|} in
+  let match_template = {|a :[2] d :[1]|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact
+    {||}];
+
+  let source = {|a b c d|} in
+  let match_template = {|a :[2] b :[1]|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {||}]
+
+let%expect_test "delimiter_matching" =
+  let source = {|(a b c) d|} in
+  let match_template = {|(:[1]) d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|a b c|}];
+
+  let source = {|(a b c) d|} in
+  let match_template = {|(:[1] b :[2]) d|} in
+  let rewrite_template = {|(:[1]) (:[2])|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(a) (c)|}];
+
+  let source = {|q(a b c) d|} in
+  let match_template = {|q(:[1] b :[2]) d|} in
+  let rewrite_template = {|(:[1]) (:[2])|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(a) (c)|}];
+
+  let source = {|((a) b)|} in
+  let match_template = {|(:[1] b)|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(a)|}];
+
+  let source = {|((a b c)) d|} in
+  let match_template = {|(:[1]) d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(a b c)|}];
+
+  let source = {|((a b c)) d|} in
+  let match_template = {|(:[1]) d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(a b c)|}];
+
+  let source = {|((a b c) q) d|} in
+  let match_template = {|((:[1]) q) d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|a b c|}];
+
+  let source = {|((a b c) q) d|} in
+  let match_template = {|((:[1] c) q) d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|a b|}];
+
+  let source = {|((a b () c) q) d|} in
+  let match_template = {|((:[1] () c) q) d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|a b|}];
+
+  let source = {|((a ((x) d) b c)) d|} in
+  let match_template = {|((a :[1] :[2] c)) d|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|((x) d)|}];
+
+  let source = {|((a ((x) d) b c)) d|} in
+  let match_template = {|((a (:[1]) :[2] c)) d|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(x) d b|}];
+
+  let source = {|(b (c) d)|} in
+  let match_template = {|(:[1])|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|b (c) d|}];
+
+  let source = {|(b (c) d.)|} in
+  let match_template = {|(:[1].)|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|b (c) d|}];
+
+  let source = {|(b (c.) d.)|} in
+  let match_template = {|(:[1].)|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|b (c.) d|}];
+
+  let source = {|(b. (c) d.)|} in
+  let match_template = {|(:[1].)|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|b. (c) d|}];
+
+  let source = {|(b (c) d.)|} in
+  let match_template = {|(b :[1] d.)|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(c)|}];
+
+  let source = {|outer(inner(dst,src),src)|} in
+  let match_template = {|outer(:[1],src)|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|inner(dst,src)|}];
+
+  let source = {|(b ((c)) d.)|} in
+  let match_template = {|(b :[1] d.)|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|((c))|}];
+
+  let source = {|a b c|} in
+  let match_template = {|a :[1] c|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|b|}];
+
+  let source = {|x = foo;|} in
+  let match_template = {|x = :[1];|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|foo|}];
+
+  let source = {|((a {{x} d} b c)) d|} in
+  let match_template = {|((a {:[1] d} :[2] c)) d|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|{x} b|}];
+
+  let source = {|((a {([{x}]) d} b c)) d|} in
+  let match_template = {|((a {:[1] d} :[2] c)) d|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|([{x}]) b|}];
+
+  let source = {|(((((x)))))|} in
+  let match_template = {|(((:[1])))|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|((x))|}];
+
+  let source = {|((((y(x)z))))|} in
+  let match_template = {|(((:[1])))|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(y(x)z)|}];
+
+  let source = {|((((y(x)z))))|} in
+  let match_template = {|(((:[1]):[2]))|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(y(x)z) |}];
+
+  let source = {|(((x)z))|} in
+  let match_template = {|(((:[1]):[2]))|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|x z|}];
+
+  let source = {|((((x))z))|} in
+  let match_template = {|(((:[1]):[2]))|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|(x) z|}];
+
+  let source = {|lolwtfbbq|} in
+  let match_template = {|lol:[1]bbq|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|wtf|}];
+
+  let source = {|x = foo; x = bar;|} in
+  let match_template = {|x = :[1];|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|foo x = bar;|}];
+
+  let source = {|[ no match prefix ] x = foo; [ no match suffix ]|} in
+  let match_template = {|x = :[1];|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|[ no match prefix ] foo [ no match suffix ]|}];
+
+  let source = {|x = a; x = b; x = c|} in
+  let match_template = {|x = :[1];|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|a x = b; x = c|}];
+
+  let source = {|x = ( x = x; );|} in
+  let match_template = {|x = :[1];|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|( x = x; )|}];
+
+  let source = {|( x = x = x; )|} in
+  let match_template = {|x = :[1];|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|( x = x )|}];
+
+  let source = {|xxx a b d c 1 2 3 b d d blah|} in
+  let match_template = {|a :[1] c :[2] d|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|xxx b d 1 2 3 b d blah|}];
+
+  let source = {|howevenlolwtfbbqispossible|} in
+  let match_template = {|lol:[1]bbq|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|howevenwtfispossible|}];
+
+  let source = {|lolhowevenlolwtfbbqispossiblebbq|} in
+  let match_template = {|lol:[1]bbq|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|howevenlolwtfispossiblebbq|}];
+
+  let source = {|hello my name is bob the builder|} in
+  let match_template = {|:[alongidentifiername] :[2] :[3] :[xyz] :[5] :[6]|} in
+  let rewrite_template = {|:[alongidentifiername] :[2] :[3] :[xyz] :[5] :[6]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|hello my name is bob the builder|}];
+
+  let source = {|www.testdofooname.com/picsinsideit/stunningpictureofkays1381737242g8k4n-280x428.jpg|} in
+  let match_template = {|www.:[1]-:[2].jpg|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|testdofooname.com/picsinsideit/stunningpictureofkays1381737242g8k4n 280x428|}];
+
+  let source = {|https://api.github.com/repos/dmjacobsen/slurm/commits/716c1499695c68afcab848a1b49653574b4fc167|} in
+  let match_template = {|:[1]api.:[2]/repos/:[3]s/:[4]|} in
+  let rewrite_template = {|:[1] :[2] :[3] :[4]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|https:// github.com dmjacobsen/slurm/commit 716c1499695c68afcab848a1b49653574b4fc167|}];
+
+  let source =
+    {|
+      assert(stream->md_len + md_len -
+             si.foo_data_begin <= MAD_BUFFER_MDLEN);
+      memcpy(*stream->foo_data + stream->md_len,
+             mad_bit_nextbyte(&stream->ptr),
+             frame_used = md_len - si.foo_data_begin);
+      stream->md_len += frame_used;
+    |} |> format
+  in
+  let match_template = {|memcpy(:[1], :[2], :[3]);|} in
+  let rewrite_template = {|:[1], :[2], :[3]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|assert(stream->md_len + md_len -
+       si.foo_data_begin <= MAD_BUFFER_MDLEN);
+*stream->foo_data + stream->md_len, mad_bit_nextbyte(&stream->ptr), frame_used = md_len - si.foo_data_begin
+stream->md_len += frame_used;|}]
+
+let%expect_test "significant_whitespace" =
+  let configuration = Configuration.create ~match_kind:Fuzzy ~significant_whitespace:true () in
+  let run = run ~configuration in
+
+  let source = {|two  spaces|} in
+  let match_template = {|:[1]  :[2]|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|two spaces|}];
+
+  (* FIXME: this should fail. also test case where separators do or do not need
+     whitespace.  e.g., strict about strcpy(src,dst) matching a template
+     strcpy(:[1],:[2]) versus strcpy(:[1], :[2]) *)
+  let source = {|two  spaces|} in
+  let match_template = {|:[1] :[2]|} in
+  let rewrite_template = {|:[1] :[2]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|two spaces|}]
+
+let%expect_test "contextual_matching" =
+  let run = run_all in
+
+  let source = {|memcpy(dst1, src1, 1); memcpy(dst2, src2, 2);|} in
+  let match_template = {|memcpy(:[1], :[2], :[3])|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|dst1; dst2;|}];
+
+  let source = {|memcpy(dst1, src1, 1); memcpy(dst2, src2, 2);|} in
+  let match_template = {|memcpy(:[1], :[2], :[3])|} in
+  let rewrite_template = {|:[1]|} in
+  run source match_template rewrite_template;
+  [%expect_exact {|dst1; dst2;|}];

test/test_go.ml

@@ -0,0 +1,101 @@
+open Core
+
+open Language
+open Matchers
+open Match
+open Rewriter
+
+let configuration = Configuration.create ~match_kind:Fuzzy ()
+
+let run ?(rule = "where true") source match_template rewrite_template =
+  let rule = Rule.create rule |> Or_error.ok_exn in
+  Go.first ~configuration match_template source
+  |> function
+  | Ok ({environment; _ } as result) ->
+    if Rule.(sat @@ apply rule environment) then
+      Rewrite.all ~source ~rewrite_template [result]
+      |> (fun x -> Option.value_exn x)
+      |> (fun { rewritten_source; _ } -> rewritten_source)    |> print_string
+    else
+      assert false
+  | Error _ ->
+    print_string rewrite_template
+
+let%expect_test "gosimple_s1000" =
+  let source =
+    {|
+      select {
+      case x := <-ch:
+        fmt.Println(x)
+      }
+    |}
+  in
+
+  let match_template =
+    {|
+      select {
+      case :[1] := :[2]:
+        :[3]
+      }
+    |}
+  in
+
+  let rewrite_template =
+    {|
+      :[1] := :[2]
+      :[3]
+    |}
+  in
+  run source match_template rewrite_template;
+  [%expect_exact {|
+      x := <-ch
+      fmt.Println(x)
+    |}]
+
+let%expect_test "gosimple_s1001" =
+  let source =
+    {|
+      for i, x := range src {
+        dst[i] = x
+      }
+    |}
+  in
+
+  let match_template =
+    {|
+      for :[index_define], :[src_element_define] := range :[src_array] {
+        :[dst_array][:[index_use]] = :[src_element_use]
+      }
+    |}
+  in
+
+  let rewrite_template =
+    {|
+      copy(:[dst_array], :[src_array])
+    |}
+  in
+
+  let rule = {|where :[index_define] == :[index_use], :[src_element_define] == :[src_element_use]|} in
+
+  run ~rule source match_template rewrite_template;
+  [%expect_exact {|
+      copy(dst, src)
+    |}]
+
+let%expect_test "gosimple_s1003" =
+  let source =
+    {|
+      if strings.Index(x, y) != -1 { ignore }
+    |}
+  in
+
+  let match_template =
+    {|
+      if strings.:[1](x, y) != -1 { :[_] }
+    |}
+  in
+
+  let rewrite_template = {|:[1]|} in
+
+  run source match_template rewrite_template;
+  [%expect_exact {|Index|}]

test/test_integration.ml

@@ -0,0 +1,295 @@
+open Core
+
+open Matchers
+open Rewriter
+
+let configuration = Configuration.create ~match_kind:Fuzzy ()
+
+let all ?(configuration = configuration) template source =
+  Generic.all ~configuration ~template ~source
+
+let print_matches matches =
+  List.map matches ~f:Match.to_yojson
+  |> (fun matches -> `List matches)
+  |> Yojson.Safe.pretty_to_string
+  |> print_string
+
+let%expect_test "dont_get_stuck" =
+  let template = "" in
+  let source = "a" in
+  let matches = all ~configuration template source in
+  print_matches matches;
+  [%expect_exact {|[]|}]
+
+let%expect_test "dont_get_stuck" =
+  let template = "a" in
+  let source = "a" in
+  let matches = all template source in
+  print_matches matches;
+  [%expect_exact {|[
+  {
+    "range": {
+      "start": { "offset": 0, "line": 1, "column": 1 },
+      "end": { "offset": 1, "line": 1, "column": 2 }
+    },
+    "environment": [],
+    "matched": "a"
+  }
+]|}]
+
+let%expect_test "dont_get_stuck" =
+  let template = "a" in
+  let source = "aaa" in
+  let matches = all template source in
+  print_matches matches;
+  [%expect_exact {|[
+  {
+    "range": {
+      "start": { "offset": 0, "line": 1, "column": 1 },
+      "end": { "offset": 1, "line": 1, "column": 2 }
+    },
+    "environment": [],
+    "matched": "a"
+  },
+  {
+    "range": {
+      "start": { "offset": 1, "line": 1, "column": 2 },
+      "end": { "offset": 2, "line": 1, "column": 3 }
+    },
+    "environment": [],
+    "matched": "a"
+  },
+  {
+    "range": {
+      "start": { "offset": 2, "line": 1, "column": 3 },
+      "end": { "offset": 3, "line": 1, "column": 4 }
+    },
+    "environment": [],
+    "matched": "a"
+  }
+]|}]
+
+let%expect_test "rewrite_awesome_1" =
+  let template = "replace this :[1] end" in
+  let source = "xreplace this () end" in
+  let rewrite_template = "X" in
+
+  all template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact "xX"]
+
+let%expect_test "rewrite_whole_template_matches" =
+  let template = {|rewrite :[1] <- this string|} in
+  let source = {|rewrite hello world <- this string|} in
+  let rewrite_template = "?" in
+
+  all template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact "?"]
+
+let%expect_test "single_token" =
+  let template = {|:[[1]] this|} in
+  let source = {|the problem is this|} in
+  let matches = all template source in
+  print_matches matches;
+  [%expect_exact {|[
+  {
+    "range": {
+      "start": { "offset": 12, "line": 1, "column": 13 },
+      "end": { "offset": 19, "line": 1, "column": 20 }
+    },
+    "environment": [
+      [
+        "1",
+        {
+          "value": "is",
+          "range": {
+            "start": { "offset": 12, "line": 1, "column": 13 },
+            "end": { "offset": 14, "line": 1, "column": 15 }
+          }
+        }
+      ]
+    ],
+    "matched": "is this"
+  }
+]|}]
+
+
+let%expect_test "single_token_with_preceding_whitespace" =
+  let template = {| :[[1]] this|} in
+  let source = {|the problem is this|} in
+  let matches = all template source in
+  print_matches matches;
+  [%expect_exact {|[
+  {
+    "range": {
+      "start": { "offset": 11, "line": 1, "column": 12 },
+      "end": { "offset": 19, "line": 1, "column": 20 }
+    },
+    "environment": [
+      [
+        "1",
+        {
+          "value": "is",
+          "range": {
+            "start": { "offset": 12, "line": 1, "column": 13 },
+            "end": { "offset": 14, "line": 1, "column": 15 }
+          }
+        }
+      ]
+    ],
+    "matched": " is this"
+  }
+]|}]
+
+let%expect_test "single_token_rewrite" =
+  let template = {| :[[1]] this|} in
+  let source = {|the problem is this|} in
+  let rewrite_template = ":[1]" in
+  all template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact "the problemis"]
+
+let%expect_test "single_token_match_inside_paren_no_succeeding_whitespace" =
+  let template = {|:[[1]](:[[2]])|} in
+  let source = {|foo(bar)|} in
+  let rewrite_template = ":[1] : :[2]" in
+  all template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact "foo : bar"]
+
+
+let%expect_test "shift_or_at_least_dont_get_stuck" =
+  let template = ":[1]" in
+  let source = "a" in
+  let matches = all template source in
+  print_matches matches;
+  [%expect_exact {|[
+  {
+    "range": {
+      "start": { "offset": 0, "line": 1, "column": 1 },
+      "end": { "offset": 1, "line": 1, "column": 2 }
+    },
+    "environment": [
+      [
+        "1",
+        {
+          "value": "a",
+          "range": {
+            "start": { "offset": 0, "line": 1, "column": 1 },
+            "end": { "offset": 1, "line": 1, "column": 2 }
+          }
+        }
+      ]
+    ],
+    "matched": "a"
+  }
+]|}]
+
+let%expect_test "shift_or_at_least_dont_get_stuck" =
+  let template = ":[1]" in
+  let source = "aa" in
+  let matches = all template source in
+  print_matches matches;
+  [%expect_exact {|[
+  {
+    "range": {
+      "start": { "offset": 0, "line": 1, "column": 1 },
+      "end": { "offset": 2, "line": 1, "column": 3 }
+    },
+    "environment": [
+      [
+        "1",
+        {
+          "value": "aa",
+          "range": {
+            "start": { "offset": 0, "line": 1, "column": 1 },
+            "end": { "offset": 2, "line": 1, "column": 3 }
+          }
+        }
+      ]
+    ],
+    "matched": "aa"
+  }
+]|}]
+
+let%expect_test "nested_rewrite1" =
+  let source =
+    {|
+      x x y strcpy(strcpy(dst1, src1), src2); blah blah XXX
+    |}
+  in
+
+  let template =
+    {|
+      strcpy(:[1], :[2])
+    |}
+  in
+
+  let matches = all template source in
+  print_matches matches;
+  [%expect_exact "[]"]
+
+(* FIXME(RVT) nested rewrites *)
+let%expect_test "nested_rewrite2" =
+  let template =
+    {|
+    if :[var_check] != nil {
+      for :[defines] := range :[var_use] {:[inner_body]}
+    }
+    |}
+  in
+  let source =
+    {|
+    if fields.List != nil {
+      for _, field := range fields.List {
+        if field.Names != nil {
+          for _, fieldName := range field.Names {
+            stuff with fields and things
+          }
+        }
+      }
+    }
+    |}
+  in
+  let rewrite_template = "for :[defines] := range :[var_use] {:[inner_body]}" in
+  all template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|for _, field := range fields.List {
+        if field.Names != nil {
+          for _, fieldName := range field.Names {
+            stuff with fields and things
+          }
+        }
+      }|}]
+
+let%expect_test "match_:[[1]]" =
+  let template =
+    {|
+    :[[1]].next()
+    |}
+  in
+  let source =
+    {|
+    col_names = reader.next()
+    }
+    |}
+  in
+  let rewrite_template = "next(:[1])" in
+  all template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|
+    col_names =next(reader)}
+    |}]

test/test_match_rule.ml

@@ -0,0 +1,719 @@
+open Core
+
+open Language
+open Matchers
+open Match
+
+let rule_parses rule =
+  match Rule.create rule with
+  | Ok _ -> "true"
+  | Error _ -> "false"
+
+let print_matches matches =
+  List.map matches ~f:Match.to_yojson
+  |> (fun matches -> `List matches)
+  |> Yojson.Safe.pretty_to_string
+  |> print_string
+
+let%expect_test "parse_rule" =
+  let rule = {| where :[1] == :[2], :[3] == "y" |} in
+  rule_parses rule |> print_string;
+  [%expect_exact {|true|}];
+
+  let rule = {| where :[1] == :[2], :[3] != "x" |} in
+  rule_parses rule |> print_string;
+  [%expect_exact {|true|}];
+
+  let rule =  {| where :[1] != :[3] |} in
+  rule_parses rule |> print_string;
+  [%expect_exact {|true|}]
+
+let%expect_test "parse_basic" =
+  Rule.create {|where "a" == "a"|}
+  |> Or_error.ok_exn
+  |> fun rule -> print_s [%message (rule : Ast.expression list)];
+  [%expect_exact {|(rule ((Equal (String a) (String a))))
+|}]
+
+let%expect_test "parse_match_one_case" =
+  Rule.create {|where match "match_me" { | "case_one" -> true }|}
+  |> Or_error.ok_exn
+  |> fun rule -> print_s [%message (rule : Ast.expression list)];
+  [%expect_exact "(rule ((Match (String match_me) (((String case_one) (True))))))
+"]
+
+let%expect_test "parse_match_multi_case" =
+  Rule.create
+    {| where
+       match "match_me" {
+       | "case_one" -> true
+       | "case_two" -> false
+       }
+    |}
+  |> Or_error.ok_exn
+  |> fun rule -> print_s [%message (rule : Ast.expression list)];
+  [%expect_exact "(rule
+ ((Match (String match_me)
+   (((String case_one) (True)) ((String case_two) (False))))))
+"]
+
+let sat ?(env = Environment.create ()) rule =
+  let rule = Rule.create rule |> Or_error.ok_exn in
+  Format.sprintf "%b" (Rule.(sat @@ apply rule env))
+
+let make_env bindings =
+  List.fold bindings
+    ~init:(Environment.create ())
+    ~f:(fun env (var, value) -> Environment.add env var value)
+
+let%expect_test "rule_sat" =
+  let rule = {| where "x" != "y" |} in
+  sat rule |> print_string;
+  [%expect_exact {|true|}];
+
+  let rule = {| where "x" != "x" |} in
+  sat rule |> print_string;
+  [%expect_exact {|false|}];
+
+  let rule = {| where "x" == "x" |} in
+  sat rule |> print_string;
+  [%expect_exact {|true|}];
+
+  let rule = {| where "x" == "y" |} in
+  sat rule |> print_string;
+  [%expect_exact {|false|}];
+
+  let rule = {| where :[x] == "y" |} in
+  sat rule |> print_string;
+  [%expect_exact {|false|}];
+
+  let rule = {| where :[x] == :[x] |} in
+  sat rule |> print_string;
+  [%expect_exact {|false|}]
+
+let%expect_test "rule_sat_with_env" =
+  let env = make_env ["1", "x"; "2", "y"; "3", "x"] in
+
+  let rule = {| where :[1] == :[3], :[1] != :[2] |} in
+  sat ~env rule |> print_string;
+  [%expect_exact {|true|}];
+
+  let rule = {| where :[1] == :[3], :[1] != "y" |} in
+  sat ~env rule |> print_string;
+  [%expect_exact {|true|}];
+
+  let rule = {| where :[1] == :[3], :[1] == "x" |} in
+  sat ~env rule |> print_string;
+  [%expect_exact {|true|}];
+
+  let rule = {| where :[1] == :[2], :[1] != :[2] |} in
+  sat ~env rule |> print_string;
+  [%expect_exact {|false|}]
+
+let configuration = Configuration.create ~match_kind:Fuzzy ()
+
+let format s =
+  let s = s |> String.chop_prefix_exn ~prefix:"\n" in
+  let leading_indentation =
+    Option.value_exn (String.lfindi s ~f:(fun _ c -> c <> ' ')) in
+  s
+  |> String.split ~on:'\n'
+  |> List.map ~f:(Fn.flip String.drop_prefix leading_indentation)
+  |> String.concat ~sep:"\n"
+  |> String.chop_suffix_exn ~suffix:"\n"
+
+
+let%expect_test "where_true" =
+  let template =
+    {|
+      (:[1]) => {}
+    |}
+    |> format
+  in
+
+  let source =
+    {|
+      (b,c) => {}
+    |}
+    |> format
+  in
+
+  let rule =
+    {| where true
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [
+      {
+        "range": {
+          "start": { "offset": 0, "line": 1, "column": 1 },
+          "end": { "offset": 11, "line": 1, "column": 12 }
+        },
+        "environment": [
+          [
+            "1",
+            {
+              "value": "b,c",
+              "range": {
+                "start": { "offset": 1, "line": 1, "column": 2 },
+                "end": { "offset": 4, "line": 1, "column": 5 }
+              }
+            }
+          ]
+        ],
+        "matched": "(b,c) => {}"
+      }
+    ] |}]
+
+let%expect_test "match_sat" =
+  let template =
+    {|
+      (:[1]) => {}
+    |}
+    |> format
+  in
+
+  let source =
+    {|
+      (b,c) => {}
+    |}
+    |> format
+  in
+
+  let rule =
+    {| where
+       match :[1] {
+       | ":[_],:[_]" -> false
+       }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [] |}];
+
+  let rule =
+    {| where
+       match :[1] {
+       | ":[_],:[_]" -> true
+       }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [
+      {
+        "range": {
+          "start": { "offset": 0, "line": 1, "column": 1 },
+          "end": { "offset": 11, "line": 1, "column": 12 }
+        },
+        "environment": [
+          [
+            "1",
+            {
+              "value": "b,c",
+              "range": {
+                "start": { "offset": 1, "line": 1, "column": 2 },
+                "end": { "offset": 4, "line": 1, "column": 5 }
+              }
+            }
+          ]
+        ],
+        "matched": "(b,c) => {}"
+      }
+    ] |}];
+
+  let source =
+    {|
+      (a) => {}
+      (b,c) => {}
+    |}
+    |> format
+  in
+
+  let rule =
+    {| where
+       match :[1] {
+       | ":[_],:[_]" -> false
+       | ":[_]" -> true
+       }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [
+      {
+        "range": {
+          "start": { "offset": 0, "line": 1, "column": 1 },
+          "end": { "offset": 9, "line": 1, "column": 10 }
+        },
+        "environment": [
+          [
+            "1",
+            {
+              "value": "a",
+              "range": {
+                "start": { "offset": 1, "line": 1, "column": 2 },
+                "end": { "offset": 2, "line": 1, "column": 3 }
+              }
+            }
+          ]
+        ],
+        "matched": "(a) => {}"
+      }
+    ] |}];
+
+  let rule =
+    {|
+       where
+       match :[1] {
+       | ":[_],:[_]" -> false
+       | ":[_]" -> :[1] == "a"
+       }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [
+      {
+        "range": {
+          "start": { "offset": 0, "line": 1, "column": 1 },
+          "end": { "offset": 9, "line": 1, "column": 10 }
+        },
+        "environment": [
+          [
+            "1",
+            {
+              "value": "a",
+              "range": {
+                "start": { "offset": 1, "line": 1, "column": 2 },
+                "end": { "offset": 2, "line": 1, "column": 3 }
+              }
+            }
+          ]
+        ],
+        "matched": "(a) => {}"
+      }
+    ] |}];
+
+  let rule =
+    {| where
+       match :[1] {
+       | ":[_],:[_]" -> false
+       | ":[_]" -> :[1] == "b"
+       }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [] |}]
+
+
+let%expect_test "match_s_suffix" =
+  let template = ":[1]s" in
+
+  let source = "names" in
+
+  let rule =
+    {| where true
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [
+      {
+        "range": {
+          "start": { "offset": 0, "line": 1, "column": 1 },
+          "end": { "offset": 5, "line": 1, "column": 6 }
+        },
+        "environment": [
+          [
+            "1",
+            {
+              "value": "name",
+              "range": {
+                "start": { "offset": 0, "line": 1, "column": 1 },
+                "end": { "offset": 4, "line": 1, "column": 5 }
+              }
+            }
+          ]
+        ],
+        "matched": "names"
+      }
+    ] |}]
+
+let%expect_test "match_s_suffix" =
+  let template = ":[1]" in
+
+  let source = "names" in
+
+  let rule =
+    {| where true
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [
+      {
+        "range": {
+          "start": { "offset": 0, "line": 1, "column": 1 },
+          "end": { "offset": 5, "line": 1, "column": 6 }
+        },
+        "environment": [
+          [
+            "1",
+            {
+              "value": "names",
+              "range": {
+                "start": { "offset": 0, "line": 1, "column": 1 },
+                "end": { "offset": 5, "line": 1, "column": 6 }
+              }
+            }
+          ]
+        ],
+        "matched": "names"
+      }
+    ] |}]
+
+let%expect_test "configuration_choice_based_on_case" =
+  let template = ":[1]" in
+
+  let source = "names" in
+
+  let rule =
+    {| where match :[1] {
+       | "ame" -> true
+       }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [] |}];
+
+  let template = ":[1]" in
+
+  let source = "names" in
+
+  let rule =
+    {| where match :[1] {
+       | "names" -> true
+       }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [
+      {
+        "range": {
+          "start": { "offset": 0, "line": 1, "column": 1 },
+          "end": { "offset": 5, "line": 1, "column": 6 }
+        },
+        "environment": [
+          [
+            "1",
+            {
+              "value": "names",
+              "range": {
+                "start": { "offset": 0, "line": 1, "column": 1 },
+                "end": { "offset": 5, "line": 1, "column": 6 }
+              }
+            }
+          ]
+        ],
+        "matched": "names"
+      }
+    ] |}];
+
+  let template = ":[1]" in
+
+  let source = "namesXXXXX" in
+
+  let rule =
+    {| where match :[1] {
+       | "names" -> true
+       }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [] |}]
+
+
+
+
+let%expect_test "match_using_environment_merge" =
+  let template = "{:[1]}" in
+
+  let source = "{{ a : a } { a : a }}" in
+
+  let rule =
+    {| where match :[1] { | "{ :[x] : :[y] }" -> :[x] == :[y] }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [
+      {
+        "range": {
+          "start": { "offset": 0, "line": 1, "column": 1 },
+          "end": { "offset": 21, "line": 1, "column": 22 }
+        },
+        "environment": [
+          [
+            "1",
+            {
+              "value": "{ a : a } { a : a }",
+              "range": {
+                "start": { "offset": 1, "line": 1, "column": 2 },
+                "end": { "offset": 20, "line": 1, "column": 21 }
+              }
+            }
+          ]
+        ],
+        "matched": "{{ a : a } { a : a }}"
+      }
+    ] |}];
+
+  let template = "{:[1]}" in
+
+  let source = "{{ a : a } { a : b }}" in
+
+  let rule =
+    {| where match :[1] { | "{ :[x] : :[y] }" -> :[x] == :[y] }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [] |}]
+
+
+let%expect_test "nested_matches" =
+  let template = ":[1]" in
+
+  let source = "{ { foo : { bar : { baz : qux } } } }" in
+
+  let rule =
+    {| where match :[1] {
+       | "{ :[foo] : :[tail1] }" -> match :[tail1] {
+                     | "{ :[bar] : :[tail2] }" -> match :[tail2] {
+                                                 | "{ baz : :[qux] }" -> :[qux] == "qux", :[bar] == "bar"
+                                                 }
+                     }
+       }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [
+      {
+        "range": {
+          "start": { "offset": 0, "line": 1, "column": 1 },
+          "end": { "offset": 37, "line": 1, "column": 38 }
+        },
+        "environment": [
+          [
+            "1",
+            {
+              "value": "{ { foo : { bar : { baz : qux } } } }",
+              "range": {
+                "start": { "offset": 0, "line": 1, "column": 1 },
+                "end": { "offset": 37, "line": 1, "column": 38 }
+              }
+            }
+          ]
+        ],
+        "matched": "{ { foo : { bar : { baz : qux } } } }"
+      }
+    ] |}];
+
+  let template = ":[1]" in
+
+  let source = "{ { foo : { bar : { baz : qux } } } }" in
+
+  let rule =
+    {| where match :[1] {
+       | "{ :[foo] : :[tail1] }" -> match :[tail1] {
+                     | "{ :[bar] : :[tail2] }" -> match :[tail2] {
+                                                 | "{ baz : :[qux] }" -> :[qux] == "fail"
+                                                 }
+                     }
+       }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [] |}]
+
+
+let%expect_test "match_on_template" =
+  let template = ":[1]" in
+
+  let source = "oodles" in
+
+  let rule =
+    {| where match "p:[1]" {
+       | "poodles" -> true
+       }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [
+      {
+        "range": {
+          "start": { "offset": 0, "line": 1, "column": 1 },
+          "end": { "offset": 6, "line": 1, "column": 7 }
+        },
+        "environment": [
+          [
+            "1",
+            {
+              "value": "oodles",
+              "range": {
+                "start": { "offset": 0, "line": 1, "column": 1 },
+                "end": { "offset": 6, "line": 1, "column": 7 }
+              }
+            }
+          ]
+        ],
+        "matched": "oodles"
+      }
+    ] |}];
+
+  let template = ":[1]" in
+
+  let source = "poodle" in
+
+  let rule =
+    {| where match ":[1]s" {
+       | "poodles" -> true
+       }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [
+      {
+        "range": {
+          "start": { "offset": 0, "line": 1, "column": 1 },
+          "end": { "offset": 6, "line": 1, "column": 7 }
+        },
+        "environment": [
+          [
+            "1",
+            {
+              "value": "poodle",
+              "range": {
+                "start": { "offset": 0, "line": 1, "column": 1 },
+                "end": { "offset": 6, "line": 1, "column": 7 }
+              }
+            }
+          ]
+        ],
+        "matched": "poodle"
+      }
+    ] |}];
+
+  let template = ":[1]" in
+
+  let source = "poodle" in
+
+  let rule =
+    {| where match ":[1]," {
+       | "poodle" -> true
+       }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  Generic.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } -> Rule.(sat @@ apply rule environment))
+  |> print_matches;
+  [%expect {|
+    [] |}];

test/test_rewrite_parts.ml

@@ -0,0 +1,223 @@
+open Core
+
+open Match
+open Matchers
+open Rewriter
+
+let%expect_test "get_offsets_for_holes" =
+  let rewrite_template = {|1234:[1]1234:[2]|} in
+  let result = Rewrite_template.get_offsets_for_holes rewrite_template ["1"; "2"] in
+  print_s [%message (result : (string * int) list)];
+  [%expect_exact {|(result ((2 8) (1 4)))
+|}]
+
+let%expect_test "get_offsets_for_holes_after_substitution_1" =
+  let rewrite_template = {|1234:[1]1234:[2]|} in
+  let offsets = Rewrite_template.get_offsets_for_holes rewrite_template ["1"; "2"] in
+  let environment =
+    Environment.create ()
+    |> (fun environment -> Environment.add environment "1" "333")
+    |> (fun environment -> Environment.add environment "2" "22")
+  in
+  let result = Rewrite_template.get_offsets_after_substitution offsets environment in
+  print_s [%message (result : (string * int) list)];
+  [%expect_exact {|(result ((2 11) (1 4)))
+|}]
+
+let%expect_test "get_offsets_for_holes_after_substitution_1" =
+  let rewrite_template = {|1234:[1]1234:[3]11:[2]|} in
+  let offsets = Rewrite_template.get_offsets_for_holes rewrite_template ["1"; "3"; "2"] in
+  let environment =
+    Environment.create ()
+    |> (fun environment -> Environment.add environment "1" "333")
+    |> (fun environment -> Environment.add environment "3" "333")
+    |> (fun environment -> Environment.add environment "2" "22")
+  in
+  let result = Rewrite_template.get_offsets_after_substitution offsets environment in
+  print_s [%message (result : (string * int) list)];
+  [%expect_exact {|(result ((2 16) (3 11) (1 4)))
+|}]
+
+
+let configuration = Configuration.create ~match_kind:Fuzzy ()
+
+let all ?(configuration = configuration) template source =
+  C.all ~configuration ~template ~source
+
+let%expect_test "comments_in_string_literals_should_not_be_treated_as_comments_by_fuzzy" =
+  let source = {|123433312343331122|} in
+  let match_template = {|1234:[1]1234:[3]11:[2]|} in
+  let rewrite_template = {|1234:[1]1234:[3]11:[2]|} in
+  all match_template source
+  |> Rewrite.all ~source ~rewrite_template
+  |> (function
+      | Some rewrite_result -> print_string (Yojson.Safe.pretty_to_string (Rewrite.result_to_yojson rewrite_result))
+      | None -> print_string "BROKEN EXPECT");
+  [%expect_exact {|{
+  "rewritten_source": "123433312343331122",
+  "contextual_substitutions": [
+    {
+      "range": {
+        "start": { "offset": 0, "line": -1, "column": -1 },
+        "end": { "offset": 18, "line": -1, "column": -1 }
+      },
+      "replacement_content": "123433312343331122",
+      "environment": [
+        [
+          "1",
+          {
+            "value": "333",
+            "range": {
+              "start": { "offset": 4, "line": -1, "column": -1 },
+              "end": { "offset": 7, "line": -1, "column": -1 }
+            }
+          }
+        ],
+        [
+          "2",
+          {
+            "value": "22",
+            "range": {
+              "start": { "offset": 16, "line": -1, "column": -1 },
+              "end": { "offset": 18, "line": -1, "column": -1 }
+            }
+          }
+        ],
+        [
+          "3",
+          {
+            "value": "333",
+            "range": {
+              "start": { "offset": 11, "line": -1, "column": -1 },
+              "end": { "offset": 14, "line": -1, "column": -1 }
+            }
+          }
+        ]
+      ]
+    }
+  ]
+}|}]
+
+let%expect_test "comments_in_string_literals_should_not_be_treated_as_comments_by_fuzzy" =
+  let source = {|123433312343331122;123433312343331122;|} in
+  let match_template = {|1234:[1]1234:[3]11:[2];|} in
+  let rewrite_template = {|1234:[1]1234:[3]11:[2];|} in
+  all match_template source
+  |> Rewrite.all ~source ~rewrite_template
+  |> (function
+      | Some rewrite_result -> print_string (Yojson.Safe.pretty_to_string (Rewrite.result_to_yojson rewrite_result))
+      | None -> print_string "BROKEN EXPECT");
+  [%expect_exact {|{
+  "rewritten_source": "123433312343331122;123433312343331122;",
+  "contextual_substitutions": [
+    {
+      "range": {
+        "start": { "offset": 19, "line": -1, "column": -1 },
+        "end": { "offset": 38, "line": -1, "column": -1 }
+      },
+      "replacement_content": "123433312343331122;",
+      "environment": [
+        [
+          "1",
+          {
+            "value": "333",
+            "range": {
+              "start": { "offset": 4, "line": -1, "column": -1 },
+              "end": { "offset": 7, "line": -1, "column": -1 }
+            }
+          }
+        ],
+        [
+          "2",
+          {
+            "value": "22",
+            "range": {
+              "start": { "offset": 16, "line": -1, "column": -1 },
+              "end": { "offset": 18, "line": -1, "column": -1 }
+            }
+          }
+        ],
+        [
+          "3",
+          {
+            "value": "333",
+            "range": {
+              "start": { "offset": 11, "line": -1, "column": -1 },
+              "end": { "offset": 14, "line": -1, "column": -1 }
+            }
+          }
+        ]
+      ]
+    },
+    {
+      "range": {
+        "start": { "offset": 0, "line": -1, "column": -1 },
+        "end": { "offset": 19, "line": -1, "column": -1 }
+      },
+      "replacement_content": "123433312343331122;",
+      "environment": [
+        [
+          "1",
+          {
+            "value": "333",
+            "range": {
+              "start": { "offset": 4, "line": -1, "column": -1 },
+              "end": { "offset": 7, "line": -1, "column": -1 }
+            }
+          }
+        ],
+        [
+          "2",
+          {
+            "value": "22",
+            "range": {
+              "start": { "offset": 16, "line": -1, "column": -1 },
+              "end": { "offset": 18, "line": -1, "column": -1 }
+            }
+          }
+        ],
+        [
+          "3",
+          {
+            "value": "333",
+            "range": {
+              "start": { "offset": 11, "line": -1, "column": -1 },
+              "end": { "offset": 14, "line": -1, "column": -1 }
+            }
+          }
+        ]
+      ]
+    }
+  ]
+}|}]
+
+let%expect_test "multiple_contextual_substitutions" =
+  let source = {|foo bar foo|} in
+  let match_template = {|foo|} in
+  let rewrite_template = {|xxxx|} in
+  all match_template source
+  |> Rewrite.all ~source ~rewrite_template
+  |> (function
+      | Some rewrite_result -> print_string (Yojson.Safe.pretty_to_string (Rewrite.result_to_yojson rewrite_result))
+      | None -> print_string "BROKEN EXPECT");
+  [%expect_exact {|{
+  "rewritten_source": "xxxx bar xxxx",
+  "contextual_substitutions": [
+    {
+      "range": {
+        "start": { "offset": 9, "line": -1, "column": -1 },
+        "end": { "offset": 13, "line": -1, "column": -1 }
+      },
+      "replacement_content": "xxxx",
+      "environment": []
+    },
+    {
+      "range": {
+        "start": { "offset": 0, "line": -1, "column": -1 },
+        "end": { "offset": 4, "line": -1, "column": -1 }
+      },
+      "replacement_content": "xxxx",
+      "environment": []
+    }
+  ]
+}|}]

test/test_rewrite_rule.ml

@@ -0,0 +1,169 @@
+open Core
+
+open Language
+open Matchers
+open Match
+open Rewriter
+
+let configuration = Configuration.create ~match_kind:Fuzzy ()
+
+let format s =
+  let s = String.chop_prefix_exn ~prefix:"\n" s in
+  let leading_indentation = Option.value_exn (String.lfindi s ~f:(fun _ c -> c <> ' ')) in
+  s
+  |> String.split ~on:'\n'
+  |> List.map ~f:(Fn.flip String.drop_prefix leading_indentation)
+  |> String.concat ~sep:"\n"
+  |> String.chop_suffix_exn ~suffix:"\n"
+
+let run_rule source match_template rewrite_template rule =
+  Generic.first ~configuration match_template source
+  |> function
+  | Error _ -> print_string "bad"
+  | Ok result ->
+    match result with
+    | ({ environment; _ } as m) ->
+      let e = Rule.(result_env @@ apply rule environment) in
+      match e with
+      | None -> print_string "bad bad"
+      | Some e ->
+        { m with environment = e }
+        |> List.return
+        |> Rewrite.all ~source ~rewrite_template
+        |> (fun x -> Option.value_exn x)
+        |> (fun { rewritten_source; _ } -> rewritten_source)
+        |> print_string
+
+let%expect_test "rewrite_rule" =
+  let source = {|int|} in
+  let match_template = {|:[1]|} in
+  let rewrite_template = {|:[1]|} in
+
+  let rule =
+    {|
+      where rewrite :[1] {
+      | "int" -> "expect"
+      }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  run_rule source match_template rewrite_template rule;
+  [%expect_exact {|expect|}]
+
+let%expect_test "rewrite_rule" =
+  let source = {|string expect|} in
+  let match_template = {|:[1] :[2]|} in
+  let rewrite_template = {|:[2]|} in
+
+  let rule =
+    {|
+      where rewrite :[1] {
+      | "int" -> "5"
+      | "string" -> ":[2]"
+      }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  run_rule source match_template rewrite_template rule;
+  [%expect_exact {|expect|}]
+
+let%expect_test "conditional_rewrite_rule" =
+  let source = {|{ { a : { b : { c : d } } } }|} in
+  let match_template = {|:[1]|} in
+  let rewrite_template = {|:[1]|} in
+
+  let rule =
+    {|
+      where rewrite :[1] {
+      | "{ :[a] : :[rest] }" -> "a" == :[a], "doot"
+      }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  run_rule source match_template rewrite_template rule;
+  [%expect_exact {|doot|}]
+
+let%expect_test "rewrite_rule_using_match_result" =
+  let source = {|{ { a : { b : { c : d } } } }|} in
+  let match_template = {|:[1]|} in
+  let rewrite_template = {|:[1]|} in
+
+  let rule =
+    {|
+      where rewrite :[1] {
+      | "{ :[a] : :[rest] }" -> "a" == :[a], ":[rest]"
+      }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  run_rule source match_template rewrite_template rule;
+  [%expect_exact {|{ b : { c : d } }|}];
+
+  let source = {|{ { a : { b : { c : d } } } }|} in
+  let match_template = {|:[1]|} in
+  let rewrite_template = {|:[1]|} in
+  let rule =
+    {|
+      where rewrite :[1] {
+      | "{ :[a] : :[rest] }" -> "b" == :[a], ":[rest]"
+      }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  run_rule source match_template rewrite_template rule;
+  [%expect_exact {|{ { a : { b : { c : d } } } }|}]
+
+
+let%expect_test "nested_rewrite_rule" =
+  let source = {|{ { a : { b : { c : d } } } }|} in
+  let match_template = {|:[1]|} in
+  let rewrite_template = {|:[1]|} in
+
+  let rule =
+    {|
+      where
+      rewrite :[1] {
+      | "{ :[a] : :[rest] }" ->
+            rewrite :[a] {
+            | "a" -> "b"
+            }, "{ :[a] : :[rest] }"
+      }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  run_rule source match_template rewrite_template rule;
+  [%expect_exact {|{ b : { b : { c : d } } }|}]
+
+let%expect_test "sequenced_rewrite_rule" =
+  let source = {|{ { a : { b : { c : d } } } }|} in
+  let match_template = {|{ :[a] : :[rest] }|} in
+  let rewrite_template = {|{ :[a] : :[rest] }|} in
+
+  let rule =
+    {|
+      where
+      rewrite :[a] {
+      | "a" -> "qqq"
+      },
+      rewrite :[rest] {
+      | "{ b : { :[other] } }" -> "{ :[other] }"
+      }
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+
+  run_rule source match_template rewrite_template rule;
+  [%expect_exact {|{ { qqq : { c : d } } }|}]

test/test_statistics.ml

@@ -0,0 +1,87 @@
+open Core
+
+open Language
+open Matchers
+open Match
+
+let configuration = Configuration.create ~match_kind:Fuzzy ()
+
+let format s =
+  let s = s |> String.chop_prefix_exn ~prefix:"\n" in
+  let leading_indentation =
+    Option.value_exn (String.lfindi s ~f:(fun _ c -> c <> ' ')) in
+  s
+  |> String.split ~on:'\n'
+  |> List.map ~f:(Fn.flip String.drop_prefix leading_indentation)
+  |> String.concat ~sep:"\n"
+  |> String.chop_suffix_exn ~suffix:"\n"
+
+let %expect_test "statistics" =
+  let template =
+    {|
+      def :[fn_name](:[fn_params])
+    |}
+    |> format
+  in
+
+  let source =
+    {|
+      def foo(bar):
+        pass
+
+      def bar(bazz):
+        pass
+    |}
+    |> format
+  in
+
+  let rule =
+    {| where true
+    |}
+    |> Rule.create
+    |> Or_error.ok_exn
+  in
+  Go.all ~configuration ~template ~source
+  |> List.filter ~f:(fun { environment; _ } ->
+      Rule.(sat @@ apply rule environment))
+  |> fun matches ->
+  let statistics =
+    Statistics.
+      { number_of_files = 1
+      ; lines_of_code = 5
+      ; number_of_matches = List.length matches
+      ; total_time = 0.0
+      }
+  in
+  statistics
+  |> Statistics.to_yojson
+  |> Yojson.Safe.pretty_to_string
+  |> print_string;
+  [%expect {|
+    {
+      "number_of_files": 1,
+      "lines_of_code": 5,
+      "number_of_matches": 2,
+      "total_time": 0.0
+    } |}];
+
+  let statistics' =
+    Statistics.merge
+      { number_of_files = 1
+      ; lines_of_code = 10
+      ; number_of_matches = 1
+      ; total_time = 1.5
+      }
+      statistics
+  in
+  statistics'
+  |> Statistics.to_yojson
+  |> Yojson.Safe.pretty_to_string
+  |> print_string;
+  [%expect {|
+    {
+      "number_of_files": 2,
+      "lines_of_code": 15,
+      "number_of_matches": 3,
+      "total_time": 1.5
+    } |}]

test/test_string_literals.ml

@@ -0,0 +1,412 @@
+open Core
+
+open Matchers
+open Rewriter
+
+let format s =
+  let s = String.chop_prefix_exn ~prefix:"\n" s in
+  let leading_indentation = Option.value_exn (String.lfindi s ~f:(fun _ c -> c <> ' ')) in
+  s
+  |> String.split ~on:'\n'
+  |> List.map ~f:(Fn.flip String.drop_prefix leading_indentation)
+  |> String.concat ~sep:"\n"
+  |> String.chop_suffix_exn ~suffix:"\n"
+
+let configuration = Configuration.create ~match_kind:Fuzzy ()
+
+let all ?(configuration = configuration) template source =
+  C.all ~configuration ~template ~source
+
+let print_matches matches =
+  List.map matches ~f:Match.to_yojson
+  |> (fun matches -> `List matches)
+  |> Yojson.Safe.pretty_to_string
+  |> print_string
+
+let%expect_test "comments_in_string_literals_should_not_be_treated_as_comments_by_fuzzy" =
+  let source = {|"/*"(x)|} in
+  let template = {|(:[1])|} in
+  let rewrite_template = {|:[1]|} in
+  all template source
+  |> Rewrite.all ~source ~rewrite_template
+  |> (function
+      | Some { rewritten_source; _ } -> print_string rewritten_source
+      | None -> print_string "BROKEN EXPECT");
+  [%expect_exact {|"/*"x|}]
+
+let%expect_test "comments_in_string_literals_should_not_be_treated_as_comments_by_fuzzy_go_raw" =
+  let source = {|`//`(x)|} in
+  let template = {|(:[1])|} in
+  let rewrite_template = {|:[1]|} in
+  Go.all ~configuration ~template ~source
+  |> Rewrite.all ~source ~rewrite_template
+  |> (function
+      | Some { rewritten_source; _ } -> print_string rewritten_source
+      | None -> print_string "BROKEN EXPECT");
+  [%expect_exact {|`//`x|}]
+
+let%expect_test "tolerate_unbalanced_stuff_in_string_literals" =
+  let template = {|"("|} in
+  let source = {|"("|} in
+  let matches = all ~configuration template source in
+  print_matches matches;
+  [%expect_exact {|[
+  {
+    "range": {
+      "start": { "offset": 0, "line": 1, "column": 1 },
+      "end": { "offset": 3, "line": 1, "column": 4 }
+    },
+    "environment": [],
+    "matched": "\"(\""
+  }
+]|}]
+
+let%expect_test "base_literal_matching" =
+  let source = {|"hello"|} in
+  let match_template = {|":[1]"|} in
+  let rewrite_template = {|:[1]|} in
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|hello|}]
+
+let%expect_test "base_literal_matching" =
+  let source = {|rewrite ("hello") this string|} in
+  let match_template = {|rewrite (":[1]") this string|} in
+  let rewrite_template = {|:[1]|} in
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|hello|}]
+
+let%expect_test "match_string_literals" =
+  let source = {|rewrite (".") this string|} in
+  let match_template = {|rewrite (":[1]") this string|} in
+  let rewrite_template = {|:[1]|} in
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|.|}]
+
+let%expect_test "match_string_literals" =
+  let source = {|rewrite ("") this string|} in
+  let match_template = {|rewrite (":[1]") this string|} in
+  let rewrite_template = {|:[1]|} in
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {||}]
+
+let%expect_test "match_string_literals" =
+  let source = {|"(" match "a""a" this "(" |} in
+  let match_template = {|match :[1] this|} in
+  let rewrite_template = {|:[1]|} in
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|"(" "a""a" "(" |}]
+
+(* this tests special functionality in non-literal hole parser
+   but which must still ignore unbalanced delims within strings *)
+let%expect_test "match_string_literals" =
+  let source = {|"(" match "(""(" this "(" |} in
+  let match_template = {|match :[1] this|} in
+  let rewrite_template = {|:[1]|} in
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|"(" "(""(" "(" |}]
+
+let%expect_test "match_string_literals" =
+  let source = {|rewrite ("") this string|} in
+  let match_template = {|rewrite (:[1]) this string|} in
+  let rewrite_template = {|:[1]|} in
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|""|}]
+
+let%expect_test "base_literal_matching" =
+  let source = {|"("|} in
+  let match_template = {|":[1]"|} in
+  let rewrite_template = {|:[1]|} in
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|(|}]
+
+let%expect_test "base_literal_matching" =
+  let source = {|"(""("|} in
+  let match_template = {|":[1]"|} in
+  let rewrite_template = {|:[1]|} in
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|((|}]
+
+let%expect_test "base_literal_matching" =
+  let source = {|"(""("|} in
+  let match_template = {|":[1]"|} in
+  let rewrite_template = {|:[1]|} in
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|((|}]
+
+let%expect_test "base_literal_matching" =
+  let source = {|"hello world"|} in
+  let match_template = {|":[x] :[y]"|} in
+  let rewrite_template = {|:[x] :[y]|} in
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|hello world|}]
+
+(* complex test: basically, we are checking that the inside of this literal is only matched by the val b part *)
+let%expect_test "base_literal_matching" =
+  let source = {|val a = "class = ${String::class}" val b = "not $a"|} in
+  let match_template = {|":[x]$:[[y]]"|} in
+  let rewrite_template = {|(rewritten part: (:[x]) ([y]))|} in
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|val a = "class = ${String::class}" val b = (rewritten part: (not ) ([y]))|}]
+
+let%expect_test "base_literal_matching" =
+  let source = {|get("type") rekt ("enabled", True)|} in
+  let match_template = {|(":[1]", :[[3]])|} in
+  let rewrite_template = {|(rewritten part: (:[1]) (:[3]))|} in
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|get("type") rekt (rewritten part: (enabled) (True))|}]
+
+let%expect_test "rewrite_string_literals_8" =
+  let source = {|match "\"" this|} in
+  let match_template = {|match "\"" this|} in
+  let rewrite_template = "" in
+
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {||}]
+
+let%expect_test "rewrite_string_literals_8" =
+  let source = {|match "\"" this|} in
+  let match_template = {|match :[1] this|} in
+  let rewrite_template = ":[1]" in
+
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|"\""|}]
+
+let%expect_test "rewrite_string_literals_8" =
+  let source = {|match "\"\"" this|} in
+  let match_template = {|match :[1] this|} in
+  let rewrite_template = ":[1]" in
+
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|"\"\""|}]
+
+let%expect_test "rewrite_string_literals_8" =
+  let source = {|match "\"(\"" "(\"" this|} in
+  let match_template = {|match :[1] this|} in
+  let rewrite_template = ":[1]" in
+
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|"\"(\"" "(\""|}]
+
+let%expect_test "rewrite_string_literals_8" =
+  let source = {|match "\"(\"" "(\"" this|} in
+  let match_template = {|match ":[1]" ":[2]" this|} in
+  let rewrite_template = {|:[1] :[2]|} in
+
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|\"(\" (\"|}]
+
+let%expect_test "rewrite_string_literals_8" =
+  let source = {|match 'sin(gle' 'quo(tes' this|} in
+  let match_template = {|:[1]|} in
+  let rewrite_template = {|:[1]|} in
+
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|match 'sin(gle' 'quo(tes' this|}]
+
+let%expect_test "rewrite_string_literals_8" =
+  let source = {|match '\''|} in
+  let match_template = {|:[1]|} in
+  let rewrite_template = {|:[1]|} in
+
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|match '\''|}]
+
+let%expect_test "rewrite_string_literals_8" =
+  let source = {|match 'asdf'|} in
+  let match_template = {|':[1]'|} in
+  let rewrite_template = {|:[1]|} in
+
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|match asdf|}]
+
+let%expect_test "rewrite_string_literals_8" =
+  let source = {|match '\''|} in
+  let match_template = {|':[1]'|} in
+  let rewrite_template = {|:[1]|} in
+
+  all match_template source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|match \'|}]
+
+
+let%expect_test "go_raw_string_literals" =
+  let source =
+    {|
+       x = x
+       y = `multi-line
+            raw str(ing literal`
+       z = `other multi-line
+            raw stri(ng literal`
+    |}
+  in
+  let match_template = {|`:[1]`|} in
+  let rewrite_template = {|:[1]|} in
+
+  Go.all ~configuration ~source ~template:match_template
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|
+       x = x
+       y = multi-line
+            raw str(ing literal
+       z = other multi-line
+            raw stri(ng literal
+    |}]
+
+let%expect_test "go_raw_string_literals" =
+  let source = {|blah `(` quux|} in
+  let match_template = {|:[1]|} in
+  let rewrite_template = {|:[1]|} in
+
+  Go.all ~configuration ~source ~template:match_template
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|blah `(` quux|}]
+
+let%expect_test "match_string_literals" =
+  let source = {|`(` match `(``(` this `(` |} in
+  let match_template = {|match :[1] this|} in
+  let rewrite_template = {|:[1]|} in
+  Go.all ~configuration ~template:match_template ~source
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|`(` `(``(` `(` |}]
+
+
+let%expect_test "go_raw_string_literals" =
+  let source =
+    {|
+       x = x
+       y = `multi-line
+            raw "str"(ing literal`
+       z = `other multi-line
+            raw '"'\"\\s\\\\\tr\ni(ng literal`
+    |}
+  in
+  let match_template = {|`:[1]`|} in
+  let rewrite_template = {|:[1]|} in
+
+  Go.all ~configuration ~source ~template:match_template
+  |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches))
+  |> (fun { rewritten_source; _ } -> rewritten_source)
+  |> print_string;
+  [%expect_exact {|
+       x = x
+       y = multi-line
+            raw "str"(ing literal
+       z = other multi-line
+            raw '"'\"\\s\\\\\tr\ni(ng literal
+    |}]
+
+let%expect_test "regression_matching_kubernetes" =
+  let source = {|"\n" y = 5|} in
+  let template = {|y = :[1]|} in
+  let rewrite_template = {|:[1]|} in
+  Go.all ~configuration ~template ~source
+  |> Rewrite.all ~source ~rewrite_template
+  |> (function
+      | Some { rewritten_source; _ } -> print_string rewritten_source
+      | None -> print_string "BROKEN EXPECT");
+  [%expect_exact {|"\n" 5|}]
+
+
+let%expect_test "match_escaped_any_char" =
+  let source = {|printf("hello world\n");|} in
+  let template = {|printf(":[1]");|} in
+  let rewrite_template = {|:[1]|} in
+  Go.all ~configuration ~template ~source
+  |> Rewrite.all ~source ~rewrite_template
+  |> (function
+      | Some { rewritten_source; _ } -> print_string rewritten_source
+      | None -> print_string "BROKEN EXPECT");
+  [%expect_exact {|hello world\n|}]
+
+let%expect_test "match_escaped_escaped" =
+  let source = {|printf("hello world\n\\");|} in
+  let template = {|printf(":[1]");|} in
+  let rewrite_template = {|:[1]|} in
+  Go.all ~configuration ~template ~source
+  |> Rewrite.all ~source ~rewrite_template
+  |> (function
+      | Some { rewritten_source; _ } -> print_string rewritten_source
+      | None -> print_string "BROKEN EXPECT");
+  [%expect_exact {|hello world\n\\|}]
+
+let%expect_test "match_escaped_escaped" =
+  let source = {|printf("hello world\n\");|} in
+  let template = {|printf(":[1]");|} in
+  let rewrite_template = {|:[1]|} in
+  Go.all ~configuration ~template ~source
+  |> Rewrite.all ~source ~rewrite_template
+  |> (function
+      | Some { rewritten_source; _ } -> print_string rewritten_source
+      | None -> print_string "EXPECT SUCCESS");
+  [%expect_exact {|EXPECT SUCCESS|}]