propagate metasyntax and factor out external module

2024-08-16 16:50:37 +03:00 · 2021-06-05 22:25:54 -07:00 · 2021-06-05 22:25:54 -07:00 · 22b732fbe7
commit 22b732fbe7
parent ae770938a4
33 changed files with 489 additions and 317 deletions
--- a/lib/app/configuration/command_configuration.ml
+++ b/lib/app/configuration/command_configuration.ml
@ -104,15 +104,15 @@ let read filename =
  String.chop_suffix template ~suffix:"\n"
  |> Option.value ~default:template

-let create_rule rule =
-  match Option.map rule ~f:Matchers.Rule.create with
+let create_rule ~metasyntax rule =
+  match Option.map rule ~f:(Matchers.Rule.create ?metasyntax) with
  | None -> None
  | Some Ok rule -> Some rule
  | Some Error error ->
    Format.eprintf "Rule parse error: %s@." (Error.to_string_hum error);
    exit 1

-let parse_toml path =
+let parse_toml ?metasyntax path =
  let open Toml.Types in
  let toml = Toml.Parser.(from_filename path |> unsafe) in
  let toml = Table.remove (Toml.Min.key "flags") toml in
@ -135,7 +135,7 @@ let parse_toml path =
          Format.eprintf "A 'match' key is required for entry %s@." name;
          exit 1
      in
-      let rule = Table.find_opt (Toml.Min.key "rule") t |> to_string |> create_rule in
+      let rule = Table.find_opt (Toml.Min.key "rule") t |> to_string |> create_rule ~metasyntax in
      let rewrite_template = Table.find_opt (Toml.Min.key "rewrite") t |> to_string in
      if debug then Format.printf "Processed ->%s<-@." match_template;
      (name, (Matchers.Specification.create ~match_template ?rule ?rewrite_template ()))::acc
@ -147,7 +147,7 @@ let parse_toml path =
  |> List.sort ~compare:(fun x y -> String.compare (fst x) (fst y))
  |> List.map ~f:snd

-let parse_templates ?(warn_for_missing_file_in_dir = false) paths =
+let parse_templates ?metasyntax ?(warn_for_missing_file_in_dir = false) paths =
  let parse_directory path =
    let read_optional filename =
      match read filename with
@ -159,7 +159,7 @@ let parse_templates ?(warn_for_missing_file_in_dir = false) paths =
      if warn_for_missing_file_in_dir then Format.eprintf "WARNING: Could not read required match file in %s@." path;
      None
    | Some match_template ->
-      let rule = create_rule @@ read_optional (path ^/ "rule") in
+      let rule = create_rule ~metasyntax @@ read_optional (path ^/ "rule") in
      let rewrite_template = read_optional (path ^/ "rewrite") in
      Matchers.Specification.create ~match_template ?rule ?rewrite_template ()
      |> Option.some
@ -181,7 +181,7 @@ let parse_templates ?(warn_for_missing_file_in_dir = false) paths =
      if Sys.is_directory path = `Yes then
        fold_directory path ~sorted:true ~init:[] ~f
      else
-        parse_toml path)
+        parse_toml ?metasyntax path)

 type interactive_review =
  { editor : string
@ -415,6 +415,23 @@ type t =
  ; substitute_in_place : bool
  }

+let parse_metasyntax metasyntax_path =
+  match metasyntax_path with
+  | None -> Matchers.Metasyntax.default_metasyntax
+  | Some metasyntax_path ->
+    match Sys.file_exists metasyntax_path with
+    | `No | `Unknown ->
+      Format.eprintf "Could not open file: %s@." metasyntax_path;
+      exit 1
+    | `Yes ->
+      Yojson.Safe.from_file metasyntax_path
+      |> Matchers.Metasyntax.of_yojson
+      |> function
+      | Ok c -> c
+      | Error error ->
+        Format.eprintf "%s@." error;
+        exit 1
+
 let emit_errors { input_options; output_options; _ } =
  let error_on =
    [ input_options.stdin && Option.is_some input_options.zip_file
@ -477,7 +494,7 @@ let emit_errors { input_options; output_options; _ } =
         Option.value_exn message
       else
         "UNREACHABLE")
-    ; (let result = Matchers.Rule.create input_options.rule in
+    ; (let result = Matchers.Rule.create ~metasyntax:(parse_metasyntax input_options.custom_metasyntax) input_options.rule in
       Or_error.is_error result
     , if Or_error.is_error result then
         Format.sprintf "Match rule parse error: %s@." @@
@ -513,7 +530,7 @@ let emit_warnings { input_options; output_options; _ } =
         | None, Some ({ match_template; _ } : anonymous_arguments) ->
           [ match_template ]
         | Some templates, _ ->
-           List.map (parse_templates templates) ~f:(fun { match_template; _ } -> match_template)
+           List.map (parse_templates ~metasyntax:(parse_metasyntax input_options.custom_metasyntax) templates) ~f:(fun { match_template; _ } -> match_template)
         | _ -> assert false
       in
       List.exists match_templates ~f:(fun match_template ->
@ -574,23 +591,6 @@ let filter_zip_entries file_filters exclude_directory_prefix exclude_file_prefix
        && not (exclude_the_file exclude_file_prefix (Filename.basename filename))
        && has_acceptable_suffix filename)

-let metasyntax metasyntax_path =
-  match metasyntax_path with
-  | None -> Matchers.Metasyntax.default_metasyntax
-  | Some metasyntax_path ->
-    match Sys.file_exists metasyntax_path with
-    | `No | `Unknown ->
-      Format.eprintf "Could not open file: %s@." metasyntax_path;
-      exit 1
-    | `Yes ->
-      Yojson.Safe.from_file metasyntax_path
-      |> Matchers.Metasyntax.of_yojson
-      |> function
-      | Ok c -> c
-      | Error error ->
-        Format.eprintf "%s@." error;
-        exit 1
-
 let syntax custom_matcher_path =
  match
    Sys.file_exists custom_matcher_path with
@ -622,40 +622,47 @@ let extension file_filters =
    | _, Some extension -> "." ^ extension
    | extension, None -> "." ^ extension

-let of_extension (module E : Matchers.Engine.S) file_filters =
+let of_extension (module Engine : Matchers.Engine.S) (module External : Matchers.External.S) file_filters =
+  let external_handler = External.handler in
  let extension = extension file_filters in
-  match E.select_with_extension extension with
+  match Engine.select_with_extension extension ~external_handler with
  | Some matcher -> matcher, Some extension, None
-  | None -> (module E.Generic), Some extension, None
+  | None -> (module Engine.Generic), Some extension, None

 let select_matcher custom_metasyntax custom_matcher override_matcher file_filters omega =
-  let (module E : Matchers.Engine.S) =
+  let (module Engine : Matchers.Engine.S) =
    if omega then
      (module Matchers.Omega)
    else
      (module Matchers.Alpha)
  in
+  let module External = struct let handler = External_semantic.lsif_hover end in
+  if debug then Format.printf "Set custom external@.";
  match custom_matcher, override_matcher, custom_metasyntax with
  | Some custom_matcher, _, custom_metasyntax ->
    (* custom matcher, optional custom metasyntax *)
-    let metasyntax = metasyntax custom_metasyntax in
+    let metasyntax = parse_metasyntax custom_metasyntax in
    let syntax = syntax custom_matcher in
-    E.create ~metasyntax syntax, None, Some metasyntax
+    if debug then Format.printf "Engine.create@.";
+    Engine.create ~metasyntax syntax, None, Some metasyntax
  | _, Some language, custom_metasyntax ->
    (* forced language, optional custom metasyntax *)
-    let metasyntax = metasyntax custom_metasyntax in
+    let metasyntax = parse_metasyntax custom_metasyntax in
    let (module Metasyntax) = Matchers.Metasyntax.create metasyntax in
    let (module Language) = force_language language in
-    (module (E.Make (Language) (Metasyntax)) : Matchers.Matcher.S), None, Some metasyntax
+    if debug then Format.printf "Engine.Make@.";
+    (module (Engine.Make (Language) (Metasyntax) (External)) : Matchers.Matcher.S), None, Some metasyntax
  | _, _, Some custom_metasyntax ->
    (* infer language from file filters, definite custom metasyntax *)
-    let metasyntax = metasyntax (Some custom_metasyntax) in
+    let metasyntax = parse_metasyntax (Some custom_metasyntax) in
    let (module Metasyntax) = Matchers.Metasyntax.create metasyntax in
    let (module Language) = force_language (extension file_filters) in
-    (module (E.Make (Language) (Metasyntax)) : Matchers.Matcher.S), None, Some metasyntax
+    if debug then Format.printf "Engine.Make2@.";
+    (module (Engine.Make (Language) (Metasyntax) (External)) : Matchers.Matcher.S), None, Some metasyntax
  | _, _, None ->
    (* infer language from file filters, use default metasyntax *)
-    of_extension (module E) file_filters
+    if debug then Format.printf "Engine.Infer@.";
+    of_extension (module Engine) (module External) file_filters

 let regex_of_specifications specifications =
  Format.sprintf "(%s)"
@ -714,7 +721,7 @@ let create
  let open Or_error in
  emit_errors configuration >>= fun () ->
  emit_warnings configuration >>= fun () ->
-  let rule =  Matchers.Rule.create rule |> Or_error.ok_exn in
+  let rule =  Matchers.Rule.create ~metasyntax:(parse_metasyntax custom_metasyntax) rule |> Or_error.ok_exn in
  let specifications =
    match templates, anonymous_arguments with
    | None, Some { match_template; rewrite_template; _ } ->
--- a/lib/app/configuration/dune
+++ b/lib/app/configuration/dune
@ -3,4 +3,4 @@
  (public_name comby.configuration)
  (instrumentation (backend bisect_ppx))
  (preprocess (pps ppx_deriving.show ppx_sexp_conv ppx_sexp_message ppx_deriving_yojson))
-  (libraries comby-kernel comby.patdiff comby.camlzip core yojson ppx_deriving_yojson toml lwt lwt.unix))
+  (libraries comby-kernel comby-semantic comby.patdiff comby.camlzip core yojson ppx_deriving_yojson toml lwt lwt.unix))
--- a/lib/app/configuration/external_semantic.ml
+++ b/lib/app/configuration/external_semantic.ml
@ -0,0 +1,21 @@
+open Core_kernel
+
+open Comby_semantic
+
+let debug =
+  match Sys.getenv "DEBUG_COMBY" with
+  | exception Not_found -> false
+  | _ -> true
+
+let lsif_hover ~name:_ ~filepath ~line ~column =
+  String.chop_prefix_if_exists filepath ~prefix:(Sys.getcwd ()) |> fun filepath_relative_root ->
+  if debug then Format.printf "File relative root: %s@." filepath;
+  if debug then Format.printf "Querying type at %d::%d@." line column;
+  let context =
+    Lsif.Context.
+      { repository = "github.com/sourcegraph/sourcegraph"
+      ; lsif_endpoint = "https://sourcegraph.com/.api/graphql"
+      ; formatting = Markdown ("```go", "```") (* Expose. *)
+      }
+  in
+  Lsif.hover_at context ~filepath:filepath_relative_root ~line ~column
--- a/lib/app/pipeline/pipeline.ml
+++ b/lib/app/pipeline/pipeline.ml
@ -85,8 +85,10 @@ let process_single_source
        (* If there are no matches, return the original source (for editor support). *)
        Replacement ([], input_text, 0)
      | matches ->
+        (* FIXME this should be configured where it's done in command_configuration.ml *)
+        let external_handler = External_semantic.lsif_hover in
        let source = if substitute_in_place then Some input_text else None in
-        match Rewrite.all ?source ?metasyntax ?fresh ?filepath ~rewrite_template matches with
+        match Rewrite.all ?source ?metasyntax ?fresh ?filepath ~external_handler ~rewrite_template matches with
        | None -> Nothing
        | Some { rewritten_source; in_place_substitutions } ->
          Replacement (in_place_substitutions, rewritten_source, List.length matches)
--- a/lib/kernel/comby_kernel.ml
+++ b/lib/kernel/comby_kernel.ml
@ -19,6 +19,8 @@ module Matchers = struct
  module Metasyntax = Matchers.Metasyntax
  type metasyntax = Matchers.Metasyntax.t

+  module External = Matchers.External
+
  module Alpha = Matchers.Alpha
  module Omega = Matchers.Omega

@ -30,7 +32,6 @@ module Matchers = struct

  module Rule = struct
    include Matchers.Rule
-    include Matchers.Rule.Parser
    include Matchers.Evaluate
  end
  type rule = Rule.t
--- a/lib/kernel/comby_kernel.mli
+++ b/lib/kernel/comby_kernel.mli
@ -335,6 +335,17 @@ module Matchers : sig

  type metasyntax = Metasyntax.t

+  module External : sig
+    type t = name:string -> filepath:string -> line:int -> column:int -> string option
+
+    module type S = sig
+      val handler : t
+    end
+
+    (** A module representing the default external handler *)
+    module Default : S
+  end
+
  (** {3 Template}

      Parse a template based on metasynax *)
@ -350,7 +361,6 @@ module Matchers : sig
      | LineEnd
      | ColumnStart
      | ColumnEnd
-      | LsifHover
      | FileName
      | FilePath
      | FileDirectory
@ -362,6 +372,7 @@ module Matchers : sig
      | LowerCamelCase
      | UpperSnakeCase
      | LowerSnakeCase
+      | External of string

    type syntax =
      { variable : string
@ -379,7 +390,7 @@ module Matchers : sig
    type t = atom list
    [@@deriving sexp]

-    module Make : Metasyntax.S -> sig
+    module Make : Metasyntax.S -> External.S -> sig
        val parse : string -> t
        val variables : string -> syntax list
      end
@ -460,7 +471,11 @@ module Matchers : sig
      { nested : bool
      }

-    val create : string -> t Core_kernel.Or_error.t
+    val create
+      :  ?metasyntax:metasyntax
+      -> ?external_handler:External.t
+      -> string
+      -> t Core_kernel.Or_error.t

    val options : t -> options

@ -476,20 +491,21 @@ module Matchers : sig
        [substitute_in_place] is true, rewrite rules substitute their values in
        place (default true). [fresh] introduces fresh variables for evaluating
        rules. [metasyntax] uses the custom metasyntax definition. *)
-    val apply :
-      ?substitute_in_place:bool ->
-      ?metasyntax:Metasyntax.t ->
-      ?filepath:string ->
-      match_all:(
-        ?configuration:Configuration.t ->
-        ?filepath:string ->
-        template:string ->
-        source:string ->
-        unit ->
-        Match.t list
-      ) ->
-      Ast.expression list ->
-      Match.Environment.t -> result
+    val apply
+      :  ?substitute_in_place:bool
+      -> ?metasyntax:Metasyntax.t
+      -> ?external_handler:External.t
+      -> ?filepath:string
+      -> match_all:(
+          ?configuration:Configuration.t
+          -> ?filepath:string
+          -> template:string
+          -> source:string
+          -> unit
+          -> Match.t list)
+      -> Ast.expression list
+      -> Match.Environment.t
+      -> result
  end

  type rule = Rule.t
@ -651,7 +667,7 @@ module Matchers : sig

  module Engine : sig
    module type S = sig
-      module Make : Language.S -> Metasyntax.S -> Matcher.S
+      module Make : Language.S -> Metasyntax.S -> External.S -> Matcher.S

      (** {4 Supported Matchers} *)
      module Text : Matcher.S
@ -710,18 +726,27 @@ module Matchers : sig
      (** [all] returns all default matchers. *)
      val all : (module Matcher.S) list

-      (** [select_with_extension metasyntax file_extension] is a convenience
-          function that returns a matcher associated with a [file_extension]. E.g.,
-          use ".c" to get the C matcher. For a full list of extensions associated
-          with matchers, run comby -list. If [metasyntax] is specified, the matcher
-          will use a custom metasyntax definition instead of the default. *)
-      val select_with_extension : ?metasyntax:Metasyntax.t -> string -> (module Matcher.S) option
+      (** [select_with_extension metasyntax external file_extension] is a
+          convenience function that returns a matcher associated with a
+          [file_extension]. E.g., use ".c" to get the C matcher. For a full list
+          of extensions associated with matchers, run comby -list. If
+          [metasyntax] is specified, the matcher will use a custom metasyntax
+          definition instead of the default. An experimental [external] callback
+          is a general callback for handling external properties in the rewrite
+          template. *)
+      val select_with_extension
+        :  ?metasyntax:Metasyntax.t
+        -> ?external_handler:External.t
+        -> string
+        -> (module Matcher.S) option


-      (** [create metasyntax syntax] creates a matcher for a language defined by
-          [syntax]. If [metasyntax] is specified, the matcher will use a custom
-          metasyntax definition instead of the default. *)
-      val create : ?metasyntax:Metasyntax.t -> Language.Syntax.t -> (module Matcher.S)
+      (** [create metasyntax external syntax] creates a matcher for a language
+          defined by [syntax]. If [metasyntax] is specified, the matcher will use
+          a custom metasyntax definition instead of the default. An experimental
+          [external] callback is a general callback for handling external
+          properties in the rewrite template. *)
+      val create : ?metasyntax:metasyntax -> ?external_handler:External.t -> Language.Syntax.t -> (module Matcher.S)
    end
  end

@ -741,7 +766,7 @@ module Matchers : sig

      Defines rewrite operations.  *)
  module Rewrite : sig
-    (** [all source metasyntax fresh rewrite_template matches] substitutes
+    (** [all source metasyntax external fresh rewrite_template matches] substitutes
        [rewrite_template] with each match in [matches] to create a rewrite result.
        If [source] is specified, each rewrite result is substituted in-place in
        the source. If [source] is not specified, rewritten matches are
@ -755,13 +780,14 @@ module Matchers : sig
    val all
      :  ?source:string
      -> ?metasyntax:metasyntax
+      -> ?external_handler:External.t
      -> ?fresh:(unit -> string)
      -> ?filepath:string
      -> rewrite_template:string
      -> match' list
      -> replacement option

-    (** [substitute metasyntax fresh template environment] substitutes
+    (** [substitute metasyntax external fresh template environment] substitutes
        [template] with the variable and value pairs in the [environment]. It
        returns the result after substitution. If [metasyntax] is defined, the
        rewrite template will respect custom metasyntax definitions.
@ -770,9 +796,12 @@ module Matchers : sig
        specified, the default behavior substitutes :[id()] starting with 1, and
        subsequent :[id()] values increment the ID. If [fresh] is set,
        substitutes the pattern :[id()] with the value of fresh () as the hole is
-        encountered, left to right. *)
+        encountered, left to right. An experimental [external] callback is a
+        general callback for handling external properties in the rewrite
+        template. *)
    val substitute
      :  ?metasyntax:metasyntax
+      -> ?external_handler:External.t
      -> ?fresh:(unit -> string)
      -> ?filepath:string
      -> string
--- a/lib/kernel/matchers/alpha.ml
+++ b/lib/kernel/matchers/alpha.ml
@ -45,7 +45,7 @@ let is_not p s =
 type 'a literal_parser_callback = contents:string -> left_delimiter:string -> right_delimiter:string -> 'a
 type 'a nested_delimiter_callback = left_delimiter:string -> right_delimiter:string -> 'a

-module Make (Lang : Types.Language.S) (Meta : Metasyntax.S) = struct
+module Make (Lang : Types.Language.S) (Meta : Types.Metasyntax.S) (Ext : Types.External.S) = struct
  module rec Matcher : Types.Matcher.S = struct
    module Syntax = Lang.Syntax
    include Lang.Info
@ -961,7 +961,13 @@ module Make (Lang : Types.Language.S) (Meta : Metasyntax.S) = struct
    let all ?configuration ?filepath ?(rule = [Types.Ast.True]) ~template ~source:original_source () : Match.t list =
      let _ : string option = filepath in
      let Rule.{ nested } = Rule.options rule in
-      let template, rule = Preprocess.map_aliases template (Some rule) Meta.aliases in
+      let template, rule =
+        Preprocess.map_aliases
+          (module Meta)
+          (module Ext)
+          template
+          (Some rule)
+      in
      let rule = Option.value_exn rule in (* OK in this case *)
      let rec aux_all ?configuration ?(nested = false) ~template ~source:original_source () =
        let open Or_error in
@ -994,8 +1000,24 @@ module Make (Lang : Types.Language.S) (Meta : Metasyntax.S) = struct
              let result = { result with matched } in
              let result =
                if debug then Format.printf "Rule: %s@." (Sexp.to_string @@ Rule.sexp_of_t rule);
-                (* FIXME metasyntax should propagate *)
-                let sat, env = Program.apply ~metasyntax:Metasyntax.default_metasyntax ~substitute_in_place:true ?filepath rule environment in
+                (* FIXME we should not have to convert here. Pass module, but after fixing this functor's signature. *)
+                let metasyntax =
+                  Metasyntax.
+                    { syntax = Meta.syntax
+                    ; identifier = Meta.identifier
+                    ; aliases = Meta.aliases
+                    }
+                in
+                let external_handler = Ext.handler in
+                let sat, env =
+                  Program.apply
+                    ~metasyntax
+                    ~external_handler
+                    ~substitute_in_place:true
+                    ?filepath
+                    rule
+                    environment
+                in
                if debug && Option.is_some env then Format.printf "Got back: %b %S" sat (Match.Environment.to_string @@ Option.value_exn env);
                let new_env = if sat then env else None in
                match new_env with
@ -1090,6 +1112,7 @@ module Make (Lang : Types.Language.S) (Meta : Metasyntax.S) = struct
    val apply
      :  ?substitute_in_place:bool
      -> ?metasyntax:Types.Metasyntax.t
+      -> ?external_handler:Types.External.t
      -> ?filepath:string
      -> Rule.t
      -> Match.environment
@ -1099,12 +1122,14 @@ module Make (Lang : Types.Language.S) (Meta : Metasyntax.S) = struct
    let apply
        ?(substitute_in_place = true)
        ?metasyntax
+        ?external_handler
        ?filepath
        rule
        env =
      Evaluate.apply
        ~substitute_in_place
        ?metasyntax
+        ?external_handler
        ?filepath
        ~match_all:(Matcher.all ~rule:[Types.Ast.True])
        rule
--- a/lib/kernel/matchers/alpha.mli
+++ b/lib/kernel/matchers/alpha.mli
@ -1,3 +1,3 @@
 open Types

-module Make : Language.S -> Metasyntax.S -> Matcher.S
+module Make : Language.S -> Metasyntax.S -> External.S -> Matcher.S
--- a/lib/kernel/matchers/dune
+++ b/lib/kernel/matchers/dune
@ -3,4 +3,4 @@
  (public_name comby-kernel.matchers)
  (instrumentation (backend bisect_ppx))
  (preprocess (pps ppx_here ppx_sexp_conv ppx_sexp_message ppx_deriving_yojson))
-  (libraries comby-semantic comby-kernel.replacement comby-kernel.parsers comby-kernel.match comby.vangstrom core_kernel mparser mparser-pcre re yojson ppx_deriving_yojson))
+  (libraries comby-kernel.replacement comby-kernel.parsers comby-kernel.match comby.vangstrom core_kernel mparser mparser-pcre re yojson ppx_deriving_yojson))
--- a/lib/kernel/matchers/engine.ml
+++ b/lib/kernel/matchers/engine.ml
@ -2,11 +2,12 @@ open Core_kernel

 open Languages

-module Make (Make : Types.Language.S -> Types.Metasyntax.S -> Types.Matcher.S) : Types.Engine.S = struct
+module Make (Make : Types.Language.S -> Types.Metasyntax.S -> Types.External.S -> Types.Matcher.S) : Types.Engine.S = struct
  module Make = Make

  let create
      ?(metasyntax = Metasyntax.default_metasyntax)
+      ?(external_handler = External.default_external)
      Types.Language.Syntax.
        { user_defined_delimiters
        ; escapable_string_literals
@ -31,60 +32,61 @@ module Make (Make : Types.Language.S -> Types.Metasyntax.S -> Types.Matcher.S) :
    end
    in
    let (module Metasyntax : Metasyntax.S) = Metasyntax.(create metasyntax) in
-    (module Make (User_language) (Metasyntax) : Types.Matcher.S)
+    let module External = struct let handler = external_handler end in
+    (module Make (User_language) (Metasyntax) (External) : Types.Matcher.S)

-  module Text = Make (Text) (Metasyntax.Default)
-  module Paren = Make (Paren) (Metasyntax.Default)
-  module Dyck = Make (Dyck) (Metasyntax.Default)
-  module JSON = Make (JSON) (Metasyntax.Default)
-  module JSONC = Make (JSONC) (Metasyntax.Default)
-  module GraphQL = Make (GraphQL) (Metasyntax.Default)
-  module Dhall = Make (Dhall) (Metasyntax.Default)
-  module Latex = Make (Latex) (Metasyntax.Default)
-  module Assembly = Make (Assembly) (Metasyntax.Default)
-  module Clojure = Make (Clojure) (Metasyntax.Default)
-  module Lisp = Make (Lisp) (Metasyntax.Default)
-  module Generic = Make (Generic) (Metasyntax.Default)
-  module Bash = Make (Bash) (Metasyntax.Default)
-  module Ruby = Make (Ruby) (Metasyntax.Default)
-  module Elixir = Make (Elixir) (Metasyntax.Default)
-  module Python = Make (Python) (Metasyntax.Default)
-  module Html = Make (Html) (Metasyntax.Default)
-  module Xml = Make (Xml) (Metasyntax.Default)
-  module SQL = Make (SQL) (Metasyntax.Default)
-  module Erlang = Make (Erlang) (Metasyntax.Default)
-  module C = Make (C) (Metasyntax.Default)
-  module Csharp = Make (Csharp) (Metasyntax.Default)
-  module Java = Make (Java) (Metasyntax.Default)
-  module CSS = Make (CSS) (Metasyntax.Default)
-  module Kotlin = Make (Kotlin) (Metasyntax.Default)
-  module Scala = Make (Scala) (Metasyntax.Default)
-  module Nim = Make (Nim) (Metasyntax.Default)
-  module Matlab = Make (Matlab) (Metasyntax.Default)
-  module Dart = Make (Dart) (Metasyntax.Default)
-  module Php = Make (Php) (Metasyntax.Default)
-  module Go = Make (Go) (Metasyntax.Default)
-  module Javascript = Make (Javascript) (Metasyntax.Default)
-  module Jsx = Make (Jsx) (Metasyntax.Default)
-  module Typescript = Make (Typescript) (Metasyntax.Default)
-  module Tsx = Make (Tsx) (Metasyntax.Default)
-  module Swift = Make (Swift) (Metasyntax.Default)
-  module Rust = Make (Rust) (Metasyntax.Default)
-  module R = Make (R) (Metasyntax.Default)
-  module OCaml = Make (OCaml) (Metasyntax.Default)
-  module Reason = Make (Reason) (Metasyntax.Default)
-  module Fsharp = Make (Fsharp) (Metasyntax.Default)
-  module Pascal = Make (Pascal) (Metasyntax.Default)
-  module Julia = Make (Julia) (Metasyntax.Default)
-  module Fortran = Make (Fortran) (Metasyntax.Default)
-  module Haskell = Make (Haskell) (Metasyntax.Default)
-  module HCL = Make (HCL) (Metasyntax.Default)
-  module Elm = Make (Elm) (Metasyntax.Default)
-  module Zig = Make (Zig) (Metasyntax.Default)
-  module Coq  = Make (Coq) (Metasyntax.Default)
-  module Move = Make (Move) (Metasyntax.Default)
-  module Solidity = Make (Solidity) (Metasyntax.Default)
-  module C_nested_comments = Make (C_nested_comments) (Metasyntax.Default)
+  module Text = Make (Text) (Metasyntax.Default) (External.Default)
+  module Paren = Make (Paren) (Metasyntax.Default) (External.Default)
+  module Dyck = Make (Dyck) (Metasyntax.Default) (External.Default)
+  module JSON = Make (JSON) (Metasyntax.Default) (External.Default)
+  module JSONC = Make (JSONC) (Metasyntax.Default) (External.Default)
+  module GraphQL = Make (GraphQL) (Metasyntax.Default) (External.Default)
+  module Dhall = Make (Dhall) (Metasyntax.Default) (External.Default)
+  module Latex = Make (Latex) (Metasyntax.Default) (External.Default)
+  module Assembly = Make (Assembly) (Metasyntax.Default) (External.Default)
+  module Clojure = Make (Clojure) (Metasyntax.Default) (External.Default)
+  module Lisp = Make (Lisp) (Metasyntax.Default) (External.Default)
+  module Generic = Make (Generic) (Metasyntax.Default) (External.Default)
+  module Bash = Make (Bash) (Metasyntax.Default) (External.Default)
+  module Ruby = Make (Ruby) (Metasyntax.Default) (External.Default)
+  module Elixir = Make (Elixir) (Metasyntax.Default) (External.Default)
+  module Python = Make (Python) (Metasyntax.Default) (External.Default)
+  module Html = Make (Html) (Metasyntax.Default) (External.Default)
+  module Xml = Make (Xml) (Metasyntax.Default) (External.Default)
+  module SQL = Make (SQL) (Metasyntax.Default) (External.Default)
+  module Erlang = Make (Erlang) (Metasyntax.Default) (External.Default)
+  module C = Make (C) (Metasyntax.Default) (External.Default)
+  module Csharp = Make (Csharp) (Metasyntax.Default) (External.Default)
+  module Java = Make (Java) (Metasyntax.Default) (External.Default)
+  module CSS = Make (CSS) (Metasyntax.Default) (External.Default)
+  module Kotlin = Make (Kotlin) (Metasyntax.Default) (External.Default)
+  module Scala = Make (Scala) (Metasyntax.Default) (External.Default)
+  module Nim = Make (Nim) (Metasyntax.Default) (External.Default)
+  module Matlab = Make (Matlab) (Metasyntax.Default) (External.Default)
+  module Dart = Make (Dart) (Metasyntax.Default) (External.Default)
+  module Php = Make (Php) (Metasyntax.Default) (External.Default)
+  module Go = Make (Go) (Metasyntax.Default) (External.Default)
+  module Javascript = Make (Javascript) (Metasyntax.Default) (External.Default)
+  module Jsx = Make (Jsx) (Metasyntax.Default) (External.Default)
+  module Typescript = Make (Typescript) (Metasyntax.Default) (External.Default)
+  module Tsx = Make (Tsx) (Metasyntax.Default) (External.Default)
+  module Swift = Make (Swift) (Metasyntax.Default) (External.Default)
+  module Rust = Make (Rust) (Metasyntax.Default) (External.Default)
+  module R = Make (R) (Metasyntax.Default) (External.Default)
+  module OCaml = Make (OCaml) (Metasyntax.Default) (External.Default)
+  module Reason = Make (Reason) (Metasyntax.Default) (External.Default)
+  module Fsharp = Make (Fsharp) (Metasyntax.Default) (External.Default)
+  module Pascal = Make (Pascal) (Metasyntax.Default) (External.Default)
+  module Julia = Make (Julia) (Metasyntax.Default) (External.Default)
+  module Fortran = Make (Fortran) (Metasyntax.Default) (External.Default)
+  module Haskell = Make (Haskell) (Metasyntax.Default) (External.Default)
+  module HCL = Make (HCL) (Metasyntax.Default) (External.Default)
+  module Elm = Make (Elm) (Metasyntax.Default) (External.Default)
+  module Zig = Make (Zig) (Metasyntax.Default) (External.Default)
+  module Coq  = Make (Coq) (Metasyntax.Default) (External.Default)
+  module Move = Make (Move) (Metasyntax.Default) (External.Default)
+  module Solidity = Make (Solidity) (Metasyntax.Default) (External.Default)
+  module C_nested_comments = Make (C_nested_comments) (Metasyntax.Default) (External.Default)

  let all : (module Types.Matcher.S) list =
    [ (module Assembly)
@ -140,9 +142,14 @@ module Make (Make : Types.Language.S -> Types.Metasyntax.S -> Types.Matcher.S) :
    ; (module Generic)
    ]

-  let select_with_extension ?(metasyntax = Metasyntax.default_metasyntax) extension : (module Types.Matcher.S) option =
+  let select_with_extension
+      ?(metasyntax = Metasyntax.default_metasyntax)
+      ?(external_handler = External.default_external)
+      extension
+    : (module Types.Matcher.S) option =
    let open Option in
    Languages.select_with_extension extension >>| fun (module Language : Types.Language.S) ->
    let (module Metasyntax) = Metasyntax.(create metasyntax) in
-    (module (Make (Language) (Metasyntax)) : Types.Matcher.S)
+    let module External = struct let handler = external_handler end in
+    (module (Make (Language) (Metasyntax) (External)) : Types.Matcher.S)
 end
--- a/lib/kernel/matchers/engine.mli
+++ b/lib/kernel/matchers/engine.mli
@ -1,3 +1,3 @@
 open Types

-module Make : (Language.S -> Metasyntax.S -> Matcher.S) -> Engine.S
+module Make : (Language.S -> Metasyntax.S -> External.S -> Matcher.S) -> Engine.S
--- a/lib/kernel/matchers/evaluate.ml
+++ b/lib/kernel/matchers/evaluate.ml
@ -14,35 +14,38 @@ let sat = fst

 let result_env = snd

-let match_configuration_of_syntax template =
-  (* decide match configuration based on whether there are holes *)
-  let antecedent_contains_hole_syntax case =
-    String.is_substring case ~substring:Syntax.variable_left_delimiter
-  in
-  if antecedent_contains_hole_syntax template then
-    Configuration.create ~match_kind:Fuzzy ()
-  else
-    Configuration.create ~match_kind:Exact ()
-
 let merge_match_environments matches environment' =
  List.map matches ~f:(fun { environment; _ } ->
      Environment.merge environment environment')

-(* FIXME. Propagate this. *)
-module Template = Template.Make(Metasyntax.Default)
-
-let substitute ?filepath env v =
-  match v with
-  | Template t -> Rewrite.substitute ?filepath (Template.to_string t) env
-  | String s -> s
-
 let apply
    ?(substitute_in_place = true)
-    ?metasyntax
+    ?(metasyntax = Metasyntax.default_metasyntax)
+    ?(external_handler = External.default_external)
    ?filepath
    ~(match_all:(?configuration:Configuration.t -> ?filepath:string -> template:string -> source:string -> unit -> Match.t list))
    predicates
    env =
+  let (module Metasyntax) = Metasyntax.create metasyntax in
+  let module External = struct let handler = external_handler end in
+  let (module Template : Types.Template.S) = (module (Template.Make(Metasyntax)(External))) in
+
+  let match_configuration_of_syntax template =
+    (* decide match configuration based on whether there are holes *)
+    match Template.variables template with
+    | [] -> Configuration.create ~match_kind:Exact ()
+    | _ -> Configuration.create ~match_kind:Fuzzy ()
+  in
+
+  let rewrite_substitute template env =
+    Rewrite.substitute ~metasyntax ~external_handler ?filepath template env in
+
+  let substitute env v =
+    match v with
+    | Template t ->
+      rewrite_substitute (Template.to_string t) env
+    | String s -> s
+  in

  (* accepts only one expression *)
  let rec eval env =
@ -56,15 +59,15 @@ let apply
    (* ==, != *)
    | Equal (Template t, String value)
    | Equal (String value, Template t) ->
-      let other = Rewrite.substitute ?filepath (Template.to_string t) env in
+      let other = rewrite_substitute (Template.to_string t) env in
      let result = String.equal value other in
      result, Some env
    | Equal (String left, String right) ->
      let result = String.equal left right in
      result, Some env
    | Equal (Template left, Template right) ->
-      let left = Rewrite.substitute ?filepath (Template.to_string left) env in
-      let right = Rewrite.substitute ?filepath (Template.to_string right) env in
+      let left = rewrite_substitute (Template.to_string left) env in
+      let right = rewrite_substitute (Template.to_string right) env in
      let result = String.equal left right in
      result, Some env
    | Not_equal (left, right) ->
@ -73,9 +76,9 @@ let apply

    (* match ... { ... } *)
    | Match (source, cases) ->
-      let source = substitute ?filepath env source in
+      let source = substitute env source in
      let evaluate template case_expression =
-        let template = substitute ?filepath env template in
+        let template = substitute env template in
        let configuration = match_configuration_of_syntax template in
        if debug then Format.printf "Running for template %s source %s@." template source;
        match_all ~configuration ~template ~source () |> function
@ -102,19 +105,19 @@ let apply

    (* rewrite ... { ... } *)
    | Rewrite (Template t, (match_template, rewrite_template)) ->
-      let rewrite_template = substitute ?filepath env rewrite_template in
-      let template = substitute ?filepath env match_template in
-      let source = Rewrite.substitute ?filepath (Template.to_string t) env in
+      let rewrite_template = substitute env rewrite_template in
+      let template = substitute env match_template in
+      let source = rewrite_substitute (Template.to_string t) env in
      let configuration = Configuration.create ~match_kind:Fuzzy () in
      let matches = match_all ~configuration ~template ~source () in
      let source = if substitute_in_place then Some source else None in
-      let result = Rewrite.all ?metasyntax ?source ~rewrite_template matches in
+      let result = Rewrite.all ~metasyntax ?source ~rewrite_template matches in
      if Option.is_empty result then
        true, Some env (* rewrites are always sat. *)
      else
        let Replacement.{ rewritten_source; _ } = Option.value_exn result in
        (* substitute for variables that are in the outside scope *)
-        let rewritten_source = Rewrite.substitute ?filepath ?metasyntax rewritten_source env in
+        let rewritten_source = rewrite_substitute rewritten_source env in
        let variable =
          match t with
          | [ Types.Template.Hole { variable; _ } ] -> variable
--- a/lib/kernel/matchers/external.ml
+++ b/lib/kernel/matchers/external.ml
@ -0,0 +1,13 @@
+include Types.External
+
+let default_external =
+  fun ~name:_ ~filepath:_ ~line:_ ~column:_ -> None
+
+let default =
+  let module External = struct
+    let handler = default_external
+  end
+  in
+  (module External : Types.External.S)
+
+module Default = (val default)
--- a/lib/kernel/matchers/external.mli
+++ b/lib/kernel/matchers/external.mli
@ -0,0 +1,9 @@
+open Types
+
+include module type of External
+
+val default_external : External.t
+
+val default : (module External.S)
+
+module Default : External.S
--- a/lib/kernel/matchers/matchers.ml
+++ b/lib/kernel/matchers/matchers.ml
@ -1,6 +1,7 @@
 module Configuration = Configuration
 module Languages = Languages
 module Metasyntax = Metasyntax
+module External = External
 module Rule = Rule
 module Ast = struct
  include Types.Ast
--- a/lib/kernel/matchers/matchers.mli
+++ b/lib/kernel/matchers/matchers.mli
@ -1,6 +1,7 @@
 module Configuration = Configuration
 module Languages = Languages
 module Metasyntax = Metasyntax
+module External = External
 module Rule = Rule

 (* Only need to expose Types.Ast. module type of to export sexp. *)
--- a/lib/kernel/matchers/omega.ml
+++ b/lib/kernel/matchers/omega.ml
@ -46,11 +46,11 @@ let actual = Buffer.create 10

 let rewrite_template = ref ""

-module Make (Language : Types.Language.S) (Meta : Metasyntax.S) = struct
+module Make (Language : Types.Language.S) (Meta : Metasyntax.S) (Ext : External.S) = struct
  module rec Matcher : Types.Matcher.S = struct
    include Language.Info

-    module Template = Template.Make(Meta)
+    module Template = Template.Make(Meta)(Ext)

    let wildcard = "_"

@ -164,10 +164,19 @@ module Make (Language : Types.Language.S) (Meta : Metasyntax.S) = struct
      | Some rule ->
        push_environment_ref := !current_environment_ref;
        push_implicit_equals_match_satisfied := !implicit_equals_match_satisfied;
-        (* FIXME Metasyntax should be propagated here. *)
+        (* FIXME we should not have to convert here. Pass module, but after fixing this functor's signature. *)
+        let metasyntax =
+          Metasyntax.
+            { syntax = Meta.syntax
+            ; identifier = Meta.identifier
+            ; aliases = Meta.aliases
+            }
+        in
+        let external_handler = Ext.handler in
        let sat, env =
          Program.apply
-            ~metasyntax:Metasyntax.default_metasyntax
+            ~metasyntax
+            ~external_handler
            ~substitute_in_place:true
            ?filepath:!filepath_ref
            rule
@ -1014,7 +1023,13 @@ module Make (Language : Types.Language.S) (Meta : Metasyntax.S) = struct
      push_matches_ref := !matches_ref;
      configuration_ref := Option.value configuration ~default:!configuration_ref;
      let Rule.{ nested } = Rule.options rule in
-      let template, rule = Preprocess.map_aliases template (Some rule) Meta.aliases in
+      let template, rule =
+        Preprocess.map_aliases
+          (module Meta)
+          (module Ext)
+          template
+          (Some rule)
+      in
      let rec aux_all ?configuration ?(nested = false) ~template ~source () =
        matches_ref := [];
        if String.is_empty template && String.is_empty source then [trivial]
@ -1101,6 +1116,7 @@ module Make (Language : Types.Language.S) (Meta : Metasyntax.S) = struct
    val apply
      :  ?substitute_in_place:bool
      -> ?metasyntax:Metasyntax.t
+      -> ?external_handler:External.t
      -> ?filepath:string
      -> Rule.t
      -> Match.environment
@ -1109,12 +1125,14 @@ module Make (Language : Types.Language.S) (Meta : Metasyntax.S) = struct
    let apply
        ?(substitute_in_place = true)
        ?metasyntax
+        ?external_handler
        ?filepath
        rule
        env =
      Evaluate.apply
        ~substitute_in_place
        ?metasyntax
+        ?external_handler
        ?filepath
        ~match_all:(Matcher.all ~rule:[Types.Ast.True])
        rule
--- a/lib/kernel/matchers/omega.mli
+++ b/lib/kernel/matchers/omega.mli
@ -1,3 +1,3 @@
 open Types

-module Make : Language.S -> Metasyntax.S -> Matcher.S
+module Make : Language.S -> Metasyntax.S -> External.S -> Matcher.S
--- a/lib/kernel/matchers/preprocess.ml
+++ b/lib/kernel/matchers/preprocess.ml
@ -5,8 +5,13 @@ let debug =
  | exception Not_found -> false
  | _ -> true

-let map_aliases template parent_rule aliases =
-  List.fold aliases
+let map_aliases
+    (module Metasyntax : Metasyntax.S)
+    (module External : External.S)
+    template
+    parent_rule =
+  let module Parser = Rule.Make (Metasyntax) (External) in
+  List.fold Metasyntax.aliases
    ~init:(template, parent_rule)
    ~f:(fun (template, parent_rule) Types.Metasyntax.{ pattern; match_template; rule } ->
        let open Option in
@ -18,7 +23,7 @@ let map_aliases template parent_rule aliases =
          let rule' =
            let rule =
              rule
-              >>| Rule.Parser.create
+              >>| Parser.create
              >>| function
              | Ok rule -> rule
              | Error e -> failwith @@ "Could not parse rule for alias entry:"^(Error.to_string_hum e)
--- a/lib/kernel/matchers/rewrite.ml
+++ b/lib/kernel/matchers/rewrite.ml
@ -75,12 +75,14 @@ let substitute_fresh

 let substitute_in_rewrite_template
    ?fresh
+    ?(external_handler = External.default_external)
    ?(metasyntax = Metasyntax.default_metasyntax)
    ?filepath
    template
    environment =
  let (module M) = Metasyntax.create metasyntax in
-  let module Template_parser = Template.Make(M) in
+  let module External = struct let handler = external_handler end in
+  let module Template_parser = Template.Make(M)(External) in (* FIXME factor out Template_parser *)
  let template = substitute_fresh ~metasyntax ?fresh template in
  let terms = Template_parser.parse template in
  let replacement_content, environment = Template_parser.substitute ?filepath terms environment in
@ -92,8 +94,15 @@ let substitute_in_rewrite_template
      }
  }

-let substitute ?(metasyntax = Metasyntax.default_metasyntax) ?fresh ?filepath template env =
-  let { replacement_content; _ } = substitute_in_rewrite_template ?fresh ?filepath ~metasyntax template env
+let substitute
+    ?(metasyntax = Metasyntax.default_metasyntax)
+    ?external_handler
+    ?fresh
+    ?filepath
+    template
+    env =
+  let { replacement_content; _ } =
+    substitute_in_rewrite_template ~metasyntax ?external_handler ?fresh ?filepath template env
  in replacement_content

 let substitute_matches (matches: Match.t list) source replacements =
@ -115,19 +124,20 @@ let substitute_matches (matches: Match.t list) source replacements =
  ; in_place_substitutions
  }

-let all ?source ?metasyntax ?fresh ?filepath ~rewrite_template rev_matches : result option =
+(* FIXME: all the functors help nothing if we end up calling this without parameterizing by metasyntax, etc. *)
+let all ?source ?metasyntax ?external_handler ?fresh ?filepath ~rewrite_template rev_matches : result option =
  Option.some_if (not (List.is_empty rev_matches)) @@
  match source with
  (* in-place substitution *)
  | Some source ->
    rev_matches
-    |> List.map ~f:(fun Match.{ environment; _ } -> substitute_in_rewrite_template ?filepath ?metasyntax ?fresh rewrite_template environment)
+    |> List.map ~f:(fun Match.{ environment; _ } -> substitute_in_rewrite_template ?filepath ?metasyntax ?external_handler ?fresh rewrite_template environment)
    |> substitute_matches rev_matches source
  (* no in place substitution, emit result separated by newlines *)
  | None ->
    let buf = Buffer.create 20 in
    List.iter rev_matches ~f:(fun m ->
-        substitute_in_rewrite_template ?metasyntax ?fresh rewrite_template m.environment
+        substitute_in_rewrite_template ?metasyntax ?external_handler ?fresh rewrite_template m.environment
        |> fun { replacement_content; _ } ->
        Buffer.add_string buf replacement_content;
        Buffer.add_char buf '\n');
--- a/lib/kernel/matchers/rewrite.mli
+++ b/lib/kernel/matchers/rewrite.mli
@ -7,13 +7,21 @@ open Match
 val substitute_fresh : ?metasyntax:Metasyntax.t -> ?fresh:(unit -> string) -> string -> string

 (** substitute returns the result of substituting env in template *)
-val substitute : ?metasyntax:Metasyntax.t -> ?fresh:(unit -> string) -> ?filepath:string -> string -> Environment.t -> string
+val substitute
+  :  ?metasyntax:Metasyntax.t
+  -> ?external_handler:External.t
+  -> ?fresh:(unit -> string)
+  -> ?filepath:string
+  -> string
+  -> Environment.t
+  -> string

 (** if [source] is given, substitute in-place. If not,
    emit result separated by newlines *)
 val all
  :  ?source:string
  -> ?metasyntax:Types.Metasyntax.t
+  -> ?external_handler:External.t
  -> ?fresh:(unit -> string)
  -> ?filepath:string
  -> rewrite_template:string
--- a/lib/kernel/matchers/rule.ml
+++ b/lib/kernel/matchers/rule.ml
@ -3,9 +3,10 @@ open Vangstrom

 open Types.Ast

-module Template = Template.Make(Metasyntax.Default)

-module Parser = struct
+module Make (Metasyntax : Types.Metasyntax.S) (External : Types.External.S) = struct
+
+  module Template = Template.Make (Metasyntax) (External)

  let is_whitespace = function
    | ' ' | '\t' | '\r' | '\n' -> true
@ -31,12 +32,6 @@ module Parser = struct
    | [ Constant c ] -> String c
    | t -> Template t

-  let variable_parser =
-    lift3 (fun _ v _ -> String.of_char_list v)
-      (string Syntax.variable_left_delimiter)
-      (many (alphanum <|> char '_'))
-      (string Syntax.variable_right_delimiter)
-
  (** Interpret escape sequences inside quotes *)
  let char_token_s =
    (char '\\' *> any_char >>|
@ -216,6 +211,15 @@ end
 type t = Types.Rule.t
 [@@deriving sexp]

+let create
+    ?(metasyntax = Metasyntax.default_metasyntax)
+    ?(external_handler = External.default_external)
+    rule =
+  let (module Metasyntax) = Metasyntax.create metasyntax in
+  let module External = struct let handler = external_handler end in
+  let (module Rule : Types.Rule.S) = (module (Make (Metasyntax) (External))) in
+  Rule.create rule
+
 type options =
  { nested : bool
  }
--- a/lib/kernel/matchers/script.ml
+++ b/lib/kernel/matchers/script.ml
@ -1,83 +1,87 @@
 open Core_kernel
 open Vangstrom

-open Rule
-open Parser
 open Types.Ast

-type spec = Specification.t
-[@@deriving sexp]
+module Make (Metasyntax : Types.Metasyntax.S) (External : Types.External.S) = struct

-type op =
-  | And
-  | Or
-  | Not
-[@@deriving sexp]
+  module Parser = Rule.Make (Metasyntax) (External)

-type exp =
-  | Exp of op * exp list
-  | Spec of spec
-[@@deriving sexp]
+  type spec = Specification.t
+  [@@deriving sexp]

-type t = exp list
-[@@deriving sexp]
+  type op =
+    | And
+    | Or
+    | Not
+  [@@deriving sexp]

-let ignore p =
-  p *> return ()
+  type exp =
+    | Exp of op * exp list
+    | Spec of spec
+  [@@deriving sexp]

-let spaces = many @@ satisfy (function ' ' | '\n' | '\r' | '\t' -> true | _ -> false)
-let spaces1 = many1 @@ satisfy (function ' ' | '\n' | '\r' | '\t' -> true | _ -> false)
+  type t = exp list
+  [@@deriving sexp]

-let optional s = option () (ignore @@ string s)
+  let ignore p =
+    p *> return ()

-let chainl1 e op =
-  let rec parse acc = (lift2 (fun f x -> f acc x) op e >>= parse) <|> return acc in
-  e >>= fun init -> parse init
+  let spaces = many @@ satisfy (function ' ' | '\n' | '\r' | '\t' -> true | _ -> false)
+  let spaces1 = many1 @@ satisfy (function ' ' | '\n' | '\r' | '\t' -> true | _ -> false)

-let parens p = char '(' *> (p <|> return []) <* char ')'
+  let optional s = option () (ignore @@ string s)
+
+  let chainl1 e op =
+    let rec parse acc = (lift2 (fun f x -> f acc x) op e >>= parse) <|> return acc in
+    e >>= fun init -> parse init
+
+  let parens p = char '(' *> (p <|> return []) <* char ')'


-let template_parser until =
-  choice
-    [ (lift to_atom quoted_parser)
-    ; (lift (fun v -> to_atom (String.of_char_list v)) (Omega_parser_helper.many1_till any_char until))
-    ]
+  let template_parser until =
+    choice
+      [ (lift Parser.to_atom Parser.quoted_parser)
+      ; (lift (fun v -> Parser.to_atom (String.of_char_list v)) (Omega_parser_helper.many1_till any_char until))
+      ]

-let spec =
-  let match_rewrite_parser =
-    both
-      (spaces *> template_parser (spaces *> string "->"))
-      (option None (spaces *> string Syntax.arrow *> spaces *> template_parser (spaces1 *> string "where" *> spaces1)  >>| fun v -> Some v)) (* FIXME use of reserved *)
-  in
-  match_rewrite_parser >>= fun (match_template_atom, rewrite_template_atom) ->
-  (option None (spaces1 *> parse >>| fun x -> Some x)) >>= fun rule ->
-  let match_template = Sexplib.Sexp.to_string_hum (sexp_of_atom match_template_atom) in
-  let rewrite_template =
-    match rewrite_template_atom with
-    | Some rewrite_template_atom -> Some (Sexplib.Sexp.to_string_hum (sexp_of_atom rewrite_template_atom))
-    | None -> None
-  in
-  return [(Spec (Specification.{ match_template; rule; rewrite_template }))]
+  let spec =
+    let match_rewrite_parser =
+      both
+        (spaces *> template_parser (spaces *> string "->"))
+        (option None (spaces *> string Syntax.arrow *> spaces *> template_parser (spaces1 *> string "where" *> spaces1)  >>| fun v -> Some v)) (* FIXME use of reserved *)
+    in
+    match_rewrite_parser >>= fun (match_template_atom, rewrite_template_atom) ->
+    (option None (spaces1 *> Parser.parse >>| fun x -> Some x)) >>= fun rule ->
+    let match_template = Sexplib.Sexp.to_string_hum (sexp_of_atom match_template_atom) in
+    let rewrite_template =
+      match rewrite_template_atom with
+      | Some rewrite_template_atom -> Some (Sexplib.Sexp.to_string_hum (sexp_of_atom rewrite_template_atom))
+      | None -> None
+    in
+    return [(Spec (Specification.{ match_template; rule; rewrite_template }))]

-let unop syntax exp_parser =
-  choice (List.map ~f:string syntax) *> spaces *> exp_parser >>| fun exp -> [Exp (Not, exp)]
+  let unop syntax exp_parser =
+    choice (List.map ~f:string syntax) *> spaces *> exp_parser >>| fun exp -> [Exp (Not, exp)]

-let binop syntax op =
-  spaces *> choice (List.map ~f:string syntax) *> spaces *> return (fun left right -> [Exp (op, left@right)])
+  let binop syntax op =
+    spaces *> choice (List.map ~f:string syntax) *> spaces *> return (fun left right -> [Exp (op, left@right)])

-let exp_parser =
-  fix (fun exp ->
-      let exp_parser = fix (fun exp' -> parens exp <|> unop ["NOT"; "not"] exp' <|> spec) in
-      let and_parser = chainl1 exp_parser @@ binop ["AND"; "and"] And in
-      let seq_parser = chainl1 and_parser @@ binop ["OR"; "or"] Or in
-      sep_by1 (spaces *> string Syntax.separator <* spaces) seq_parser >>| List.concat)
+  let exp_parser =
+    fix (fun exp ->
+        let exp_parser = fix (fun exp' -> parens exp <|> unop ["NOT"; "not"] exp' <|> spec) in
+        let and_parser = chainl1 exp_parser @@ binop ["AND"; "and"] And in
+        let seq_parser = chainl1 and_parser @@ binop ["OR"; "or"] Or in
+        sep_by1 (spaces *> string Syntax.separator <* spaces) seq_parser >>| List.concat)

-let parser =
-  spaces *> optional Syntax.separator *>
-  exp_parser <* optional Syntax.separator <* spaces <* end_of_input
+  let parser =
+    spaces *> optional Syntax.separator *>
+    exp_parser <* optional Syntax.separator <* spaces <* end_of_input

-let parse script =
-  parse_string ~consume:All parser script
+  let parse script =
+    parse_string ~consume:All parser script

-let to_string exp =
-  Sexplib.Sexp.to_string_hum (sexp_of_t exp)
+  let to_string exp =
+    Sexplib.Sexp.to_string_hum (sexp_of_t exp)
+
+end
--- a/lib/kernel/matchers/specification.ml
+++ b/lib/kernel/matchers/specification.ml
@ -11,7 +11,6 @@ type t =
 let create ?rewrite_template ?rule ~match_template () =
  { match_template; rule; rewrite_template }

-
 let identifier_parser () =
  many (Omega_parser_helper.alphanum <|> char '_')
  >>| String.of_char_list
--- a/lib/kernel/matchers/syntax.ml
+++ b/lib/kernel/matchers/syntax.ml
@ -3,8 +3,6 @@ let start_match_pattern = "match"
 let start_rewrite_pattern = "rewrite"
 let equal = "=="
 let not_equal = "!="
-let variable_left_delimiter = ":["
-let variable_right_delimiter = "]"
 let true' = "true"
 let false' = "false"
 let option_nested = "nested"
--- a/lib/kernel/matchers/template.ml
+++ b/lib/kernel/matchers/template.ml
@ -9,7 +9,7 @@ let debug =
  | exception Not_found -> false
  | _ -> true

-module Make (Metasyntax : Types.Metasyntax.S) = struct
+module Make (Metasyntax : Types.Metasyntax.S) (External : Types.External.S) : Types.Template.S = struct

  let up_to p =
    many1 (not_followed_by p *> any_char)
@ -88,7 +88,6 @@ module Make (Metasyntax : Types.Metasyntax.S) = struct
    | "column"
    | "column.start" -> ColumnStart
    | "column.end" -> ColumnEnd
-    | "lsif.hover" -> LsifHover
    | "file.path" -> FilePath
    | "file.name" -> FileName
    | "file.directory" -> FileDirectory
@ -100,6 +99,7 @@ module Make (Metasyntax : Types.Metasyntax.S) = struct
    | "lowerCamelCase" -> LowerCamelCase
    | "UPPER_SNAKE_CASE" -> UpperSnakeCase
    | "lower_snake_case" -> LowerSnakeCase
+    | "lsif.hover" -> External "lsif.hover"
    | s -> failwith @@ Format.sprintf "invalid attribute %S" s

  let attribute_access () =
@ -116,7 +116,6 @@ module Make (Metasyntax : Types.Metasyntax.S) = struct
      ; string "column.start"
      ; string "column.end"
      ; string "column"
-      ; string "lsif.hover"
      ; string "file.path"
      ; string "file.name"
      ; string "file.directory"
@ -128,6 +127,7 @@ module Make (Metasyntax : Types.Metasyntax.S) = struct
      ; string "lowerCamelCase"
      ; string "UPPER_SNAKE_CASE"
      ; string "lower_snake_case"
+      ; string "lsif.hover"
      ]
    <* not_followed_by (Omega_parser_helper.alphanum)

@ -174,7 +174,7 @@ module Make (Metasyntax : Types.Metasyntax.S) = struct
    many @@ choice
      [ (pos >>= fun offset -> hole >>| fun (pattern, variable, kind) ->
         Hole { pattern; variable; offset; kind = attribute_to_kind kind })
-      ; ((up_to (choice rewrite_hole_parsers)) >>| fun c -> Constant (String.of_char_list c))
+      ; (up_to (choice rewrite_hole_parsers) >>| fun c -> Constant (String.of_char_list c))
      ]

  let parse template =
@ -261,29 +261,6 @@ module Make (Metasyntax : Types.Metasyntax.S) = struct
      let _, column = Match.Offset.convert_fast ~offset index in
      Int.to_string column

-    | LsifHover ->
-      filepath >>= fun filepath ->
-      if debug then Format.printf "File for lsif.hover lookup: %s@." filepath;
-      Environment.lookup env variable >>= fun value ->
-      Environment.lookup_range env variable
-      >>= fun { match_start = { offset; _ }; _ } ->
-      if debug then Format.printf "Var offset is %d@." offset;
-      let source = In_channel.read_all filepath in (* Inefficient. *)
-      if debug then Format.printf "Read filepath, source len is %d@." @@ String.length source;
-      String.chop_prefix_if_exists filepath ~prefix:(Sys.getcwd ()) |> fun filepath_relative_root ->
-      if debug then Format.printf "File relative root: %s@." filepath;
-      let index = Match.Offset.index ~source in
-      let line, column = Match.Offset.convert_fast ~offset index in
-      let line, column = line - 1, column - 1 + String.length value - 1 in
-      if debug then Format.printf "Querying type at %d::%d@." line column;
-      let context =
-        Comby_semantic.Lsif.Context.
-          { repository = "github.com/sourcegraph/sourcegraph"
-          ; lsif_endpoint = "https://sourcegraph.com/.api/graphql"
-          ; formatting = Markdown ("```go", "```") (* Expose. *)
-          }
-      in
-      Comby_semantic.Lsif.hover_at context ~filepath:filepath_relative_root ~line ~column
    | FilePath -> filepath
    | FileName -> filepath >>| Filename.basename
    | FileDirectory -> filepath >>| Filename.dirname
@ -320,6 +297,22 @@ module Make (Metasyntax : Types.Metasyntax.S) = struct
      >>| camel_to_snake
      >>| String.lowercase

+    | External "lsif.hover" ->
+      filepath >>= fun filepath ->
+      if debug then Format.printf "File for lsif.hover lookup: %s@." filepath;
+      Environment.lookup env variable >>= fun value ->
+      Environment.lookup_range env variable
+      >>= fun { match_start = { offset; _ }; _ } ->
+      let source = In_channel.read_all filepath in (* Inefficient. *)
+      if debug then Format.printf "Read filepath, source len is %d@." @@ String.length source;
+      let index = Match.Offset.index ~source in
+      let line, column = Match.Offset.convert_fast ~offset index in
+      let line, column = line - 1, column - 1 + String.length value - 1 in
+      if debug then Format.printf "Var offset:%d line:%d col:%d @." offset line column;
+      External.handler ~name:"lsif.hover" ~filepath ~line ~column
+
+    | External _ -> assert false
+
  let substitute ?filepath template environment =
    let replacement_content, environment', _ =
      List.fold template ~init:([], Environment.create (), 0) ~f:(fun (result, env, pos) -> function
--- a/lib/kernel/matchers/template.mli
+++ b/lib/kernel/matchers/template.mli
@ -1,16 +1,3 @@
-open Types.Template
+open Types

-module Make : Metasyntax.S -> sig
-
-    module Matching : sig
-      val hole_parsers : (Types.Hole.sort * string Vangstrom.t) list
-    end
-
-    val parse : string -> t
-
-    val variables : string -> syntax list
-
-    val to_string : t -> string
-
-    val substitute : ?filepath:string -> t -> Match.Environment.t -> (string * Match.Environment.t)
-  end
+module Make : Metasyntax.S -> External.S -> Template.S
--- a/lib/kernel/matchers/types.ml
+++ b/lib/kernel/matchers/types.ml
@ -116,6 +116,14 @@ module Metasyntax = struct
  end
 end

+module External = struct
+  type t = name:string -> filepath:string -> line:int -> column:int -> string option
+
+  module type S = sig
+    val handler : t
+  end
+end
+
 type production =
  | Unit
  | String of string
@ -132,7 +140,6 @@ module Template = struct
    | LineEnd
    | ColumnStart
    | ColumnEnd
-    | LsifHover
    | FileName
    | FilePath
    | FileDirectory
@ -144,6 +151,7 @@ module Template = struct
    | LowerCamelCase
    | UpperSnakeCase
    | LowerSnakeCase
+    | External of string
  [@@deriving sexp]

  type syntax =
@ -161,6 +169,21 @@ module Template = struct

  type t = atom list
  [@@deriving sexp]
+
+  module type S = sig
+
+    module Matching : sig
+      val hole_parsers : (Hole.sort * string Vangstrom.t) list
+    end
+
+    val parse : string -> t
+
+    val variables : string -> syntax list
+
+    val to_string : t -> string
+
+    val substitute : ?filepath:string -> t -> Match.Environment.t -> (string * Match.Environment.t)
+  end
 end

 module Ast = struct
@ -187,6 +210,10 @@ end
 module Rule = struct
  type t = Ast.expression list
  [@@deriving sexp]
+
+  module type S = sig
+    val create : string -> (Ast.expression list, Error.t) result
+  end
 end

 module Matcher = struct
@ -216,7 +243,7 @@ end

 module Engine = struct
  module type S = sig
-    module Make : Language.S -> Metasyntax.S -> Matcher.S
+    module Make : Language.S -> Metasyntax.S -> External.S -> Matcher.S

    module Text : Matcher.S
    module Paren : Matcher.S
@ -272,7 +299,7 @@ module Engine = struct
    module C_nested_comments : Matcher.S

    val all : (module Matcher.S) list
-    val select_with_extension : ?metasyntax:Metasyntax.t -> string -> (module Matcher.S) option
-    val create : ?metasyntax:Metasyntax.t -> Language.Syntax.t -> (module Matcher.S)
+    val select_with_extension : ?metasyntax:Metasyntax.t -> ?external_handler:External.t -> string -> (module Matcher.S) option
+    val create : ?metasyntax:Metasyntax.t -> ?external_handler:External.t -> Language.Syntax.t -> (module Matcher.S)
  end
 end
--- a/test/common/test_cli.ml
+++ b/test/common/test_cli.ml
@ -1245,17 +1245,17 @@ let%expect_test "test_custom_metasyntax_substitute" =
  print_string result;
  [%expect "$A hello"]

-let%expect_test "test_custom_metasyntax_partial_rule_support" =
+let%expect_test "test_custom_metasyntax_rule_support" =
  let source = "a(b)" in
  let metasyntax_path = "example" ^/ "metasyntax" ^/ "dolla.json" in
  let command_args =
-    Format.sprintf {|'$A($B)' '$A $B' -rule 'where rewrite :[A] { "$C~a$" -> "$C" }' -stdin -custom-metasyntax %s -stdout -matcher .generic|} metasyntax_path
+    Format.sprintf {|'$A($B)' '$A $B' -rule 'where rewrite $A { "$C~a$" -> "$C" }' -stdin -custom-metasyntax %s -stdout -matcher .generic|} metasyntax_path
  in
  let command = Format.sprintf "%s %s" binary_path command_args in
  let result = read_expect_stdin_and_stdout command source in
  print_string result;
  [%expect "
-    $C b"]
+    a b"]

 let%expect_test "test_custom_metasyntax_reserved_identifiers" =
  let source = "fun f -> (fun x -> f (x x)) (fun x -> f (x x))" in
--- a/test/common/test_helpers.ml
+++ b/test/common/test_helpers.ml
@ -48,7 +48,7 @@ let make_env bindings =

 let parse_template metasyntax template =
  let (module M) = Matchers.Metasyntax.create metasyntax in
-  let module Template_parser = Template.Make(M) in
+  let module Template_parser = Template.Make(M)(External.Default) in
  let tree = Template_parser.parse template in
  Sexp.to_string_hum (Template.sexp_of_t tree)

--- a/test/common/test_parse_template.ml
+++ b/test/common/test_parse_template.ml
@ -3,40 +3,41 @@ open Comby_kernel

 open Test_helpers

+open Matchers

 let%expect_test "get_offsets_for_holes" =
-  let module Template_parser = Matchers.Template.Make(Matchers.Metasyntax.Default) in
+  let module Template_parser = Template.Make(Metasyntax.Default)(External.Default) in
  let rewrite_template = {|1234:[1]1234:[2]|} in
  let variables = Template_parser.variables rewrite_template in
-  print_s [%message (variables : Matchers.Template.syntax list)];
+  print_s [%message (variables : Template.syntax list)];
  [%expect {|
    (variables
     (((variable 1) (pattern :[1]) (offset 4) (kind Value))
      ((variable 2) (pattern :[2]) (offset 12) (kind Value)))) |}]

 let%expect_test "interpret_regex_shorthand" =
-  let module Template_parser = Matchers.Template.Make(Matchers.Metasyntax.Default) in
+  let module Template_parser = Template.Make(Metasyntax.Default)(External.Default) in
  let rewrite_template = {|a:[~x]b|} in
  let variables = Template_parser.variables rewrite_template in
-  print_s [%message (variables : Matchers.Template.syntax list)];
+  print_s [%message (variables : Template.syntax list)];
  [%expect {|
    (variables (((variable "") (pattern :[~x]) (offset 1) (kind Value)))) |}]

 let%expect_test "interpret_incomplete_hole_as_constant" =
  let template = ":[B :[A]" in
-  parse_template Matchers.Metasyntax.default_metasyntax template |> print_string;
+  parse_template Metasyntax.default_metasyntax template |> print_string;
  [%expect_exact {|((Constant ":[B ")
 (Hole ((variable A) (pattern :[A]) (offset 4) (kind Value))))|}];

  let template = ":[B :[A~x]" in
-  parse_template Matchers.Metasyntax.default_metasyntax template |> print_string;
+  parse_template Metasyntax.default_metasyntax template |> print_string;
  [%expect_exact {|((Constant ":[B ")
 (Hole ((variable A) (pattern :[A~x]) (offset 4) (kind Value))))|}]

 let%expect_test "interpret_incomplete_hole_as_constant_metasyntax" =
  let template = "$:x $B:x  $A" in
  let metasyntax =
-    Matchers.Metasyntax.{
+    Metasyntax.{
      syntax =
        [ Hole (Everything, Delimited (Some "$", None))
        ; Hole (Alphanum, Delimited (Some "$$", None))
@ -54,7 +55,7 @@ let%expect_test "interpret_incomplete_hole_as_constant_metasyntax" =
 let%expect_test "interpret_incomplete_hole_as_constant_metasyntax" =
  let template = "(  , ,  )" in
  let metasyntax =
-    Matchers.Metasyntax.{
+    Metasyntax.{
      syntax =
        [ Hole (Everything, Delimited (Some "NOTHING", None))
        ; Hole (Everything, Reserved_identifiers ["  "; " "])
@ -73,7 +74,7 @@ let%expect_test "interpret_incomplete_hole_as_constant_metasyntax" =
 let%expect_test "interpret_incomplete_hole_as_constant_metasyntax" =
  let template = "(..,.)" in
  let metasyntax =
-    Matchers.Metasyntax.{
+    Metasyntax.{
      syntax =
        [ Hole (Everything, Delimited (Some "NOTHING", None))
        ; Hole (Everything, Reserved_identifiers [".."; "."])
@ -90,7 +91,7 @@ let%expect_test "interpret_incomplete_hole_as_constant_metasyntax" =
 let%expect_test "parse_reserved_identifiers_as_holes" =
  let template = "(α)" in
  let metasyntax =
-    Matchers.Metasyntax.{
+    Metasyntax.{
      syntax =
        [ Hole (Expression, Reserved_identifiers ["α"])
        ]
@ -103,12 +104,11 @@ let%expect_test "parse_reserved_identifiers_as_holes" =
 (Hole ((variable "\206\177") (pattern "\206\177") (offset 1) (kind Value)))
 (Constant ")"))|}]

-
 let%expect_test "get_offsets_for_holes" =
-  let module Template_parser = Matchers.Template.Make(Matchers.Metasyntax.Default) in
+  let module Template_parser = Template.Make(Metasyntax.Default)(External.Default) in
  let template = ":[a].type :[b].length :[[c]].lengtha :[d.].length.ok (:[e].length)" in
  let variables = Template_parser.variables template in
-  print_s [%message (variables : Matchers.Template.syntax list)];
+  print_s [%message (variables : Template.syntax list)];
  [%expect {|
    (variables
     (((variable a) (pattern :[a]) (offset 0) (kind Value))
--- a/test/common/test_rewrite_parts.ml
+++ b/test/common/test_rewrite_parts.ml
@ -7,8 +7,6 @@ open Matchers
 let all ?(configuration = configuration) template source =
  Alpha.C.all ~configuration ~template ~source ()

-module Template_parser = Template.Make(Metasyntax.Default)
-
 let%expect_test "comments_in_string_literals_should_not_be_treated_as_comments_by_fuzzy" =
  let source = {|123433312343331122|} in
  let match_template = {|1234:[1]1234:[3]11:[2]|} in
--- a/test/common/test_script.ml
+++ b/test/common/test_script.ml
@ -1,5 +1,7 @@
 open Matchers

+module Script = Script.Make (Metasyntax.Default) (External.Default)
+
 let run input =
  let result =
    match Script.parse input with