diff --git a/lib/app/configuration/command_configuration.ml b/lib/app/configuration/command_configuration.ml index 6619408..4d6f607 100644 --- a/lib/app/configuration/command_configuration.ml +++ b/lib/app/configuration/command_configuration.ml @@ -104,15 +104,15 @@ let read filename = String.chop_suffix template ~suffix:"\n" |> Option.value ~default:template -let create_rule rule = - match Option.map rule ~f:Matchers.Rule.create with +let create_rule ~metasyntax rule = + match Option.map rule ~f:(Matchers.Rule.create ?metasyntax) with | None -> None | Some Ok rule -> Some rule | Some Error error -> Format.eprintf "Rule parse error: %s@." (Error.to_string_hum error); exit 1 -let parse_toml path = +let parse_toml ?metasyntax path = let open Toml.Types in let toml = Toml.Parser.(from_filename path |> unsafe) in let toml = Table.remove (Toml.Min.key "flags") toml in @@ -135,7 +135,7 @@ let parse_toml path = Format.eprintf "A 'match' key is required for entry %s@." name; exit 1 in - let rule = Table.find_opt (Toml.Min.key "rule") t |> to_string |> create_rule in + let rule = Table.find_opt (Toml.Min.key "rule") t |> to_string |> create_rule ~metasyntax in let rewrite_template = Table.find_opt (Toml.Min.key "rewrite") t |> to_string in if debug then Format.printf "Processed ->%s<-@." match_template; (name, (Matchers.Specification.create ~match_template ?rule ?rewrite_template ()))::acc @@ -147,7 +147,7 @@ let parse_toml path = |> List.sort ~compare:(fun x y -> String.compare (fst x) (fst y)) |> List.map ~f:snd -let parse_templates ?(warn_for_missing_file_in_dir = false) paths = +let parse_templates ?metasyntax ?(warn_for_missing_file_in_dir = false) paths = let parse_directory path = let read_optional filename = match read filename with @@ -159,7 +159,7 @@ let parse_templates ?(warn_for_missing_file_in_dir = false) paths = if warn_for_missing_file_in_dir then Format.eprintf "WARNING: Could not read required match file in %s@." path; None | Some match_template -> - let rule = create_rule @@ read_optional (path ^/ "rule") in + let rule = create_rule ~metasyntax @@ read_optional (path ^/ "rule") in let rewrite_template = read_optional (path ^/ "rewrite") in Matchers.Specification.create ~match_template ?rule ?rewrite_template () |> Option.some @@ -181,7 +181,7 @@ let parse_templates ?(warn_for_missing_file_in_dir = false) paths = if Sys.is_directory path = `Yes then fold_directory path ~sorted:true ~init:[] ~f else - parse_toml path) + parse_toml ?metasyntax path) type interactive_review = { editor : string @@ -415,6 +415,23 @@ type t = ; substitute_in_place : bool } +let parse_metasyntax metasyntax_path = + match metasyntax_path with + | None -> Matchers.Metasyntax.default_metasyntax + | Some metasyntax_path -> + match Sys.file_exists metasyntax_path with + | `No | `Unknown -> + Format.eprintf "Could not open file: %s@." metasyntax_path; + exit 1 + | `Yes -> + Yojson.Safe.from_file metasyntax_path + |> Matchers.Metasyntax.of_yojson + |> function + | Ok c -> c + | Error error -> + Format.eprintf "%s@." error; + exit 1 + let emit_errors { input_options; output_options; _ } = let error_on = [ input_options.stdin && Option.is_some input_options.zip_file @@ -477,7 +494,7 @@ let emit_errors { input_options; output_options; _ } = Option.value_exn message else "UNREACHABLE") - ; (let result = Matchers.Rule.create input_options.rule in + ; (let result = Matchers.Rule.create ~metasyntax:(parse_metasyntax input_options.custom_metasyntax) input_options.rule in Or_error.is_error result , if Or_error.is_error result then Format.sprintf "Match rule parse error: %s@." @@ @@ -513,7 +530,7 @@ let emit_warnings { input_options; output_options; _ } = | None, Some ({ match_template; _ } : anonymous_arguments) -> [ match_template ] | Some templates, _ -> - List.map (parse_templates templates) ~f:(fun { match_template; _ } -> match_template) + List.map (parse_templates ~metasyntax:(parse_metasyntax input_options.custom_metasyntax) templates) ~f:(fun { match_template; _ } -> match_template) | _ -> assert false in List.exists match_templates ~f:(fun match_template -> @@ -574,23 +591,6 @@ let filter_zip_entries file_filters exclude_directory_prefix exclude_file_prefix && not (exclude_the_file exclude_file_prefix (Filename.basename filename)) && has_acceptable_suffix filename) -let metasyntax metasyntax_path = - match metasyntax_path with - | None -> Matchers.Metasyntax.default_metasyntax - | Some metasyntax_path -> - match Sys.file_exists metasyntax_path with - | `No | `Unknown -> - Format.eprintf "Could not open file: %s@." metasyntax_path; - exit 1 - | `Yes -> - Yojson.Safe.from_file metasyntax_path - |> Matchers.Metasyntax.of_yojson - |> function - | Ok c -> c - | Error error -> - Format.eprintf "%s@." error; - exit 1 - let syntax custom_matcher_path = match Sys.file_exists custom_matcher_path with @@ -622,40 +622,47 @@ let extension file_filters = | _, Some extension -> "." ^ extension | extension, None -> "." ^ extension -let of_extension (module E : Matchers.Engine.S) file_filters = +let of_extension (module Engine : Matchers.Engine.S) (module External : Matchers.External.S) file_filters = + let external_handler = External.handler in let extension = extension file_filters in - match E.select_with_extension extension with + match Engine.select_with_extension extension ~external_handler with | Some matcher -> matcher, Some extension, None - | None -> (module E.Generic), Some extension, None + | None -> (module Engine.Generic), Some extension, None let select_matcher custom_metasyntax custom_matcher override_matcher file_filters omega = - let (module E : Matchers.Engine.S) = + let (module Engine : Matchers.Engine.S) = if omega then (module Matchers.Omega) else (module Matchers.Alpha) in + let module External = struct let handler = External_semantic.lsif_hover end in + if debug then Format.printf "Set custom external@."; match custom_matcher, override_matcher, custom_metasyntax with | Some custom_matcher, _, custom_metasyntax -> (* custom matcher, optional custom metasyntax *) - let metasyntax = metasyntax custom_metasyntax in + let metasyntax = parse_metasyntax custom_metasyntax in let syntax = syntax custom_matcher in - E.create ~metasyntax syntax, None, Some metasyntax + if debug then Format.printf "Engine.create@."; + Engine.create ~metasyntax syntax, None, Some metasyntax | _, Some language, custom_metasyntax -> (* forced language, optional custom metasyntax *) - let metasyntax = metasyntax custom_metasyntax in + let metasyntax = parse_metasyntax custom_metasyntax in let (module Metasyntax) = Matchers.Metasyntax.create metasyntax in let (module Language) = force_language language in - (module (E.Make (Language) (Metasyntax)) : Matchers.Matcher.S), None, Some metasyntax + if debug then Format.printf "Engine.Make@."; + (module (Engine.Make (Language) (Metasyntax) (External)) : Matchers.Matcher.S), None, Some metasyntax | _, _, Some custom_metasyntax -> (* infer language from file filters, definite custom metasyntax *) - let metasyntax = metasyntax (Some custom_metasyntax) in + let metasyntax = parse_metasyntax (Some custom_metasyntax) in let (module Metasyntax) = Matchers.Metasyntax.create metasyntax in let (module Language) = force_language (extension file_filters) in - (module (E.Make (Language) (Metasyntax)) : Matchers.Matcher.S), None, Some metasyntax + if debug then Format.printf "Engine.Make2@."; + (module (Engine.Make (Language) (Metasyntax) (External)) : Matchers.Matcher.S), None, Some metasyntax | _, _, None -> (* infer language from file filters, use default metasyntax *) - of_extension (module E) file_filters + if debug then Format.printf "Engine.Infer@."; + of_extension (module Engine) (module External) file_filters let regex_of_specifications specifications = Format.sprintf "(%s)" @@ -714,7 +721,7 @@ let create let open Or_error in emit_errors configuration >>= fun () -> emit_warnings configuration >>= fun () -> - let rule = Matchers.Rule.create rule |> Or_error.ok_exn in + let rule = Matchers.Rule.create ~metasyntax:(parse_metasyntax custom_metasyntax) rule |> Or_error.ok_exn in let specifications = match templates, anonymous_arguments with | None, Some { match_template; rewrite_template; _ } -> diff --git a/lib/app/configuration/dune b/lib/app/configuration/dune index 9a53300..c647706 100644 --- a/lib/app/configuration/dune +++ b/lib/app/configuration/dune @@ -3,4 +3,4 @@ (public_name comby.configuration) (instrumentation (backend bisect_ppx)) (preprocess (pps ppx_deriving.show ppx_sexp_conv ppx_sexp_message ppx_deriving_yojson)) - (libraries comby-kernel comby.patdiff comby.camlzip core yojson ppx_deriving_yojson toml lwt lwt.unix)) + (libraries comby-kernel comby-semantic comby.patdiff comby.camlzip core yojson ppx_deriving_yojson toml lwt lwt.unix)) diff --git a/lib/app/configuration/external_semantic.ml b/lib/app/configuration/external_semantic.ml new file mode 100644 index 0000000..bdc7051 --- /dev/null +++ b/lib/app/configuration/external_semantic.ml @@ -0,0 +1,21 @@ +open Core_kernel + +open Comby_semantic + +let debug = + match Sys.getenv "DEBUG_COMBY" with + | exception Not_found -> false + | _ -> true + +let lsif_hover ~name:_ ~filepath ~line ~column = + String.chop_prefix_if_exists filepath ~prefix:(Sys.getcwd ()) |> fun filepath_relative_root -> + if debug then Format.printf "File relative root: %s@." filepath; + if debug then Format.printf "Querying type at %d::%d@." line column; + let context = + Lsif.Context. + { repository = "github.com/sourcegraph/sourcegraph" + ; lsif_endpoint = "https://sourcegraph.com/.api/graphql" + ; formatting = Markdown ("```go", "```") (* Expose. *) + } + in + Lsif.hover_at context ~filepath:filepath_relative_root ~line ~column diff --git a/lib/app/pipeline/pipeline.ml b/lib/app/pipeline/pipeline.ml index 0d2cab5..f68e2d1 100644 --- a/lib/app/pipeline/pipeline.ml +++ b/lib/app/pipeline/pipeline.ml @@ -85,8 +85,10 @@ let process_single_source (* If there are no matches, return the original source (for editor support). *) Replacement ([], input_text, 0) | matches -> + (* FIXME this should be configured where it's done in command_configuration.ml *) + let external_handler = External_semantic.lsif_hover in let source = if substitute_in_place then Some input_text else None in - match Rewrite.all ?source ?metasyntax ?fresh ?filepath ~rewrite_template matches with + match Rewrite.all ?source ?metasyntax ?fresh ?filepath ~external_handler ~rewrite_template matches with | None -> Nothing | Some { rewritten_source; in_place_substitutions } -> Replacement (in_place_substitutions, rewritten_source, List.length matches) diff --git a/lib/kernel/comby_kernel.ml b/lib/kernel/comby_kernel.ml index 5e925ba..d01b85c 100644 --- a/lib/kernel/comby_kernel.ml +++ b/lib/kernel/comby_kernel.ml @@ -19,6 +19,8 @@ module Matchers = struct module Metasyntax = Matchers.Metasyntax type metasyntax = Matchers.Metasyntax.t + module External = Matchers.External + module Alpha = Matchers.Alpha module Omega = Matchers.Omega @@ -30,7 +32,6 @@ module Matchers = struct module Rule = struct include Matchers.Rule - include Matchers.Rule.Parser include Matchers.Evaluate end type rule = Rule.t diff --git a/lib/kernel/comby_kernel.mli b/lib/kernel/comby_kernel.mli index 928f099..a815475 100644 --- a/lib/kernel/comby_kernel.mli +++ b/lib/kernel/comby_kernel.mli @@ -335,6 +335,17 @@ module Matchers : sig type metasyntax = Metasyntax.t + module External : sig + type t = name:string -> filepath:string -> line:int -> column:int -> string option + + module type S = sig + val handler : t + end + + (** A module representing the default external handler *) + module Default : S + end + (** {3 Template} Parse a template based on metasynax *) @@ -350,7 +361,6 @@ module Matchers : sig | LineEnd | ColumnStart | ColumnEnd - | LsifHover | FileName | FilePath | FileDirectory @@ -362,6 +372,7 @@ module Matchers : sig | LowerCamelCase | UpperSnakeCase | LowerSnakeCase + | External of string type syntax = { variable : string @@ -379,7 +390,7 @@ module Matchers : sig type t = atom list [@@deriving sexp] - module Make : Metasyntax.S -> sig + module Make : Metasyntax.S -> External.S -> sig val parse : string -> t val variables : string -> syntax list end @@ -460,7 +471,11 @@ module Matchers : sig { nested : bool } - val create : string -> t Core_kernel.Or_error.t + val create + : ?metasyntax:metasyntax + -> ?external_handler:External.t + -> string + -> t Core_kernel.Or_error.t val options : t -> options @@ -476,20 +491,21 @@ module Matchers : sig [substitute_in_place] is true, rewrite rules substitute their values in place (default true). [fresh] introduces fresh variables for evaluating rules. [metasyntax] uses the custom metasyntax definition. *) - val apply : - ?substitute_in_place:bool -> - ?metasyntax:Metasyntax.t -> - ?filepath:string -> - match_all:( - ?configuration:Configuration.t -> - ?filepath:string -> - template:string -> - source:string -> - unit -> - Match.t list - ) -> - Ast.expression list -> - Match.Environment.t -> result + val apply + : ?substitute_in_place:bool + -> ?metasyntax:Metasyntax.t + -> ?external_handler:External.t + -> ?filepath:string + -> match_all:( + ?configuration:Configuration.t + -> ?filepath:string + -> template:string + -> source:string + -> unit + -> Match.t list) + -> Ast.expression list + -> Match.Environment.t + -> result end type rule = Rule.t @@ -651,7 +667,7 @@ module Matchers : sig module Engine : sig module type S = sig - module Make : Language.S -> Metasyntax.S -> Matcher.S + module Make : Language.S -> Metasyntax.S -> External.S -> Matcher.S (** {4 Supported Matchers} *) module Text : Matcher.S @@ -710,18 +726,27 @@ module Matchers : sig (** [all] returns all default matchers. *) val all : (module Matcher.S) list - (** [select_with_extension metasyntax file_extension] is a convenience - function that returns a matcher associated with a [file_extension]. E.g., - use ".c" to get the C matcher. For a full list of extensions associated - with matchers, run comby -list. If [metasyntax] is specified, the matcher - will use a custom metasyntax definition instead of the default. *) - val select_with_extension : ?metasyntax:Metasyntax.t -> string -> (module Matcher.S) option + (** [select_with_extension metasyntax external file_extension] is a + convenience function that returns a matcher associated with a + [file_extension]. E.g., use ".c" to get the C matcher. For a full list + of extensions associated with matchers, run comby -list. If + [metasyntax] is specified, the matcher will use a custom metasyntax + definition instead of the default. An experimental [external] callback + is a general callback for handling external properties in the rewrite + template. *) + val select_with_extension + : ?metasyntax:Metasyntax.t + -> ?external_handler:External.t + -> string + -> (module Matcher.S) option - (** [create metasyntax syntax] creates a matcher for a language defined by - [syntax]. If [metasyntax] is specified, the matcher will use a custom - metasyntax definition instead of the default. *) - val create : ?metasyntax:Metasyntax.t -> Language.Syntax.t -> (module Matcher.S) + (** [create metasyntax external syntax] creates a matcher for a language + defined by [syntax]. If [metasyntax] is specified, the matcher will use + a custom metasyntax definition instead of the default. An experimental + [external] callback is a general callback for handling external + properties in the rewrite template. *) + val create : ?metasyntax:metasyntax -> ?external_handler:External.t -> Language.Syntax.t -> (module Matcher.S) end end @@ -741,7 +766,7 @@ module Matchers : sig Defines rewrite operations. *) module Rewrite : sig - (** [all source metasyntax fresh rewrite_template matches] substitutes + (** [all source metasyntax external fresh rewrite_template matches] substitutes [rewrite_template] with each match in [matches] to create a rewrite result. If [source] is specified, each rewrite result is substituted in-place in the source. If [source] is not specified, rewritten matches are @@ -755,13 +780,14 @@ module Matchers : sig val all : ?source:string -> ?metasyntax:metasyntax + -> ?external_handler:External.t -> ?fresh:(unit -> string) -> ?filepath:string -> rewrite_template:string -> match' list -> replacement option - (** [substitute metasyntax fresh template environment] substitutes + (** [substitute metasyntax external fresh template environment] substitutes [template] with the variable and value pairs in the [environment]. It returns the result after substitution. If [metasyntax] is defined, the rewrite template will respect custom metasyntax definitions. @@ -770,9 +796,12 @@ module Matchers : sig specified, the default behavior substitutes :[id()] starting with 1, and subsequent :[id()] values increment the ID. If [fresh] is set, substitutes the pattern :[id()] with the value of fresh () as the hole is - encountered, left to right. *) + encountered, left to right. An experimental [external] callback is a + general callback for handling external properties in the rewrite + template. *) val substitute : ?metasyntax:metasyntax + -> ?external_handler:External.t -> ?fresh:(unit -> string) -> ?filepath:string -> string diff --git a/lib/kernel/matchers/alpha.ml b/lib/kernel/matchers/alpha.ml index 1c06624..2f567d7 100644 --- a/lib/kernel/matchers/alpha.ml +++ b/lib/kernel/matchers/alpha.ml @@ -45,7 +45,7 @@ let is_not p s = type 'a literal_parser_callback = contents:string -> left_delimiter:string -> right_delimiter:string -> 'a type 'a nested_delimiter_callback = left_delimiter:string -> right_delimiter:string -> 'a -module Make (Lang : Types.Language.S) (Meta : Metasyntax.S) = struct +module Make (Lang : Types.Language.S) (Meta : Types.Metasyntax.S) (Ext : Types.External.S) = struct module rec Matcher : Types.Matcher.S = struct module Syntax = Lang.Syntax include Lang.Info @@ -961,7 +961,13 @@ module Make (Lang : Types.Language.S) (Meta : Metasyntax.S) = struct let all ?configuration ?filepath ?(rule = [Types.Ast.True]) ~template ~source:original_source () : Match.t list = let _ : string option = filepath in let Rule.{ nested } = Rule.options rule in - let template, rule = Preprocess.map_aliases template (Some rule) Meta.aliases in + let template, rule = + Preprocess.map_aliases + (module Meta) + (module Ext) + template + (Some rule) + in let rule = Option.value_exn rule in (* OK in this case *) let rec aux_all ?configuration ?(nested = false) ~template ~source:original_source () = let open Or_error in @@ -994,8 +1000,24 @@ module Make (Lang : Types.Language.S) (Meta : Metasyntax.S) = struct let result = { result with matched } in let result = if debug then Format.printf "Rule: %s@." (Sexp.to_string @@ Rule.sexp_of_t rule); - (* FIXME metasyntax should propagate *) - let sat, env = Program.apply ~metasyntax:Metasyntax.default_metasyntax ~substitute_in_place:true ?filepath rule environment in + (* FIXME we should not have to convert here. Pass module, but after fixing this functor's signature. *) + let metasyntax = + Metasyntax. + { syntax = Meta.syntax + ; identifier = Meta.identifier + ; aliases = Meta.aliases + } + in + let external_handler = Ext.handler in + let sat, env = + Program.apply + ~metasyntax + ~external_handler + ~substitute_in_place:true + ?filepath + rule + environment + in if debug && Option.is_some env then Format.printf "Got back: %b %S" sat (Match.Environment.to_string @@ Option.value_exn env); let new_env = if sat then env else None in match new_env with @@ -1090,6 +1112,7 @@ module Make (Lang : Types.Language.S) (Meta : Metasyntax.S) = struct val apply : ?substitute_in_place:bool -> ?metasyntax:Types.Metasyntax.t + -> ?external_handler:Types.External.t -> ?filepath:string -> Rule.t -> Match.environment @@ -1099,12 +1122,14 @@ module Make (Lang : Types.Language.S) (Meta : Metasyntax.S) = struct let apply ?(substitute_in_place = true) ?metasyntax + ?external_handler ?filepath rule env = Evaluate.apply ~substitute_in_place ?metasyntax + ?external_handler ?filepath ~match_all:(Matcher.all ~rule:[Types.Ast.True]) rule diff --git a/lib/kernel/matchers/alpha.mli b/lib/kernel/matchers/alpha.mli index fd25ffd..11fa292 100644 --- a/lib/kernel/matchers/alpha.mli +++ b/lib/kernel/matchers/alpha.mli @@ -1,3 +1,3 @@ open Types -module Make : Language.S -> Metasyntax.S -> Matcher.S +module Make : Language.S -> Metasyntax.S -> External.S -> Matcher.S diff --git a/lib/kernel/matchers/dune b/lib/kernel/matchers/dune index 6547c67..068e826 100644 --- a/lib/kernel/matchers/dune +++ b/lib/kernel/matchers/dune @@ -3,4 +3,4 @@ (public_name comby-kernel.matchers) (instrumentation (backend bisect_ppx)) (preprocess (pps ppx_here ppx_sexp_conv ppx_sexp_message ppx_deriving_yojson)) - (libraries comby-semantic comby-kernel.replacement comby-kernel.parsers comby-kernel.match comby.vangstrom core_kernel mparser mparser-pcre re yojson ppx_deriving_yojson)) + (libraries comby-kernel.replacement comby-kernel.parsers comby-kernel.match comby.vangstrom core_kernel mparser mparser-pcre re yojson ppx_deriving_yojson)) diff --git a/lib/kernel/matchers/engine.ml b/lib/kernel/matchers/engine.ml index c54c40b..e7aa406 100644 --- a/lib/kernel/matchers/engine.ml +++ b/lib/kernel/matchers/engine.ml @@ -2,11 +2,12 @@ open Core_kernel open Languages -module Make (Make : Types.Language.S -> Types.Metasyntax.S -> Types.Matcher.S) : Types.Engine.S = struct +module Make (Make : Types.Language.S -> Types.Metasyntax.S -> Types.External.S -> Types.Matcher.S) : Types.Engine.S = struct module Make = Make let create ?(metasyntax = Metasyntax.default_metasyntax) + ?(external_handler = External.default_external) Types.Language.Syntax. { user_defined_delimiters ; escapable_string_literals @@ -31,60 +32,61 @@ module Make (Make : Types.Language.S -> Types.Metasyntax.S -> Types.Matcher.S) : end in let (module Metasyntax : Metasyntax.S) = Metasyntax.(create metasyntax) in - (module Make (User_language) (Metasyntax) : Types.Matcher.S) + let module External = struct let handler = external_handler end in + (module Make (User_language) (Metasyntax) (External) : Types.Matcher.S) - module Text = Make (Text) (Metasyntax.Default) - module Paren = Make (Paren) (Metasyntax.Default) - module Dyck = Make (Dyck) (Metasyntax.Default) - module JSON = Make (JSON) (Metasyntax.Default) - module JSONC = Make (JSONC) (Metasyntax.Default) - module GraphQL = Make (GraphQL) (Metasyntax.Default) - module Dhall = Make (Dhall) (Metasyntax.Default) - module Latex = Make (Latex) (Metasyntax.Default) - module Assembly = Make (Assembly) (Metasyntax.Default) - module Clojure = Make (Clojure) (Metasyntax.Default) - module Lisp = Make (Lisp) (Metasyntax.Default) - module Generic = Make (Generic) (Metasyntax.Default) - module Bash = Make (Bash) (Metasyntax.Default) - module Ruby = Make (Ruby) (Metasyntax.Default) - module Elixir = Make (Elixir) (Metasyntax.Default) - module Python = Make (Python) (Metasyntax.Default) - module Html = Make (Html) (Metasyntax.Default) - module Xml = Make (Xml) (Metasyntax.Default) - module SQL = Make (SQL) (Metasyntax.Default) - module Erlang = Make (Erlang) (Metasyntax.Default) - module C = Make (C) (Metasyntax.Default) - module Csharp = Make (Csharp) (Metasyntax.Default) - module Java = Make (Java) (Metasyntax.Default) - module CSS = Make (CSS) (Metasyntax.Default) - module Kotlin = Make (Kotlin) (Metasyntax.Default) - module Scala = Make (Scala) (Metasyntax.Default) - module Nim = Make (Nim) (Metasyntax.Default) - module Matlab = Make (Matlab) (Metasyntax.Default) - module Dart = Make (Dart) (Metasyntax.Default) - module Php = Make (Php) (Metasyntax.Default) - module Go = Make (Go) (Metasyntax.Default) - module Javascript = Make (Javascript) (Metasyntax.Default) - module Jsx = Make (Jsx) (Metasyntax.Default) - module Typescript = Make (Typescript) (Metasyntax.Default) - module Tsx = Make (Tsx) (Metasyntax.Default) - module Swift = Make (Swift) (Metasyntax.Default) - module Rust = Make (Rust) (Metasyntax.Default) - module R = Make (R) (Metasyntax.Default) - module OCaml = Make (OCaml) (Metasyntax.Default) - module Reason = Make (Reason) (Metasyntax.Default) - module Fsharp = Make (Fsharp) (Metasyntax.Default) - module Pascal = Make (Pascal) (Metasyntax.Default) - module Julia = Make (Julia) (Metasyntax.Default) - module Fortran = Make (Fortran) (Metasyntax.Default) - module Haskell = Make (Haskell) (Metasyntax.Default) - module HCL = Make (HCL) (Metasyntax.Default) - module Elm = Make (Elm) (Metasyntax.Default) - module Zig = Make (Zig) (Metasyntax.Default) - module Coq = Make (Coq) (Metasyntax.Default) - module Move = Make (Move) (Metasyntax.Default) - module Solidity = Make (Solidity) (Metasyntax.Default) - module C_nested_comments = Make (C_nested_comments) (Metasyntax.Default) + module Text = Make (Text) (Metasyntax.Default) (External.Default) + module Paren = Make (Paren) (Metasyntax.Default) (External.Default) + module Dyck = Make (Dyck) (Metasyntax.Default) (External.Default) + module JSON = Make (JSON) (Metasyntax.Default) (External.Default) + module JSONC = Make (JSONC) (Metasyntax.Default) (External.Default) + module GraphQL = Make (GraphQL) (Metasyntax.Default) (External.Default) + module Dhall = Make (Dhall) (Metasyntax.Default) (External.Default) + module Latex = Make (Latex) (Metasyntax.Default) (External.Default) + module Assembly = Make (Assembly) (Metasyntax.Default) (External.Default) + module Clojure = Make (Clojure) (Metasyntax.Default) (External.Default) + module Lisp = Make (Lisp) (Metasyntax.Default) (External.Default) + module Generic = Make (Generic) (Metasyntax.Default) (External.Default) + module Bash = Make (Bash) (Metasyntax.Default) (External.Default) + module Ruby = Make (Ruby) (Metasyntax.Default) (External.Default) + module Elixir = Make (Elixir) (Metasyntax.Default) (External.Default) + module Python = Make (Python) (Metasyntax.Default) (External.Default) + module Html = Make (Html) (Metasyntax.Default) (External.Default) + module Xml = Make (Xml) (Metasyntax.Default) (External.Default) + module SQL = Make (SQL) (Metasyntax.Default) (External.Default) + module Erlang = Make (Erlang) (Metasyntax.Default) (External.Default) + module C = Make (C) (Metasyntax.Default) (External.Default) + module Csharp = Make (Csharp) (Metasyntax.Default) (External.Default) + module Java = Make (Java) (Metasyntax.Default) (External.Default) + module CSS = Make (CSS) (Metasyntax.Default) (External.Default) + module Kotlin = Make (Kotlin) (Metasyntax.Default) (External.Default) + module Scala = Make (Scala) (Metasyntax.Default) (External.Default) + module Nim = Make (Nim) (Metasyntax.Default) (External.Default) + module Matlab = Make (Matlab) (Metasyntax.Default) (External.Default) + module Dart = Make (Dart) (Metasyntax.Default) (External.Default) + module Php = Make (Php) (Metasyntax.Default) (External.Default) + module Go = Make (Go) (Metasyntax.Default) (External.Default) + module Javascript = Make (Javascript) (Metasyntax.Default) (External.Default) + module Jsx = Make (Jsx) (Metasyntax.Default) (External.Default) + module Typescript = Make (Typescript) (Metasyntax.Default) (External.Default) + module Tsx = Make (Tsx) (Metasyntax.Default) (External.Default) + module Swift = Make (Swift) (Metasyntax.Default) (External.Default) + module Rust = Make (Rust) (Metasyntax.Default) (External.Default) + module R = Make (R) (Metasyntax.Default) (External.Default) + module OCaml = Make (OCaml) (Metasyntax.Default) (External.Default) + module Reason = Make (Reason) (Metasyntax.Default) (External.Default) + module Fsharp = Make (Fsharp) (Metasyntax.Default) (External.Default) + module Pascal = Make (Pascal) (Metasyntax.Default) (External.Default) + module Julia = Make (Julia) (Metasyntax.Default) (External.Default) + module Fortran = Make (Fortran) (Metasyntax.Default) (External.Default) + module Haskell = Make (Haskell) (Metasyntax.Default) (External.Default) + module HCL = Make (HCL) (Metasyntax.Default) (External.Default) + module Elm = Make (Elm) (Metasyntax.Default) (External.Default) + module Zig = Make (Zig) (Metasyntax.Default) (External.Default) + module Coq = Make (Coq) (Metasyntax.Default) (External.Default) + module Move = Make (Move) (Metasyntax.Default) (External.Default) + module Solidity = Make (Solidity) (Metasyntax.Default) (External.Default) + module C_nested_comments = Make (C_nested_comments) (Metasyntax.Default) (External.Default) let all : (module Types.Matcher.S) list = [ (module Assembly) @@ -140,9 +142,14 @@ module Make (Make : Types.Language.S -> Types.Metasyntax.S -> Types.Matcher.S) : ; (module Generic) ] - let select_with_extension ?(metasyntax = Metasyntax.default_metasyntax) extension : (module Types.Matcher.S) option = + let select_with_extension + ?(metasyntax = Metasyntax.default_metasyntax) + ?(external_handler = External.default_external) + extension + : (module Types.Matcher.S) option = let open Option in Languages.select_with_extension extension >>| fun (module Language : Types.Language.S) -> let (module Metasyntax) = Metasyntax.(create metasyntax) in - (module (Make (Language) (Metasyntax)) : Types.Matcher.S) + let module External = struct let handler = external_handler end in + (module (Make (Language) (Metasyntax) (External)) : Types.Matcher.S) end diff --git a/lib/kernel/matchers/engine.mli b/lib/kernel/matchers/engine.mli index b679e63..5f9fd5a 100644 --- a/lib/kernel/matchers/engine.mli +++ b/lib/kernel/matchers/engine.mli @@ -1,3 +1,3 @@ open Types -module Make : (Language.S -> Metasyntax.S -> Matcher.S) -> Engine.S +module Make : (Language.S -> Metasyntax.S -> External.S -> Matcher.S) -> Engine.S diff --git a/lib/kernel/matchers/evaluate.ml b/lib/kernel/matchers/evaluate.ml index bc2bf57..9b3787b 100644 --- a/lib/kernel/matchers/evaluate.ml +++ b/lib/kernel/matchers/evaluate.ml @@ -14,35 +14,38 @@ let sat = fst let result_env = snd -let match_configuration_of_syntax template = - (* decide match configuration based on whether there are holes *) - let antecedent_contains_hole_syntax case = - String.is_substring case ~substring:Syntax.variable_left_delimiter - in - if antecedent_contains_hole_syntax template then - Configuration.create ~match_kind:Fuzzy () - else - Configuration.create ~match_kind:Exact () - let merge_match_environments matches environment' = List.map matches ~f:(fun { environment; _ } -> Environment.merge environment environment') -(* FIXME. Propagate this. *) -module Template = Template.Make(Metasyntax.Default) - -let substitute ?filepath env v = - match v with - | Template t -> Rewrite.substitute ?filepath (Template.to_string t) env - | String s -> s - let apply ?(substitute_in_place = true) - ?metasyntax + ?(metasyntax = Metasyntax.default_metasyntax) + ?(external_handler = External.default_external) ?filepath ~(match_all:(?configuration:Configuration.t -> ?filepath:string -> template:string -> source:string -> unit -> Match.t list)) predicates env = + let (module Metasyntax) = Metasyntax.create metasyntax in + let module External = struct let handler = external_handler end in + let (module Template : Types.Template.S) = (module (Template.Make(Metasyntax)(External))) in + + let match_configuration_of_syntax template = + (* decide match configuration based on whether there are holes *) + match Template.variables template with + | [] -> Configuration.create ~match_kind:Exact () + | _ -> Configuration.create ~match_kind:Fuzzy () + in + + let rewrite_substitute template env = + Rewrite.substitute ~metasyntax ~external_handler ?filepath template env in + + let substitute env v = + match v with + | Template t -> + rewrite_substitute (Template.to_string t) env + | String s -> s + in (* accepts only one expression *) let rec eval env = @@ -56,15 +59,15 @@ let apply (* ==, != *) | Equal (Template t, String value) | Equal (String value, Template t) -> - let other = Rewrite.substitute ?filepath (Template.to_string t) env in + let other = rewrite_substitute (Template.to_string t) env in let result = String.equal value other in result, Some env | Equal (String left, String right) -> let result = String.equal left right in result, Some env | Equal (Template left, Template right) -> - let left = Rewrite.substitute ?filepath (Template.to_string left) env in - let right = Rewrite.substitute ?filepath (Template.to_string right) env in + let left = rewrite_substitute (Template.to_string left) env in + let right = rewrite_substitute (Template.to_string right) env in let result = String.equal left right in result, Some env | Not_equal (left, right) -> @@ -73,9 +76,9 @@ let apply (* match ... { ... } *) | Match (source, cases) -> - let source = substitute ?filepath env source in + let source = substitute env source in let evaluate template case_expression = - let template = substitute ?filepath env template in + let template = substitute env template in let configuration = match_configuration_of_syntax template in if debug then Format.printf "Running for template %s source %s@." template source; match_all ~configuration ~template ~source () |> function @@ -102,19 +105,19 @@ let apply (* rewrite ... { ... } *) | Rewrite (Template t, (match_template, rewrite_template)) -> - let rewrite_template = substitute ?filepath env rewrite_template in - let template = substitute ?filepath env match_template in - let source = Rewrite.substitute ?filepath (Template.to_string t) env in + let rewrite_template = substitute env rewrite_template in + let template = substitute env match_template in + let source = rewrite_substitute (Template.to_string t) env in let configuration = Configuration.create ~match_kind:Fuzzy () in let matches = match_all ~configuration ~template ~source () in let source = if substitute_in_place then Some source else None in - let result = Rewrite.all ?metasyntax ?source ~rewrite_template matches in + let result = Rewrite.all ~metasyntax ?source ~rewrite_template matches in if Option.is_empty result then true, Some env (* rewrites are always sat. *) else let Replacement.{ rewritten_source; _ } = Option.value_exn result in (* substitute for variables that are in the outside scope *) - let rewritten_source = Rewrite.substitute ?filepath ?metasyntax rewritten_source env in + let rewritten_source = rewrite_substitute rewritten_source env in let variable = match t with | [ Types.Template.Hole { variable; _ } ] -> variable diff --git a/lib/kernel/matchers/external.ml b/lib/kernel/matchers/external.ml new file mode 100644 index 0000000..0455091 --- /dev/null +++ b/lib/kernel/matchers/external.ml @@ -0,0 +1,13 @@ +include Types.External + +let default_external = + fun ~name:_ ~filepath:_ ~line:_ ~column:_ -> None + +let default = + let module External = struct + let handler = default_external + end + in + (module External : Types.External.S) + +module Default = (val default) diff --git a/lib/kernel/matchers/external.mli b/lib/kernel/matchers/external.mli new file mode 100644 index 0000000..2ffcd51 --- /dev/null +++ b/lib/kernel/matchers/external.mli @@ -0,0 +1,9 @@ +open Types + +include module type of External + +val default_external : External.t + +val default : (module External.S) + +module Default : External.S diff --git a/lib/kernel/matchers/matchers.ml b/lib/kernel/matchers/matchers.ml index 8939676..87cd803 100644 --- a/lib/kernel/matchers/matchers.ml +++ b/lib/kernel/matchers/matchers.ml @@ -1,6 +1,7 @@ module Configuration = Configuration module Languages = Languages module Metasyntax = Metasyntax +module External = External module Rule = Rule module Ast = struct include Types.Ast diff --git a/lib/kernel/matchers/matchers.mli b/lib/kernel/matchers/matchers.mli index 2670db3..565605d 100644 --- a/lib/kernel/matchers/matchers.mli +++ b/lib/kernel/matchers/matchers.mli @@ -1,6 +1,7 @@ module Configuration = Configuration module Languages = Languages module Metasyntax = Metasyntax +module External = External module Rule = Rule (* Only need to expose Types.Ast. module type of to export sexp. *) diff --git a/lib/kernel/matchers/omega.ml b/lib/kernel/matchers/omega.ml index 28fe5cf..3cb4a20 100644 --- a/lib/kernel/matchers/omega.ml +++ b/lib/kernel/matchers/omega.ml @@ -46,11 +46,11 @@ let actual = Buffer.create 10 let rewrite_template = ref "" -module Make (Language : Types.Language.S) (Meta : Metasyntax.S) = struct +module Make (Language : Types.Language.S) (Meta : Metasyntax.S) (Ext : External.S) = struct module rec Matcher : Types.Matcher.S = struct include Language.Info - module Template = Template.Make(Meta) + module Template = Template.Make(Meta)(Ext) let wildcard = "_" @@ -164,10 +164,19 @@ module Make (Language : Types.Language.S) (Meta : Metasyntax.S) = struct | Some rule -> push_environment_ref := !current_environment_ref; push_implicit_equals_match_satisfied := !implicit_equals_match_satisfied; - (* FIXME Metasyntax should be propagated here. *) + (* FIXME we should not have to convert here. Pass module, but after fixing this functor's signature. *) + let metasyntax = + Metasyntax. + { syntax = Meta.syntax + ; identifier = Meta.identifier + ; aliases = Meta.aliases + } + in + let external_handler = Ext.handler in let sat, env = Program.apply - ~metasyntax:Metasyntax.default_metasyntax + ~metasyntax + ~external_handler ~substitute_in_place:true ?filepath:!filepath_ref rule @@ -1014,7 +1023,13 @@ module Make (Language : Types.Language.S) (Meta : Metasyntax.S) = struct push_matches_ref := !matches_ref; configuration_ref := Option.value configuration ~default:!configuration_ref; let Rule.{ nested } = Rule.options rule in - let template, rule = Preprocess.map_aliases template (Some rule) Meta.aliases in + let template, rule = + Preprocess.map_aliases + (module Meta) + (module Ext) + template + (Some rule) + in let rec aux_all ?configuration ?(nested = false) ~template ~source () = matches_ref := []; if String.is_empty template && String.is_empty source then [trivial] @@ -1101,6 +1116,7 @@ module Make (Language : Types.Language.S) (Meta : Metasyntax.S) = struct val apply : ?substitute_in_place:bool -> ?metasyntax:Metasyntax.t + -> ?external_handler:External.t -> ?filepath:string -> Rule.t -> Match.environment @@ -1109,12 +1125,14 @@ module Make (Language : Types.Language.S) (Meta : Metasyntax.S) = struct let apply ?(substitute_in_place = true) ?metasyntax + ?external_handler ?filepath rule env = Evaluate.apply ~substitute_in_place ?metasyntax + ?external_handler ?filepath ~match_all:(Matcher.all ~rule:[Types.Ast.True]) rule diff --git a/lib/kernel/matchers/omega.mli b/lib/kernel/matchers/omega.mli index fd25ffd..11fa292 100644 --- a/lib/kernel/matchers/omega.mli +++ b/lib/kernel/matchers/omega.mli @@ -1,3 +1,3 @@ open Types -module Make : Language.S -> Metasyntax.S -> Matcher.S +module Make : Language.S -> Metasyntax.S -> External.S -> Matcher.S diff --git a/lib/kernel/matchers/preprocess.ml b/lib/kernel/matchers/preprocess.ml index bf71b31..9fcc988 100644 --- a/lib/kernel/matchers/preprocess.ml +++ b/lib/kernel/matchers/preprocess.ml @@ -5,8 +5,13 @@ let debug = | exception Not_found -> false | _ -> true -let map_aliases template parent_rule aliases = - List.fold aliases +let map_aliases + (module Metasyntax : Metasyntax.S) + (module External : External.S) + template + parent_rule = + let module Parser = Rule.Make (Metasyntax) (External) in + List.fold Metasyntax.aliases ~init:(template, parent_rule) ~f:(fun (template, parent_rule) Types.Metasyntax.{ pattern; match_template; rule } -> let open Option in @@ -18,7 +23,7 @@ let map_aliases template parent_rule aliases = let rule' = let rule = rule - >>| Rule.Parser.create + >>| Parser.create >>| function | Ok rule -> rule | Error e -> failwith @@ "Could not parse rule for alias entry:"^(Error.to_string_hum e) diff --git a/lib/kernel/matchers/rewrite.ml b/lib/kernel/matchers/rewrite.ml index d20160d..470450c 100644 --- a/lib/kernel/matchers/rewrite.ml +++ b/lib/kernel/matchers/rewrite.ml @@ -75,12 +75,14 @@ let substitute_fresh let substitute_in_rewrite_template ?fresh + ?(external_handler = External.default_external) ?(metasyntax = Metasyntax.default_metasyntax) ?filepath template environment = let (module M) = Metasyntax.create metasyntax in - let module Template_parser = Template.Make(M) in + let module External = struct let handler = external_handler end in + let module Template_parser = Template.Make(M)(External) in (* FIXME factor out Template_parser *) let template = substitute_fresh ~metasyntax ?fresh template in let terms = Template_parser.parse template in let replacement_content, environment = Template_parser.substitute ?filepath terms environment in @@ -92,8 +94,15 @@ let substitute_in_rewrite_template } } -let substitute ?(metasyntax = Metasyntax.default_metasyntax) ?fresh ?filepath template env = - let { replacement_content; _ } = substitute_in_rewrite_template ?fresh ?filepath ~metasyntax template env +let substitute + ?(metasyntax = Metasyntax.default_metasyntax) + ?external_handler + ?fresh + ?filepath + template + env = + let { replacement_content; _ } = + substitute_in_rewrite_template ~metasyntax ?external_handler ?fresh ?filepath template env in replacement_content let substitute_matches (matches: Match.t list) source replacements = @@ -115,19 +124,20 @@ let substitute_matches (matches: Match.t list) source replacements = ; in_place_substitutions } -let all ?source ?metasyntax ?fresh ?filepath ~rewrite_template rev_matches : result option = +(* FIXME: all the functors help nothing if we end up calling this without parameterizing by metasyntax, etc. *) +let all ?source ?metasyntax ?external_handler ?fresh ?filepath ~rewrite_template rev_matches : result option = Option.some_if (not (List.is_empty rev_matches)) @@ match source with (* in-place substitution *) | Some source -> rev_matches - |> List.map ~f:(fun Match.{ environment; _ } -> substitute_in_rewrite_template ?filepath ?metasyntax ?fresh rewrite_template environment) + |> List.map ~f:(fun Match.{ environment; _ } -> substitute_in_rewrite_template ?filepath ?metasyntax ?external_handler ?fresh rewrite_template environment) |> substitute_matches rev_matches source (* no in place substitution, emit result separated by newlines *) | None -> let buf = Buffer.create 20 in List.iter rev_matches ~f:(fun m -> - substitute_in_rewrite_template ?metasyntax ?fresh rewrite_template m.environment + substitute_in_rewrite_template ?metasyntax ?external_handler ?fresh rewrite_template m.environment |> fun { replacement_content; _ } -> Buffer.add_string buf replacement_content; Buffer.add_char buf '\n'); diff --git a/lib/kernel/matchers/rewrite.mli b/lib/kernel/matchers/rewrite.mli index e387d0c..64535b3 100644 --- a/lib/kernel/matchers/rewrite.mli +++ b/lib/kernel/matchers/rewrite.mli @@ -7,13 +7,21 @@ open Match val substitute_fresh : ?metasyntax:Metasyntax.t -> ?fresh:(unit -> string) -> string -> string (** substitute returns the result of substituting env in template *) -val substitute : ?metasyntax:Metasyntax.t -> ?fresh:(unit -> string) -> ?filepath:string -> string -> Environment.t -> string +val substitute + : ?metasyntax:Metasyntax.t + -> ?external_handler:External.t + -> ?fresh:(unit -> string) + -> ?filepath:string + -> string + -> Environment.t + -> string (** if [source] is given, substitute in-place. If not, emit result separated by newlines *) val all : ?source:string -> ?metasyntax:Types.Metasyntax.t + -> ?external_handler:External.t -> ?fresh:(unit -> string) -> ?filepath:string -> rewrite_template:string diff --git a/lib/kernel/matchers/rule.ml b/lib/kernel/matchers/rule.ml index 8d6b63e..e1d48c4 100644 --- a/lib/kernel/matchers/rule.ml +++ b/lib/kernel/matchers/rule.ml @@ -3,9 +3,10 @@ open Vangstrom open Types.Ast -module Template = Template.Make(Metasyntax.Default) -module Parser = struct +module Make (Metasyntax : Types.Metasyntax.S) (External : Types.External.S) = struct + + module Template = Template.Make (Metasyntax) (External) let is_whitespace = function | ' ' | '\t' | '\r' | '\n' -> true @@ -31,12 +32,6 @@ module Parser = struct | [ Constant c ] -> String c | t -> Template t - let variable_parser = - lift3 (fun _ v _ -> String.of_char_list v) - (string Syntax.variable_left_delimiter) - (many (alphanum <|> char '_')) - (string Syntax.variable_right_delimiter) - (** Interpret escape sequences inside quotes *) let char_token_s = (char '\\' *> any_char >>| @@ -216,6 +211,15 @@ end type t = Types.Rule.t [@@deriving sexp] +let create + ?(metasyntax = Metasyntax.default_metasyntax) + ?(external_handler = External.default_external) + rule = + let (module Metasyntax) = Metasyntax.create metasyntax in + let module External = struct let handler = external_handler end in + let (module Rule : Types.Rule.S) = (module (Make (Metasyntax) (External))) in + Rule.create rule + type options = { nested : bool } diff --git a/lib/kernel/matchers/script.ml b/lib/kernel/matchers/script.ml index 3d25a1d..5153818 100644 --- a/lib/kernel/matchers/script.ml +++ b/lib/kernel/matchers/script.ml @@ -1,83 +1,87 @@ open Core_kernel open Vangstrom -open Rule -open Parser open Types.Ast -type spec = Specification.t -[@@deriving sexp] +module Make (Metasyntax : Types.Metasyntax.S) (External : Types.External.S) = struct -type op = - | And - | Or - | Not -[@@deriving sexp] + module Parser = Rule.Make (Metasyntax) (External) -type exp = - | Exp of op * exp list - | Spec of spec -[@@deriving sexp] + type spec = Specification.t + [@@deriving sexp] -type t = exp list -[@@deriving sexp] + type op = + | And + | Or + | Not + [@@deriving sexp] -let ignore p = - p *> return () + type exp = + | Exp of op * exp list + | Spec of spec + [@@deriving sexp] -let spaces = many @@ satisfy (function ' ' | '\n' | '\r' | '\t' -> true | _ -> false) -let spaces1 = many1 @@ satisfy (function ' ' | '\n' | '\r' | '\t' -> true | _ -> false) + type t = exp list + [@@deriving sexp] -let optional s = option () (ignore @@ string s) + let ignore p = + p *> return () -let chainl1 e op = - let rec parse acc = (lift2 (fun f x -> f acc x) op e >>= parse) <|> return acc in - e >>= fun init -> parse init + let spaces = many @@ satisfy (function ' ' | '\n' | '\r' | '\t' -> true | _ -> false) + let spaces1 = many1 @@ satisfy (function ' ' | '\n' | '\r' | '\t' -> true | _ -> false) -let parens p = char '(' *> (p <|> return []) <* char ')' + let optional s = option () (ignore @@ string s) + + let chainl1 e op = + let rec parse acc = (lift2 (fun f x -> f acc x) op e >>= parse) <|> return acc in + e >>= fun init -> parse init + + let parens p = char '(' *> (p <|> return []) <* char ')' -let template_parser until = - choice - [ (lift to_atom quoted_parser) - ; (lift (fun v -> to_atom (String.of_char_list v)) (Omega_parser_helper.many1_till any_char until)) - ] + let template_parser until = + choice + [ (lift Parser.to_atom Parser.quoted_parser) + ; (lift (fun v -> Parser.to_atom (String.of_char_list v)) (Omega_parser_helper.many1_till any_char until)) + ] -let spec = - let match_rewrite_parser = - both - (spaces *> template_parser (spaces *> string "->")) - (option None (spaces *> string Syntax.arrow *> spaces *> template_parser (spaces1 *> string "where" *> spaces1) >>| fun v -> Some v)) (* FIXME use of reserved *) - in - match_rewrite_parser >>= fun (match_template_atom, rewrite_template_atom) -> - (option None (spaces1 *> parse >>| fun x -> Some x)) >>= fun rule -> - let match_template = Sexplib.Sexp.to_string_hum (sexp_of_atom match_template_atom) in - let rewrite_template = - match rewrite_template_atom with - | Some rewrite_template_atom -> Some (Sexplib.Sexp.to_string_hum (sexp_of_atom rewrite_template_atom)) - | None -> None - in - return [(Spec (Specification.{ match_template; rule; rewrite_template }))] + let spec = + let match_rewrite_parser = + both + (spaces *> template_parser (spaces *> string "->")) + (option None (spaces *> string Syntax.arrow *> spaces *> template_parser (spaces1 *> string "where" *> spaces1) >>| fun v -> Some v)) (* FIXME use of reserved *) + in + match_rewrite_parser >>= fun (match_template_atom, rewrite_template_atom) -> + (option None (spaces1 *> Parser.parse >>| fun x -> Some x)) >>= fun rule -> + let match_template = Sexplib.Sexp.to_string_hum (sexp_of_atom match_template_atom) in + let rewrite_template = + match rewrite_template_atom with + | Some rewrite_template_atom -> Some (Sexplib.Sexp.to_string_hum (sexp_of_atom rewrite_template_atom)) + | None -> None + in + return [(Spec (Specification.{ match_template; rule; rewrite_template }))] -let unop syntax exp_parser = - choice (List.map ~f:string syntax) *> spaces *> exp_parser >>| fun exp -> [Exp (Not, exp)] + let unop syntax exp_parser = + choice (List.map ~f:string syntax) *> spaces *> exp_parser >>| fun exp -> [Exp (Not, exp)] -let binop syntax op = - spaces *> choice (List.map ~f:string syntax) *> spaces *> return (fun left right -> [Exp (op, left@right)]) + let binop syntax op = + spaces *> choice (List.map ~f:string syntax) *> spaces *> return (fun left right -> [Exp (op, left@right)]) -let exp_parser = - fix (fun exp -> - let exp_parser = fix (fun exp' -> parens exp <|> unop ["NOT"; "not"] exp' <|> spec) in - let and_parser = chainl1 exp_parser @@ binop ["AND"; "and"] And in - let seq_parser = chainl1 and_parser @@ binop ["OR"; "or"] Or in - sep_by1 (spaces *> string Syntax.separator <* spaces) seq_parser >>| List.concat) + let exp_parser = + fix (fun exp -> + let exp_parser = fix (fun exp' -> parens exp <|> unop ["NOT"; "not"] exp' <|> spec) in + let and_parser = chainl1 exp_parser @@ binop ["AND"; "and"] And in + let seq_parser = chainl1 and_parser @@ binop ["OR"; "or"] Or in + sep_by1 (spaces *> string Syntax.separator <* spaces) seq_parser >>| List.concat) -let parser = - spaces *> optional Syntax.separator *> - exp_parser <* optional Syntax.separator <* spaces <* end_of_input + let parser = + spaces *> optional Syntax.separator *> + exp_parser <* optional Syntax.separator <* spaces <* end_of_input -let parse script = - parse_string ~consume:All parser script + let parse script = + parse_string ~consume:All parser script -let to_string exp = - Sexplib.Sexp.to_string_hum (sexp_of_t exp) + let to_string exp = + Sexplib.Sexp.to_string_hum (sexp_of_t exp) + +end diff --git a/lib/kernel/matchers/specification.ml b/lib/kernel/matchers/specification.ml index a0a83c6..b8996f7 100644 --- a/lib/kernel/matchers/specification.ml +++ b/lib/kernel/matchers/specification.ml @@ -11,7 +11,6 @@ type t = let create ?rewrite_template ?rule ~match_template () = { match_template; rule; rewrite_template } - let identifier_parser () = many (Omega_parser_helper.alphanum <|> char '_') >>| String.of_char_list diff --git a/lib/kernel/matchers/syntax.ml b/lib/kernel/matchers/syntax.ml index cfa4f34..524943d 100644 --- a/lib/kernel/matchers/syntax.ml +++ b/lib/kernel/matchers/syntax.ml @@ -3,8 +3,6 @@ let start_match_pattern = "match" let start_rewrite_pattern = "rewrite" let equal = "==" let not_equal = "!=" -let variable_left_delimiter = ":[" -let variable_right_delimiter = "]" let true' = "true" let false' = "false" let option_nested = "nested" diff --git a/lib/kernel/matchers/template.ml b/lib/kernel/matchers/template.ml index 43e96fb..d6f9fbd 100644 --- a/lib/kernel/matchers/template.ml +++ b/lib/kernel/matchers/template.ml @@ -9,7 +9,7 @@ let debug = | exception Not_found -> false | _ -> true -module Make (Metasyntax : Types.Metasyntax.S) = struct +module Make (Metasyntax : Types.Metasyntax.S) (External : Types.External.S) : Types.Template.S = struct let up_to p = many1 (not_followed_by p *> any_char) @@ -88,7 +88,6 @@ module Make (Metasyntax : Types.Metasyntax.S) = struct | "column" | "column.start" -> ColumnStart | "column.end" -> ColumnEnd - | "lsif.hover" -> LsifHover | "file.path" -> FilePath | "file.name" -> FileName | "file.directory" -> FileDirectory @@ -100,6 +99,7 @@ module Make (Metasyntax : Types.Metasyntax.S) = struct | "lowerCamelCase" -> LowerCamelCase | "UPPER_SNAKE_CASE" -> UpperSnakeCase | "lower_snake_case" -> LowerSnakeCase + | "lsif.hover" -> External "lsif.hover" | s -> failwith @@ Format.sprintf "invalid attribute %S" s let attribute_access () = @@ -116,7 +116,6 @@ module Make (Metasyntax : Types.Metasyntax.S) = struct ; string "column.start" ; string "column.end" ; string "column" - ; string "lsif.hover" ; string "file.path" ; string "file.name" ; string "file.directory" @@ -128,6 +127,7 @@ module Make (Metasyntax : Types.Metasyntax.S) = struct ; string "lowerCamelCase" ; string "UPPER_SNAKE_CASE" ; string "lower_snake_case" + ; string "lsif.hover" ] <* not_followed_by (Omega_parser_helper.alphanum) @@ -174,7 +174,7 @@ module Make (Metasyntax : Types.Metasyntax.S) = struct many @@ choice [ (pos >>= fun offset -> hole >>| fun (pattern, variable, kind) -> Hole { pattern; variable; offset; kind = attribute_to_kind kind }) - ; ((up_to (choice rewrite_hole_parsers)) >>| fun c -> Constant (String.of_char_list c)) + ; (up_to (choice rewrite_hole_parsers) >>| fun c -> Constant (String.of_char_list c)) ] let parse template = @@ -261,29 +261,6 @@ module Make (Metasyntax : Types.Metasyntax.S) = struct let _, column = Match.Offset.convert_fast ~offset index in Int.to_string column - | LsifHover -> - filepath >>= fun filepath -> - if debug then Format.printf "File for lsif.hover lookup: %s@." filepath; - Environment.lookup env variable >>= fun value -> - Environment.lookup_range env variable - >>= fun { match_start = { offset; _ }; _ } -> - if debug then Format.printf "Var offset is %d@." offset; - let source = In_channel.read_all filepath in (* Inefficient. *) - if debug then Format.printf "Read filepath, source len is %d@." @@ String.length source; - String.chop_prefix_if_exists filepath ~prefix:(Sys.getcwd ()) |> fun filepath_relative_root -> - if debug then Format.printf "File relative root: %s@." filepath; - let index = Match.Offset.index ~source in - let line, column = Match.Offset.convert_fast ~offset index in - let line, column = line - 1, column - 1 + String.length value - 1 in - if debug then Format.printf "Querying type at %d::%d@." line column; - let context = - Comby_semantic.Lsif.Context. - { repository = "github.com/sourcegraph/sourcegraph" - ; lsif_endpoint = "https://sourcegraph.com/.api/graphql" - ; formatting = Markdown ("```go", "```") (* Expose. *) - } - in - Comby_semantic.Lsif.hover_at context ~filepath:filepath_relative_root ~line ~column | FilePath -> filepath | FileName -> filepath >>| Filename.basename | FileDirectory -> filepath >>| Filename.dirname @@ -320,6 +297,22 @@ module Make (Metasyntax : Types.Metasyntax.S) = struct >>| camel_to_snake >>| String.lowercase + | External "lsif.hover" -> + filepath >>= fun filepath -> + if debug then Format.printf "File for lsif.hover lookup: %s@." filepath; + Environment.lookup env variable >>= fun value -> + Environment.lookup_range env variable + >>= fun { match_start = { offset; _ }; _ } -> + let source = In_channel.read_all filepath in (* Inefficient. *) + if debug then Format.printf "Read filepath, source len is %d@." @@ String.length source; + let index = Match.Offset.index ~source in + let line, column = Match.Offset.convert_fast ~offset index in + let line, column = line - 1, column - 1 + String.length value - 1 in + if debug then Format.printf "Var offset:%d line:%d col:%d @." offset line column; + External.handler ~name:"lsif.hover" ~filepath ~line ~column + + | External _ -> assert false + let substitute ?filepath template environment = let replacement_content, environment', _ = List.fold template ~init:([], Environment.create (), 0) ~f:(fun (result, env, pos) -> function diff --git a/lib/kernel/matchers/template.mli b/lib/kernel/matchers/template.mli index 048b440..67525fa 100644 --- a/lib/kernel/matchers/template.mli +++ b/lib/kernel/matchers/template.mli @@ -1,16 +1,3 @@ -open Types.Template +open Types -module Make : Metasyntax.S -> sig - - module Matching : sig - val hole_parsers : (Types.Hole.sort * string Vangstrom.t) list - end - - val parse : string -> t - - val variables : string -> syntax list - - val to_string : t -> string - - val substitute : ?filepath:string -> t -> Match.Environment.t -> (string * Match.Environment.t) - end +module Make : Metasyntax.S -> External.S -> Template.S diff --git a/lib/kernel/matchers/types.ml b/lib/kernel/matchers/types.ml index 07ca410..0f67d00 100644 --- a/lib/kernel/matchers/types.ml +++ b/lib/kernel/matchers/types.ml @@ -116,6 +116,14 @@ module Metasyntax = struct end end +module External = struct + type t = name:string -> filepath:string -> line:int -> column:int -> string option + + module type S = sig + val handler : t + end +end + type production = | Unit | String of string @@ -132,7 +140,6 @@ module Template = struct | LineEnd | ColumnStart | ColumnEnd - | LsifHover | FileName | FilePath | FileDirectory @@ -144,6 +151,7 @@ module Template = struct | LowerCamelCase | UpperSnakeCase | LowerSnakeCase + | External of string [@@deriving sexp] type syntax = @@ -161,6 +169,21 @@ module Template = struct type t = atom list [@@deriving sexp] + + module type S = sig + + module Matching : sig + val hole_parsers : (Hole.sort * string Vangstrom.t) list + end + + val parse : string -> t + + val variables : string -> syntax list + + val to_string : t -> string + + val substitute : ?filepath:string -> t -> Match.Environment.t -> (string * Match.Environment.t) + end end module Ast = struct @@ -187,6 +210,10 @@ end module Rule = struct type t = Ast.expression list [@@deriving sexp] + + module type S = sig + val create : string -> (Ast.expression list, Error.t) result + end end module Matcher = struct @@ -216,7 +243,7 @@ end module Engine = struct module type S = sig - module Make : Language.S -> Metasyntax.S -> Matcher.S + module Make : Language.S -> Metasyntax.S -> External.S -> Matcher.S module Text : Matcher.S module Paren : Matcher.S @@ -272,7 +299,7 @@ module Engine = struct module C_nested_comments : Matcher.S val all : (module Matcher.S) list - val select_with_extension : ?metasyntax:Metasyntax.t -> string -> (module Matcher.S) option - val create : ?metasyntax:Metasyntax.t -> Language.Syntax.t -> (module Matcher.S) + val select_with_extension : ?metasyntax:Metasyntax.t -> ?external_handler:External.t -> string -> (module Matcher.S) option + val create : ?metasyntax:Metasyntax.t -> ?external_handler:External.t -> Language.Syntax.t -> (module Matcher.S) end end diff --git a/test/common/test_cli.ml b/test/common/test_cli.ml index 420e416..398cfe2 100644 --- a/test/common/test_cli.ml +++ b/test/common/test_cli.ml @@ -1245,17 +1245,17 @@ let%expect_test "test_custom_metasyntax_substitute" = print_string result; [%expect "$A hello"] -let%expect_test "test_custom_metasyntax_partial_rule_support" = +let%expect_test "test_custom_metasyntax_rule_support" = let source = "a(b)" in let metasyntax_path = "example" ^/ "metasyntax" ^/ "dolla.json" in let command_args = - Format.sprintf {|'$A($B)' '$A $B' -rule 'where rewrite :[A] { "$C~a$" -> "$C" }' -stdin -custom-metasyntax %s -stdout -matcher .generic|} metasyntax_path + Format.sprintf {|'$A($B)' '$A $B' -rule 'where rewrite $A { "$C~a$" -> "$C" }' -stdin -custom-metasyntax %s -stdout -matcher .generic|} metasyntax_path in let command = Format.sprintf "%s %s" binary_path command_args in let result = read_expect_stdin_and_stdout command source in print_string result; [%expect " - $C b"] + a b"] let%expect_test "test_custom_metasyntax_reserved_identifiers" = let source = "fun f -> (fun x -> f (x x)) (fun x -> f (x x))" in diff --git a/test/common/test_helpers.ml b/test/common/test_helpers.ml index 3382d17..db60b12 100644 --- a/test/common/test_helpers.ml +++ b/test/common/test_helpers.ml @@ -48,7 +48,7 @@ let make_env bindings = let parse_template metasyntax template = let (module M) = Matchers.Metasyntax.create metasyntax in - let module Template_parser = Template.Make(M) in + let module Template_parser = Template.Make(M)(External.Default) in let tree = Template_parser.parse template in Sexp.to_string_hum (Template.sexp_of_t tree) diff --git a/test/common/test_parse_template.ml b/test/common/test_parse_template.ml index b16bb28..7e58490 100644 --- a/test/common/test_parse_template.ml +++ b/test/common/test_parse_template.ml @@ -3,40 +3,41 @@ open Comby_kernel open Test_helpers +open Matchers let%expect_test "get_offsets_for_holes" = - let module Template_parser = Matchers.Template.Make(Matchers.Metasyntax.Default) in + let module Template_parser = Template.Make(Metasyntax.Default)(External.Default) in let rewrite_template = {|1234:[1]1234:[2]|} in let variables = Template_parser.variables rewrite_template in - print_s [%message (variables : Matchers.Template.syntax list)]; + print_s [%message (variables : Template.syntax list)]; [%expect {| (variables (((variable 1) (pattern :[1]) (offset 4) (kind Value)) ((variable 2) (pattern :[2]) (offset 12) (kind Value)))) |}] let%expect_test "interpret_regex_shorthand" = - let module Template_parser = Matchers.Template.Make(Matchers.Metasyntax.Default) in + let module Template_parser = Template.Make(Metasyntax.Default)(External.Default) in let rewrite_template = {|a:[~x]b|} in let variables = Template_parser.variables rewrite_template in - print_s [%message (variables : Matchers.Template.syntax list)]; + print_s [%message (variables : Template.syntax list)]; [%expect {| (variables (((variable "") (pattern :[~x]) (offset 1) (kind Value)))) |}] let%expect_test "interpret_incomplete_hole_as_constant" = let template = ":[B :[A]" in - parse_template Matchers.Metasyntax.default_metasyntax template |> print_string; + parse_template Metasyntax.default_metasyntax template |> print_string; [%expect_exact {|((Constant ":[B ") (Hole ((variable A) (pattern :[A]) (offset 4) (kind Value))))|}]; let template = ":[B :[A~x]" in - parse_template Matchers.Metasyntax.default_metasyntax template |> print_string; + parse_template Metasyntax.default_metasyntax template |> print_string; [%expect_exact {|((Constant ":[B ") (Hole ((variable A) (pattern :[A~x]) (offset 4) (kind Value))))|}] let%expect_test "interpret_incomplete_hole_as_constant_metasyntax" = let template = "$:x $B:x $A" in let metasyntax = - Matchers.Metasyntax.{ + Metasyntax.{ syntax = [ Hole (Everything, Delimited (Some "$", None)) ; Hole (Alphanum, Delimited (Some "$$", None)) @@ -54,7 +55,7 @@ let%expect_test "interpret_incomplete_hole_as_constant_metasyntax" = let%expect_test "interpret_incomplete_hole_as_constant_metasyntax" = let template = "( , , )" in let metasyntax = - Matchers.Metasyntax.{ + Metasyntax.{ syntax = [ Hole (Everything, Delimited (Some "NOTHING", None)) ; Hole (Everything, Reserved_identifiers [" "; " "]) @@ -73,7 +74,7 @@ let%expect_test "interpret_incomplete_hole_as_constant_metasyntax" = let%expect_test "interpret_incomplete_hole_as_constant_metasyntax" = let template = "(..,.)" in let metasyntax = - Matchers.Metasyntax.{ + Metasyntax.{ syntax = [ Hole (Everything, Delimited (Some "NOTHING", None)) ; Hole (Everything, Reserved_identifiers [".."; "."]) @@ -90,7 +91,7 @@ let%expect_test "interpret_incomplete_hole_as_constant_metasyntax" = let%expect_test "parse_reserved_identifiers_as_holes" = let template = "(α)" in let metasyntax = - Matchers.Metasyntax.{ + Metasyntax.{ syntax = [ Hole (Expression, Reserved_identifiers ["α"]) ] @@ -103,12 +104,11 @@ let%expect_test "parse_reserved_identifiers_as_holes" = (Hole ((variable "\206\177") (pattern "\206\177") (offset 1) (kind Value))) (Constant ")"))|}] - let%expect_test "get_offsets_for_holes" = - let module Template_parser = Matchers.Template.Make(Matchers.Metasyntax.Default) in + let module Template_parser = Template.Make(Metasyntax.Default)(External.Default) in let template = ":[a].type :[b].length :[[c]].lengtha :[d.].length.ok (:[e].length)" in let variables = Template_parser.variables template in - print_s [%message (variables : Matchers.Template.syntax list)]; + print_s [%message (variables : Template.syntax list)]; [%expect {| (variables (((variable a) (pattern :[a]) (offset 0) (kind Value)) diff --git a/test/common/test_rewrite_parts.ml b/test/common/test_rewrite_parts.ml index f3785a7..6fef9fa 100644 --- a/test/common/test_rewrite_parts.ml +++ b/test/common/test_rewrite_parts.ml @@ -7,8 +7,6 @@ open Matchers let all ?(configuration = configuration) template source = Alpha.C.all ~configuration ~template ~source () -module Template_parser = Template.Make(Metasyntax.Default) - let%expect_test "comments_in_string_literals_should_not_be_treated_as_comments_by_fuzzy" = let source = {|123433312343331122|} in let match_template = {|1234:[1]1234:[3]11:[2]|} in diff --git a/test/common/test_script.ml b/test/common/test_script.ml index 0479043..38808b1 100644 --- a/test/common/test_script.ml +++ b/test/common/test_script.ml @@ -1,5 +1,7 @@ open Matchers +module Script = Script.Make (Metasyntax.Default) (External.Default) + let run input = let result = match Script.parse input with