comby-tools/comby
1
1
Fork 0
mirror of https://github.com/comby-tools/comby.git synced 2024-09-11 13:25:36 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add alphanumeric delimiter support

Browse Source 
Adds support for nestable alphanumeric delimiters (e.g., `def ... end` in Ruby,`case ... esac` in Bash, and `begin ... end` in OCaml).
This commit is contained in:
Rijnard van Tonder 2019-07-06 01:19:51 -04:00 committed by GitHub
parent fc37d0d295
commit 7708c056d1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 685 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
docs/match-semantics.md Normal file
View File

@ -0,0 +1,48 @@
## Weak delimiter matching
Suppose we have a pattern like `(:[1])`. We could weaken delimiter matching to
allow matching, for example, anything except parens, i.e., `(])` would be valid
and could be matched against. In strict delimiter matching, where `[]` must be
balanced, `(])` would not be valid. This could, in theory, give a bit of a
performance bump since we wouldn't neet to ensure well-balancedness with
respect to any other delimiters besides `()`.
Weak delimiter matching only works for unique delimiters. For example, the
trick does not work for a closing delimiter like `end` in Ruby where multiple
opening delimiters like `class` or `def` close with `end`. In this case, we
have no choice but to check well-balancedness of all delimiters with the same
closing delimiter.
## Matching alphanumeric delimiters
A neat thing about Comby is that holes can match at the character level (not
just word boundaries). This makes it a little bit more challenging to identify
alphanumeric delimiters like `for`, `end`, because a character sequence like
`for` in the word `before` would, under normal circumstances, under character
matching, trigger delimiter matching, and Comby will look for an `end` to the
`for` in `before`. This problem doesn't come up for `()` delimiters, for
example, because punctuation isn't mixed with alphanumberic sequences. The way
we deal with this complexity in Comby is as follows:
We detect alpanumeric delimiters by requiring surrounding content (such as
whitespace or other delimiters). Before something like 'def', we expect
whitespace, punctuation delimiters like ')', and nothing. After 'def', we
expect similar sequences. Buf after something like 'end', we expect the same
but also punctuation like `;` or `.`.
There's some complexity for detecting these cases: we don't consume the
trailing whitespace, because that would stop us from detecting `begin begin`,
separated by a single space. So the trailing part is only checked as a look
ahead, but not consumed. We do, however, need to consume the prefix in order to
advance the state (otherwise, the prefix whitespace would be handled normally,
and we would re-encounter the delimiter subsequently).
## Choice operator behavior and 'attempt'
If `a1` succees in the parse sequence `a1 >>= a2 >>= a3 <|> b1 >>= b2 >>= b3`,
the whole parser will fail, and `b1 >>= ...` will never be attempted. Putting
`attempt @@ a1 >>= a2 >>= a3 <|> attempt @@ b1 >>= ...` will backtrack if `a1`
succeeds but `a2` fails, and will then try `b1`. This pattern is important for
disambiguating holes `:[1]` and `:[[1]]`, and alphanumeric sequences (like
`begin`, `struct`) where we need to check if the sequence initiates a balanced
delimiter matching or not.

13
lib/matchers/languages.ml
View File

@ -1,4 +1,4 @@
let experimental = false
let enable_alphanum_delimiters = true
module Text = struct
module Syntax = struct
@ -137,6 +137,9 @@ module Bash = struct
Dyck.Syntax.user_defined_delimiters @
[ "if ", "fi"
; "case ", "esac"
; "for ", "done"
; "until ", "done"
; "while ", "done"
]
let comment_parser =
@ -153,7 +156,7 @@ module Ruby = struct
let user_defined_delimiters =
Generic.Syntax.user_defined_delimiters
@ if experimental then
@ if enable_alphanum_delimiters then
[ "class", "end"
; "def", "end"
; "do", "end"
@ -189,7 +192,7 @@ module Elixir = struct
let user_defined_delimiters =
Generic.Syntax.user_defined_delimiters
@ if experimental then
@ if enable_alphanum_delimiters then
[ "fn", "end"
; "do", "end"
; "case", "end"
@ -265,7 +268,7 @@ module Erlang = struct
let user_defined_delimiters =
Generic.Syntax.user_defined_delimiters
@ if experimental then
@ if enable_alphanum_delimiters then
[ "fun", "end"
; "case", "end"
; "if", "end"
@ -385,7 +388,7 @@ module OCaml = struct
let user_defined_delimiters =
Generic.Syntax.user_defined_delimiters
@ if experimental then
@ if enable_alphanum_delimiters then
[ "begin", "end"
; "struct", "end"
; "sig", "end"

389
lib/matchers/matcher.ml
View File

@ -10,7 +10,17 @@ open Types
let configuration_ref = ref (Configuration.create ())
let weaken_delimiter_hole_matching = false
let debug = false
let debug =
Sys.getenv "DEBUG_COMBY"
|> Option.is_some
let debug_hole =
Sys.getenv "DEBUG_COMBY_HOLE"
|> Option.is_some
let debug_position =
Sys.getenv "DEBUG_COMBY_POS"
|> Option.is_some
let f _ = return Unit
@ -41,7 +51,6 @@ module Make (Syntax : Syntax.S) = struct
return (f ~contents ~left_delimiter:delimiter ~right_delimiter:delimiter))
|> choice
let raw_string_literal_parser (f : 'a literal_parser_callback) =
List.map Syntax.raw_string_literals ~f:(fun (left_delimiter, right_delimiter) ->
let module M =
@ -102,24 +111,77 @@ module Make (Syntax : Syntax.S) = struct
is_not (string right_delimiter) |>> String.of_char
let generate_spaces_parser () =
(* at least a space followed by comments and spaces *)
(* At least a space followed by comments and spaces. *)
(spaces1
>> many comment_parser << spaces
>>= fun result -> f result)
<|>
(* This case not covered by tests, may not be needed *)
(* This case not covered by tests, may not be needed. *)
(many1 comment_parser << spaces >>= fun result -> f result)
let sequence_chain (plist : ('c, Match.t) parser sexp_list) : ('c, Match.t) parser =
List.fold plist ~init:(return Unit) ~f:(>>)
let with_debug_matcher s tag =
if debug then
match tag with
| `Position tag ->
if debug_position then
let prev = prev_char s in
let curr = read_char s in
let next = next_char s in
let print_if = function
| Some s -> s
| None -> '?'
in
Format.printf "Position Tag: %s@." tag;
Format.printf "H_prev: %c H_curr: %c H_next: %c@."
(print_if prev)
(print_if curr)
(print_if next)
| `Delimited delimited ->
Format.printf "<d>%s</d>%!" delimited
| `Delimited_suffix suffix ->
Format.printf "<d_s>%s</d_s>%!" suffix
| `Checkpoint (label, s) ->
Format.printf "Point(%s):<d>%s</d>" label s
| _ -> assert false
let is_alphanum delim = Pcre.(pmatch ~rex:(regexp "^[[:alnum:]]+$") delim)
let whitespace : (id, Match.t) parser = many1 space |>> String.of_char_list
let not_alphanum = many1 (is_not alphanum) |>> String.of_char_list
let reserved_alphanum_delimiter_must_satisfy =
Syntax.user_defined_delimiters
|> List.filter_map ~f:(fun (from, until) ->
if not (is_alphanum from) then
Some [from; until]
else
None)
|> List.concat
|> List.map ~f:string
|> List.map ~f:attempt
let nested_delimiters_parser (f : 'a nested_delimiter_callback) =
(* All alphanum delimiter fixups happen in the generated parser, not here. *)
let between p from until s =
(string from >>= fun from ->
if debug then with_debug_matcher s (`Delimited from);
p >>= fun p_result ->
string until >>= fun until ->
if debug then with_debug_matcher s (`Delimited until);
return p_result)
s
in
Syntax.user_defined_delimiters
|> List.map ~f:(fun (left_delimiter, right_delimiter) ->
Parsers.Delimiters.between
between
(f ~left_delimiter ~right_delimiter)
left_delimiter right_delimiter)
left_delimiter
right_delimiter
)
|> choice
(* Backtrack on failure, specifically for alphanum. *)
|> attempt
(** All code can have comments interpolated *)
let generate_string_token_parser str : ('c, _) parser =
@ -128,22 +190,23 @@ module Make (Syntax : Syntax.S) = struct
>> many comment_parser
>>= fun result -> f result
let identifier () =
(many (alphanum <|> char '_') |>> String.of_char_list)
let everything_hole_parser () =
string ":[" >> (many (alphanum <|> char '_') |>> String.of_char_list) << string "]"
string ":[" >> identifier () << string "]"
let non_space_hole_parser () =
string ":[" >> (many (alphanum <|> char '_') |>> String.of_char_list) << string ".]"
string ":[" >> identifier () << string ".]"
let line_hole_parser () =
string ":[" >> (many (alphanum <|> char '_') |>> String.of_char_list) << string "\\n]"
string ":[" >> identifier () << string "\\n]"
let blank_hole_parser () =
string ":[" >> (many1 blank) >> (many (alphanum <|> char '_') |>> String.of_char_list) << string "]"
string ":[" >> (many1 blank) >> identifier () << string "]"
let alphanum_hole_parser () =
string ":[[" >> (many (alphanum <|> char '_') |>> String.of_char_list) << string "]]"
>>= fun id ->
return id
string ":[[" >> identifier () << string "]]"
let reserved_holes () =
let alphanum = alphanum_hole_parser () in
@ -158,10 +221,37 @@ module Make (Syntax : Syntax.S) = struct
; everything
]
let reserved_delimiters =
let reserved_delimiters _s =
let required_from_prefix = not_alphanum in
let required_from_suffix = not_alphanum in
let required_until_suffix = not_alphanum in
let handle_alphanum_delimiters_reserved_trigger from until =
let from_parser =
required_from_prefix >>= fun _ ->
string from >>= fun from ->
look_ahead required_from_suffix >>= fun _ ->
return from
in
let until_parser s =
(string until >>= fun until ->
eof <|> look_ahead (skip required_until_suffix) >>= fun _ ->
(* if current char/next_char is alphanum, make unsat. *)
let prev = prev_char s in
if debug then with_debug_matcher s (`Position "reserved_delimiter_until");
match prev with
| Some prev when is_alphanum (Char.to_string prev) -> fail "unsat"
| _ -> return until
)
s
in
[from_parser; until_parser]
in
let reserved_delimiters =
List.concat_map Syntax.user_defined_delimiters ~f:(fun (from, until) -> [from; until])
|> List.map ~f:string
List.concat_map Syntax.user_defined_delimiters ~f:(fun (from, until) ->
if is_alphanum from && is_alphanum until then
handle_alphanum_delimiters_reserved_trigger from until
else
[ string from; string until])
in
let reserved_escapable_strings =
List.concat_map Syntax.escapable_string_literals ~f:(fun x -> [x])
@ -176,13 +266,13 @@ module Make (Syntax : Syntax.S) = struct
@ reserved_escapable_strings
@ reserved_raw_strings
|> List.map ~f:skip
(* attempt the reserved: otherwise, if something passes partially,
it won't detect that single or greedy is reserved *)
(* Attempt the reserved: otherwise, if a parser partially succeeds,
it won't detect that single or greedy is reserved. *)
|> List.map ~f:attempt
|> choice
let reserved =
reserved_delimiters
let reserved s =
reserved_delimiters s
<|> skip (space |>> Char.to_string)
let until_of_from from =
@ -225,55 +315,160 @@ module Make (Syntax : Syntax.S) = struct
{ result with environment })
>>= fun () -> f matched
let alphanum_delimiter_must_satisfy =
many1 (is_not (skip (choice reserved_alphanum_delimiter_must_satisfy) <|> skip alphanum))
|>> String.of_char_list
let generate_everything_hole_parser
?priority_left_delimiter:left_delimiter
?priority_right_delimiter:right_delimiter =
let between_nested_delims p from =
let until = until_of_from from in
between (string from) (string until) p
|>> fun result -> (String.concat @@ [from] @ result @ [until])
let with_debug_hole tag =
match tag with
| `Spaces chars ->
if debug_hole then Format.printf "<H_sp>";
return (String.of_char_list chars)
| `Delimited delimited ->
if debug_hole then Format.printf "<H_d>%s</H_d>" delimited;
return delimited
| `Character c ->
if debug_hole then Format.printf "<H_c>%c</H_c>" c;
return (String.of_char c)
| `Body body ->
let body = String.concat body in
if debug_hole then Format.printf "<H_body>%s</H_body>" body;
return body
| `Checkpoint (label, s) ->
if debug_hole then Format.printf "Point(%s):<d>%s</d>" label s;
return s
| _ -> assert false
in
let between_nested_delims p =
let trigger_nested_parsing_prefix =
if weaken_delimiter_hole_matching then
match left_delimiter, right_delimiter with
| Some left_delimiter, Some right_delimiter ->
[ (left_delimiter, right_delimiter) ]
| _ -> Syntax.user_defined_delimiters
else
Syntax.user_defined_delimiters
let delimiters =
if weaken_delimiter_hole_matching then
match left_delimiter, right_delimiter with
| Some left_delimiter, Some right_delimiter ->
[ (left_delimiter, right_delimiter) ]
| _ -> Syntax.user_defined_delimiters
else
Syntax.user_defined_delimiters
in
let handle_alphanum_delimiters from until p =
(* mandatory_prefix: needs to be not alphanum AND not non-alphanum delim.
If it is a paren, we need to fail and get out of this alphanum block
parser (how did we end up in here? because we said that we'd accept
anything as prefix to 'def', including '(', and so '(' is not handled
as a delim.) *)
let mandatory_prefix = alphanum_delimiter_must_satisfy in
(* mandatory_suffix: be more strict with suffix of pening delimiter: don't
use 'any non-alphanum', but instead use whitespace. This since 'def;'
is undesirable, and 'def.foo' may be intentional. But 'end.' or 'end;'
probably still refer to a closing delim. *)
let mandatory_suffix = choice reserved_alphanum_delimiter_must_satisfy <|> whitespace in
let satisfy_opening_delimiter prev =
(match prev with
| Some prev when is_alphanum (Char.to_string prev) -> fail "unsat"
(* Try parse whitespace, and we want to cpature its length, in case
this is a space between, like 'def def end end'. But in the case
where there's no space, it means we have just entered the beginning
of the hole which may start with the 'd' of 'def', but since we
already know that the previous char is not alphanum in this branch
(so it is a delimiter or whitespace) it is OK to continue: in this
case, return "". *)
| _ -> mandatory_prefix <|> return "")
>>= fun prefix ->
string from >>= fun open_delimiter ->
with_debug_hole (`Checkpoint ("open_delimiter_<pre>"^prefix^"</pre>_sat_for", open_delimiter)) >>= fun _ ->
(* Use look_ahead to ensure that there is, e.g., whitespace after this
possible delimiter, but without consuming input. Whitespace needs to
not be consumed so that we can detect subsequent delimiters. *)
look_ahead mandatory_suffix >>= fun suffix ->
with_debug_hole (`Checkpoint ("open_delimiter_<suf>"^suffix^"</suf>_sat_for", open_delimiter)) >>= fun _ ->
return (prefix, open_delimiter)
in
trigger_nested_parsing_prefix
|> List.map ~f:fst
|> List.map ~f:(between_nested_delims p)
|> choice
let satisfy_closing_delimiter =
string until >>= fun close_delimiter ->
look_ahead @@ mandatory_suffix >>= fun suffix ->
with_debug_hole (`Checkpoint ("close_delimiter_<suf>"^suffix^"</suf>_sat_for", close_delimiter)) >>= fun _ ->
return close_delimiter
in
(fun s ->
let prev = prev_char s in
(satisfy_opening_delimiter prev >>= fun (prefix, open_delimiter) ->
p >>= fun in_between ->
with_debug_hole (`Body in_between) >>= fun in_between ->
satisfy_closing_delimiter >>= fun close_delimiter ->
return
((prefix^open_delimiter)
^in_between
^close_delimiter)) s)
(* Use attempt so that, e.g., 'struct' is tried after 'begin' delimiters under choice. *)
|> attempt
in
(* applies looser delimiter constraints for matching *)
let handle_alphanum_delimiters_reserved_trigger from until =
(* If it's alphanum, only consider it reserved if there is, say, whitespace after and so
handle alternatively. Otherwise, return empty to indicate 'this sequence of characters
is not reserved'. *)
let reserved =
Syntax.user_defined_delimiters
|> List.filter_map ~f:(fun (from, _) ->
if not (is_alphanum from) then
Some from
else
None)
|> List.map ~f:string
|> List.map ~f:attempt
in
let required_delimiter_terminal =
many1 (is_not (skip (choice reserved) <|> skip alphanum)) |>> String.of_char_list
in
List.map [from; until] ~f:(fun delimiter ->
(fun s ->
let prev = prev_char s in
(match prev with
| Some prev when is_alphanum (Char.to_string prev) ->
(* If prev is alphanum, this can't possibly be a reserved delimiter. Just continue. *)
fail "unsat"
| _ -> string delimiter >>= fun _ ->
look_ahead required_delimiter_terminal) s))
in
(* The cases for which we need to stop parsing just characters
and consider delimiters. *)
let reserved =
let trigger_nested_parsing_prefix =
if weaken_delimiter_hole_matching then
match left_delimiter, right_delimiter with
| Some left_delimiter, Some right_delimiter ->
[ (left_delimiter, right_delimiter) ]
| _ -> Syntax.user_defined_delimiters
else
Syntax.user_defined_delimiters
in
trigger_nested_parsing_prefix
|> List.concat_map ~f:(fun (from, until) -> [from; until])
|> List.map ~f:string
List.concat_map delimiters ~f:(fun (from, until) ->
if is_alphanum from then
handle_alphanum_delimiters_reserved_trigger from until
else
[string from; string until]
)
|> List.map ~f:attempt
|> choice
in
(* a parser that understands the hole matching cut off points happen at
delimiters *)
(* A parser that understands the hole matching cut off points happen at
delimiters. *)
let rec nested_grammar s =
(comment_parser
<|> raw_string_literal_parser (fun ~contents ~left_delimiter:_ ~right_delimiter:_ -> contents)
<|> escapable_string_literal_parser (fun ~contents ~left_delimiter:_ ~right_delimiter:_ -> contents)
<|> delimsx
<|> (is_not reserved |>> String.of_char))
<|> (many1 space >>= fun r -> with_debug_hole (`Spaces r))
<|> (attempt @@ delims_over_holes >>= fun r -> with_debug_hole (`Delimited r))
(* Only consume if not reserved. If it is reserved, we want to trigger the 'many'
in (many nested_grammar) to continue. *)
<|> (is_not (reserved <|> (space |>> Char.to_string)) >>= fun r -> with_debug_hole (`Character r)))
s
and delimsx s = (between_nested_delims (many nested_grammar)) s
and delims_over_holes s =
let between_nested_delims p =
let capture_delimiter_result p ~from =
let until = until_of_from from in
if is_alphanum from then
handle_alphanum_delimiters from until p
else
between (string from) (string until) p
>>= fun result -> return (String.concat @@ [from] @ result @ [until])
in
delimiters
|> List.map ~f:(fun pair -> capture_delimiter_result p ~from:(fst pair))
|> choice
in
(between_nested_delims (many nested_grammar)) s
in
nested_grammar
@ -291,7 +486,6 @@ module Make (Syntax : Syntax.S) = struct
match result with
| Hole Alphanum (identifier, _) ->
let allowed = choice [alphanum; char '_'] |>> String.of_char in
(* if we collapse the not_followed_by part, we will disallow substring matching. *)
let hole_semantics = many1 (not_followed_by rest "" >> allowed) in
record_matches identifier hole_semantics
@ -330,7 +524,7 @@ module Make (Syntax : Syntax.S) = struct
raw_literal_grammar ~right_delimiter
| Comment -> failwith "Unimplemented"
in
(* continue until rest, but don't consume rest. *)
(* Continue until rest, but don't consume rest. *)
let hole_semantics = many (not_followed_by rest "" >> matcher) in
record_matches identifier hole_semantics
@ -339,23 +533,25 @@ module Make (Syntax : Syntax.S) = struct
process_hole::acc)
let hole_parser sort dimension =
let skip_signal result = skip (string "_signal_hole") |>> fun () -> result in
let skip_signal hole =
skip (string "_signal_hole") |>> fun () -> Hole hole
in
match sort with
| `Everything ->
everything_hole_parser () |>> fun id ->
skip_signal (Hole (Everything (id, dimension)))
skip_signal (Everything (id, dimension))
| `Non_space ->
non_space_hole_parser () |>> fun id ->
skip_signal (Hole (Non_space (id, dimension)))
skip_signal (Non_space (id, dimension))
| `Line ->
line_hole_parser () |>> fun id ->
skip_signal (Hole (Line (id, dimension)))
skip_signal (Line (id, dimension))
| `Blank ->
blank_hole_parser () |>> fun id ->
skip_signal (Hole (Blank (id, dimension)))
skip_signal (Blank (id, dimension))
| `Alphanum ->
alphanum_hole_parser () |>> fun id ->
skip_signal (Hole (Alphanum (id, dimension)))
skip_signal (Alphanum (id, dimension))
let generate_hole_for_literal sort ~contents ~left_delimiter ~right_delimiter s =
let holes =
@ -397,24 +593,56 @@ module Make (Syntax : Syntax.S) = struct
|> List.map ~f:(fun kind -> attempt (hole_parser kind Code))
in
choice holes
(* string literals are handled specially because match semantics change inside string delimiters *)
(* String literals are handled specially because match semantics change inside string delimiters. *)
<|> (raw_string_literal_parser (generate_hole_for_literal Raw_string_literal))
<|> (escapable_string_literal_parser (generate_hole_for_literal Escapable_string_literal))
(* whitespace is handled specially because we may change whether they are significant for matching *)
<|> (spaces1 |>> generate_spaces_parser)
(* nested delimiters are handled specially for nestedness *)
(* Nested delimiters are handled specially for nestedness. *)
<|> (nested_delimiters_parser generate_outer_delimiter_parsers)
(* everything else *)
<|> ((many1 (is_not reserved) |>> String.of_char_list) |>> generate_string_token_parser)
(* Whitespace is handled specially because we may change whether they are significant for matching. *)
<|> (spaces1 |>> generate_spaces_parser)
(* Everything else. *)
<|>
((many1 (is_not (reserved _s)) >>= fun cl ->
if debug then Format.printf "<cl>%s</cl>" @@ String.of_char_list cl;
return @@ String.of_char_list cl)
|>> generate_string_token_parser)
and generate_outer_delimiter_parsers ~left_delimiter ~right_delimiter s =
let before s =
begin
if debug_hole then with_debug_matcher s (`Position "generate_outer_delimiter");
let p =
if is_alphanum left_delimiter then
(* This logic is needed for cases where we say 'def :[1] end' in the template,
and don't match partially on, say, 'adef body endq' in the underlying generated
parser. *)
let prev = prev_char s in
match prev with
| Some prev when is_alphanum (Char.to_string prev) -> fail "unsat"
| _ -> string left_delimiter
else
string left_delimiter
in
p >>= fun _ -> f [left_delimiter]
end s
in
let after =
let p =
if is_alphanum right_delimiter then
(* This handles the case for something like 'def body endly'. *)
string right_delimiter >>= fun delim ->
look_ahead @@ (eof <|> skip not_alphanum) >>= fun _ ->
return delim
else
string right_delimiter
in
p >>= fun _ -> f [right_delimiter]
in
(generate_parsers s >>= fun p_list ->
(turn_holes_into_matchers_for_this_level ~left_delimiter ~right_delimiter
([ string left_delimiter
>>= fun _ -> f [left_delimiter]]
@ p_list
@ [ string right_delimiter
>>= fun _ -> f [right_delimiter]])
(turn_holes_into_matchers_for_this_level
~left_delimiter
~right_delimiter
([before] @ p_list @ [after])
|> sequence_chain)
|> return
) s
@ -422,9 +650,9 @@ module Make (Syntax : Syntax.S) = struct
let general_parser_generator s =
let outer_p =
generate_parsers s >>= fun p_list ->
(* eof of template is here *)
eof >> (* result is unit so ignore *)
(* customize the inner parser *)
(* EOF of template is here. *)
eof >> (* Result is unit so ignore. *)
(* Customize the inner parser. *)
let inner_p =
let matcher : ('a, Match.t) parser =
turn_holes_into_matchers_for_this_level p_list
@ -458,7 +686,7 @@ module Make (Syntax : Syntax.S) = struct
many
(not_followed_by matcher "" >>
(
(* respect grammar but ignore contents up to a match *)
(* Respect grammar but ignore contents up to a match. *)
skip comment_parser
<|> skip (raw_string_literal_parser (fun ~contents:_ ~left_delimiter:_ ~right_delimiter:_ -> ()))
<|> skip (escapable_string_literal_parser (fun ~contents:_ ~left_delimiter:_ ~right_delimiter:_ -> ()))
@ -471,7 +699,8 @@ module Make (Syntax : Syntax.S) = struct
outer_p s
let to_template template : ('a, Match.t) MParser.t Or_error.t =
match parse_string general_parser_generator template 0 with
(* Use a match type for state so we can reuse parsers for inner and outer. *)
match parse_string general_parser_generator template (Match.create ()) with
| Success p -> Ok p
| Failed (msg, _) -> Or_error.error_string msg

4
lib/parsers/delimiters.ml
View File

@ -1,4 +0,0 @@
open MParser
let between p from until =
string from >> p << string until

1
lib/parsers/posix_character_classes.ml
View File

@ -1 +0,0 @@

4
scripts/release.sh
View File

@ -50,5 +50,5 @@ docker rmi -f comby-alpine-binary-build:latest
./build-docker-binary.sh
docker tag comby-alpine-binary-build:latest comby/comby:$VERSION
docker push comby/comby:$VERSION
# docker pull comby/comby:$VERSION
# docker run -it comby/comby:$VERSION -version
echo "run: 'docker pull comby/comby:$VERSION'"
echo "test: 'docker run -it comby/comby:$VERSION' -version"

2
src/main.ml
View File

@ -409,4 +409,4 @@ let default_command =
let () =
Scheduler.Daemon.check_entry_point ();
default_command
|> Command.run ~version:"0.6.0"
|> Command.run ~version:"0.7.0"

335
test/test_special_matcher_cases.ml
View File

@ -57,7 +57,6 @@ let%expect_test "strict_nested_matching" =
(*
let%expect_test "ocaml_blocks" =
let source = {|
module M : sig
@ -72,11 +71,14 @@ let%expect_test "ocaml_blocks" =
|}
in
let match_template = {|struct :[1] end|} in
let rewrite_template = {|struct <bye> end|} in
let rewrite_template = {|struct <deleted> end|} in
run (module Matchers.OCaml) source match_template rewrite_template;
[%expect_exact {|No matches.|}]
*)
[%expect_exact {|
module M : sig
type t
end = struct <deleted> end
|}]
let%expect_test "ocaml_complex_blocks_with_same_end" =
let source = {|
@ -94,21 +96,21 @@ let%expect_test "ocaml_complex_blocks_with_same_end" =
|}
in
let match_template = {|begin :[1] end|} in
let rewrite_template = {|-Body->:[1]<-Body-|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.OCaml) source match_template rewrite_template;
[%expect_exact {|
-Body->match x with
<1>match x with
| _ ->
let module M = struct type t<-Body-
-Body->begin
let module M = struct type t end
begin
begin
match y with
| _ -> ()<-Body-
| _ -> ()
end
end
end</1>
|}]
(* FIXME(#35): "before" triggers "for" block *)
let%expect_test "ruby_blocks" =
let source = {|
class ActionController::Base
@ -121,23 +123,320 @@ end
|}
in
let match_template = {|class :[1] end|} in
let rewrite_template = {|-Block->:[1]<-Block-|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|
-Block->ActionController::Base
<1>ActionController::Base
before_filter :generate_css_from_less
def generate_css_from_less
Less::More.generate_all<-Block-
end
Less::More.generate_all
end</1>
|}]
let%expect_test "erlang_blocks" =
let source = {|Big = fun(X) -> if X > 10 -> true; true -> false end end.|} in
let match_template = {|fun(:[1]) :[rest] end|} in
let rewrite_template = {|-Block->:[rest]<-Block-|} in
let rewrite_template = {|<rest>:[rest]</rest>|} in
run (module Matchers.Erlang) source match_template rewrite_template;
[%expect_exact {|Big = -Block->-> if X > 10 -> true; true -> false<-Block- end.|}]
[%expect_exact {|Big = <rest>-> if X > 10 -> true; true -> false end</rest>.|}]
let%expect_test "ruby_blocks" =
let source = {|class x end|} in
let match_template = {|class :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|<1>x</1>|}]
let%expect_test "ruby_blocks_1" =
let source = {|class class def body1 end def body2 end end end|} in
let match_template = {|class :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|<1>class def body1 end def body2 end end</1>|}]
let%expect_test "ruby_blocks_2" =
let source = {|class class (def body1 end) (def body2 end) end end|} in
let match_template = {|class :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|<1>class (def body1 end) (def body2 end) end</1>|}]
let%expect_test "ruby_blocks_3" =
let source = {| def (def b end)(def b end) end |} in
let match_template = {|def :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {| <1>(def b end)(def b end)</1> |}]
let%expect_test "ruby_blocks_4" =
let source = {| def (def a end) f (def b end)end |} in
let match_template = {|def :[1]end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {| <1>(def a end) f (def b end)</1> |}]
(* this is correct: there's no space in the source 'end', so it matches the inner def/end blocks *)
let%expect_test "ruby_blocks_5" =
let source = {| def (def a end) f (def b end) end |} in
let match_template = {|def :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {| <1>(def a end) f (def b end)</1> |}]
let%expect_test "ruby_blocks_5" =
let source = {| def(df b ed) (df b ed)end |} in
let match_template = {|def:[1]end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {| <1>(df b ed) (df b ed)</1> |}]
let%expect_test "ruby_blocks_5" =
let source = {|class class ((def (x) end) f (def body end)) end end|} in
let match_template = {|class :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|<1>class ((def (x) end) f (def body end)) end</1>|}]
let%expect_test "ruby_blocks_5" =
let source = {|class class (def body1 end) (def body2 end) end end|} in
let match_template = {|def :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|class class (<1>body1</1>) (<1>body2</1>) end end|}]
let%expect_test "ruby_blocks_5" =
let source = {|class class (def body1 end) (def body2 end) end end|} in
let match_template = {|class :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|<1>class (def body1 end) (def body2 end) end</1>|}]
let%expect_test "ruby_blocks_5" =
let source = {|class class def (def body1 end) (def body2 end) end end end|} in
let match_template = {|def :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|class class <1>(def body1 end) (def body2 end)</1> end end|}]
let%expect_test "ruby_blocks_5" =
let source = {|class class def () (def body2 end) end end end|} in
let match_template = {|def :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|class class <1>() (def body2 end)</1> end end|}]
let%expect_test "ruby_blocks_5" =
let source = {|def (def end) (def end) end|} in
let match_template = {|def :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|<1>(def end) (def end)</1>|}]
let%expect_test "ruby_blocks_5" =
let source = {|def (def end) a (def end) end|} in
let match_template = {|def :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|<1>(def end) a (def end)</1>|}]
let%expect_test "ruby_blocks_5" =
let source = {|def (defa aend) (adef aend) end|} in
let match_template = {|def :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|<1>(defa aend) (adef aend)</1>|}]
let%expect_test "ruby_blocks_5" =
let source = {|def (defa aend) a (adef aend) end|} in
let match_template = {|def :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|<1>(defa aend) a (adef aend)</1>|}]
let%expect_test "ruby_blocks_5" =
let source = {|def (adef a endq) end|} in
let match_template = {|def :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|<1>(adef a endq)</1>|}]
let%expect_test "ruby_blocks_5" =
let source = {|def adef a endq end|} in
let match_template = {|def :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|<1>adef a endq</1>|}]
let%expect_test "ruby_blocks_5" =
let source = {|def def foo end end|} in
let match_template = {|def :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|<1>def foo end</1>|}]
let%expect_test "ruby_blocks_5" =
let source = {|def def end endq|} in
let match_template = {|def :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|No matches.|}]
let%expect_test "ruby_blocks_5" =
let source = {|def adef a endq end |} in
let match_template = {|def :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|<1>adef a endq</1> |}]
let%expect_test "ruby_blocks_5" =
let source = {|def fadef a qendq end|} in
let match_template = {|def :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|<1>fadef a qendq</1>|}]
let%expect_test "ruby_blocks_5" =
let source = {|def defa aend end|} in
let match_template = {|def :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|<1>defa aend</1>|}]
let%expect_test "ruby_blocks_5" =
let source = {|(adef a endq)|} in
let match_template = {|def :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|No matches.|}]
let%expect_test "ruby_blocks_5" =
let source = {|def adef a endq end|} in
let match_template = {|def :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|<1>adef a endq</1>|}]
let%expect_test "ruby_blocks_5" =
let source = {|def (adef a endq) end|} in
let match_template = {|def :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|<1>(adef a endq)</1>|}]
let%expect_test "ruby_blocks_5" =
let source = {|class (def ( body )end) end|} in
let match_template = {|class :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|<1>(def ( body )end)</1>|}]
let%expect_test "ruby_blocks_5" =
let source = {|class def ( body )end end|} in
let match_template = {|class :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|<1>def ( body )end</1>|}]
let%expect_test "ruby_blocks_5" =
let source = {|class def( body ) end end|} in
let match_template = {|class :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|<1>def( body ) end</1>|}]
let%expect_test "ruby_blocks_5" =
let source = {|class def() end end|} in
let match_template = {|class :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|<1>def() end</1>|}]
let%expect_test "ruby_blocks_5" =
let source = {|class def () end end|} in
let match_template = {|class :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|<1>def () end</1>|}]
let%expect_test "ruby_blocks_5" =
let source = {|class def( body )end end|} in
let match_template = {|class :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|<1>def( body )end</1>|}]
let%expect_test "ruby_blocks_5" =
let source = {|class ( def( body )end) end|} in
let match_template = {|class :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|<1>( def( body )end)</1>|}]
let%expect_test "ruby_blocks_5" =
let source = {|class (def( body )end) end|} in
let match_template = {|class :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|<1>(def( body )end)</1>|}]
let%expect_test "ruby_blocks_5" =
let source = {|(def a endq)|} in
let match_template = {|def :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|No matches.|}]
let%expect_test "ruby_blocks_5" =
let source = {|def end|} in
let match_template = {|def :[1] end|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|No matches.|}]
let%expect_test "ruby_blocks_5" =
let source = {|(def foo end)|} in
let match_template = {|(def :[1] end)|} in
let rewrite_template = {|<1>:[1]</1>|} in
run (module Matchers.Ruby) source match_template rewrite_template;
[%expect_exact {|<1>foo</1>|}]