comby-tools/comby
1
1
Fork 0
mirror of https://github.com/comby-tools/comby.git synced 2024-08-16 16:50:37 +03:00
Code Issues Packages Projects Releases Wiki Activity

abstract rewriter over fresh variable generator (#269)

Browse Source
This commit is contained in:
Rijnard van Tonder 2021-04-05 21:05:52 -07:00 committed by GitHub
parent a2ad280fe8
commit 110dbe6f16
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 82 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
lib/comby.mli
View File

@ -503,36 +503,42 @@ type replacement = Replacement.result
Defines rewrite operations. *)
module Rewrite : sig
(** [all source metasyntax sequential rewrite_template matches] substitutes
(** [all source metasyntax fresh rewrite_template matches] substitutes
[rewrite_template] with each match in [matches] to create a rewrite result.
If [source] is specified, each rewrite result is substituted in-place in
the source. If [source] is not specified, rewritten matches are
newline-separated. If [metasyntax] is defined, the
rewrite template will respect custom metasyntax definitions.
newline-separated. If [metasyntax] is defined, the rewrite template will
respect custom metasyntax definitions.
If the rewrite template contains the syntax :[id()], then it is
substituted with fresh values. [sequential] determines whether fresh values
are monitonically increasing or a random hash. See [substitute] for more. *)
substituted with fresh values. [fresh] may be specified to supply custom
fresh values. If not specified, fresh variables are generated in increasing
rank starting with 1, and incremented. See [substitute] for more. *)
val all
: ?source:string
-> ?metasyntax:Matchers.metasyntax
-> ?sequential:bool
-> ?fresh:(unit -> string)
-> rewrite_template:string
-> match' list
-> replacement option
(** [substitute metasyntax sequential template environment] substitutes
[template] with the variable and value pairs in the [environment]. It
returns the result after substitution, and the list of variables in
[environment] that were substituted for. If [metasyntax] is defined, the
rewrite template will respect custom metasyntax definitions.
(** [substitute metasyntax fresh template environment] substitutes [template]
with the variable and value pairs in the [environment]. It returns the
result after substitution, and the list of variables in [environment] that
were substituted for. If [metasyntax] is defined, the rewrite template will
respect custom metasyntax definitions.
The syntax :[id()] is substituted with fresh values. If [sequential] is
true, it substitutes :[id()] starting with 1, and subsequent :[id()] values
increment the ID. Otherwise if [sequential] is false, it substitutes the
pattern :[id()] with a fresh hex string based on the last 48-bit part of a
UUID v3 identifier. *)
val substitute : ?metasyntax:Matchers.metasyntax -> ?sequential:bool -> string -> Match.environment -> (string * string list)
The syntax :[id()] is substituted with fresh values. If [fresh] is not
specified, the default behavior substitutes :[id()] starting with 1, and
subsequent :[id()] values increment the ID. If [fresh] is set, substitutes
the pattern :[id()] with the value of fresh () as the hole is encountered,
left to right. *)
val substitute
: ?metasyntax:Matchers.metasyntax
-> ?fresh:(unit -> string)
-> string
-> Match.environment
-> (string * string list)
end
(** {2 Pipeline}
@ -577,19 +583,21 @@ module Pipeline : sig
| Replacement of (Replacement.t list * string * int)
| Nothing
(** [execute matcher metasyntax subst timeout config source spec] runs a
[matcher] on [source] for [spec] parameterized by [config].
[substitute_in_place] sets whether rewrite output should substitute
rewritten values in place. [timeout] specifies a timeout in seconds
(default 3). If [metasyntax] is defined, rewrite operations will respect
custom metasyntax definitions. Note that [metasyntax] here does not affect
matching: [matcher] should be defined with a metasyntax definition if
desired. *)
(** [execute matcher subst timeout metasyntax fresh config source spec] runs a
[matcher] on [source] for [spec] parameterized by [config]. [subst] sets
whether rewrite output should substitute rewritten values in place.
[timeout] specifies a timeout in seconds (default 3). If [metasyntax] is
defined, rewrite operations will respect custom metasyntax definitions.
Note that [metasyntax] here does not affect matching: [matcher] should be
defined with a metasyntax definition if desired. A custom [fresh] variable
generator may supply values to use for substitution; see
[Rewrite.substitute] for more. *)
val execute
: (module Matchers.Matcher.S)
-> ?substitute_in_place:bool
-> ?timeout:int
-> ?metasyntax:Matchers.metasyntax
-> ?fresh:(unit -> string)
-> ?configuration:Matchers.configuration
-> single_source
-> specification

20
lib/pipeline/pipeline.ml
View File

@ -78,13 +78,13 @@ let log_to_file path =
let process_single_source
matcher
?(sequential = false)
?(omega = false)
?(fast_offset_conversion = false)
?(substitute_in_place = false)
?(verbose = false)
?(timeout = 3)
?metasyntax
?fresh
configuration
source
(Specification.{ rewrite_template; _ } as specification)
@ -117,7 +117,7 @@ let process_single_source
(* If there are no matches, return the original source (for editor support). *)
Replacement ([], input_text, 0)
| matches ->
match Rewrite.all ~source:input_text ?metasyntax ~sequential ~rewrite_template matches with
match Rewrite.all ~source:input_text ?metasyntax ?fresh ~rewrite_template matches with
| None -> Nothing
| Some { rewritten_source; in_place_substitutions } ->
Replacement (in_place_substitutions, rewritten_source, List.length matches)
@ -219,7 +219,6 @@ let run_batch ~f:per_unit sources compute_mode bound_count =
let run_interactive
specifications
sequential
matcher
omega
fast_offset_conversion
@ -236,7 +235,6 @@ let run_interactive
(fun (input : single_source) specification ->
process_single_source
matcher
~sequential
~omega
~fast_offset_conversion
~substitute_in_place
@ -284,7 +282,6 @@ let run
; metasyntax
}
=
let sequential = match compute_mode with | `Sequential -> true | _ -> false in
let match_configuration =
Matchers.Configuration.create
~disable_substring_matching
@ -294,6 +291,11 @@ let run
in
let start_time = Statistics.Time.start () in
let fresh = match compute_mode with
| `Sequential -> None
| _ -> Some (fun () -> Uuid_unix.(Fn.compose Uuid.to_string create ()))
in
let per_unit ~(input : single_source) ~output_path =
run_on_specifications
specifications
@ -301,13 +303,13 @@ let run
(fun input specification ->
process_single_source
matcher
?metasyntax
~sequential
~omega
~fast_offset_conversion
~substitute_in_place
~verbose
~timeout
?metasyntax
?fresh
match_configuration
input
specification)
@ -324,7 +326,6 @@ let run
| Some interactive_review ->
run_interactive
specifications
sequential
matcher
omega
fast_offset_conversion
@ -343,18 +344,19 @@ let execute
?substitute_in_place
?timeout
?metasyntax
?fresh
?(configuration = Matchers.Configuration.create ())
source
specification =
process_single_source
matcher
~sequential:true
~omega:false
~fast_offset_conversion:false
?substitute_in_place
~verbose:false
?timeout
?metasyntax
?fresh
configuration
source
specification

3
lib/pipeline/pipeline.mli
View File

@ -8,13 +8,13 @@ type output =
val process_single_source
: (module Matchers.Matcher.S)
-> ?sequential:bool
-> ?omega:bool
-> ?fast_offset_conversion:bool
-> ?substitute_in_place:bool
-> ?verbose:bool
-> ?timeout:int
-> ?metasyntax:Matchers.Metasyntax.t
-> ?fresh:(unit -> string)
-> Matchers.Configuration.t
-> single_source
-> Specification.t
@ -25,6 +25,7 @@ val execute
-> ?substitute_in_place:bool
-> ?timeout:int
-> ?metasyntax:Matchers.Metasyntax.t
-> ?fresh:(unit -> string)
-> ?configuration:Matchers.Configuration.t
-> single_source
-> Specification.t

19
lib/rewriter/rewrite.ml
View File

@ -8,7 +8,7 @@ let debug =
|> Option.is_some
let substitute_match_contexts ?sequential ?metasyntax (matches: Match.t list) source replacements =
let substitute_match_contexts ?fresh ?metasyntax (matches: Match.t list) source replacements =
if debug then Format.printf "Matches: %d | Replacements: %d@." (List.length matches) (List.length replacements);
let rewrite_template, environment =
List.fold2_exn
@ -18,7 +18,8 @@ let substitute_match_contexts ?sequential ?metasyntax (matches: Match.t list) so
({ environment = _match_environment; _ } as match_)
{ replacement_content; _ } ->
(* create a hole in the rewrite template based on this match context *)
let hole_id, rewrite_template = Rewrite_template.of_match_context ?metasyntax match_ ~source:rewrite_template in
let sub_fresh = Option.map fresh ~f:(fun f -> fun () -> ("sub_" ^ f ())) in (* ensure custom fresh function is unique for substition. *)
let hole_id, rewrite_template = Rewrite_template.of_match_context ?metasyntax ?fresh:sub_fresh match_ ~source:rewrite_template in
if debug then Format.printf "Hole: %s in %s@." hole_id rewrite_template;
(* add this match context replacement to the environment *)
let accumulator_environment = Environment.add accumulator_environment hole_id replacement_content in
@ -27,7 +28,7 @@ let substitute_match_contexts ?sequential ?metasyntax (matches: Match.t list) so
in
if debug then Format.printf "Env:@.%s" (Environment.to_string environment);
if debug then Format.printf "Rewrite in:@.%s@." rewrite_template;
let rewritten_source = Rewrite_template.substitute ?metasyntax ?sequential rewrite_template environment |> fst in
let rewritten_source = Rewrite_template.substitute ?metasyntax ?fresh rewrite_template environment |> fst in
let offsets = Rewrite_template.get_offsets_for_holes ?metasyntax rewrite_template (Environment.vars environment) in
if debug then
Format.printf "Replacements: %d | Offsets 1: %d@." (List.length replacements) (List.length offsets);
@ -52,11 +53,11 @@ let substitute_match_contexts ?sequential ?metasyntax (matches: Match.t list) so
(b) its replacement context (to calculate the range)
(c) an environment of values that are updated to reflect their relative offset in the rewrite template
*)
let substitute_in_rewrite_template ?sequential ?metasyntax rewrite_template ({ environment; _ } : Match.t) =
let substitute_in_rewrite_template ?fresh ?metasyntax rewrite_template ({ environment; _ } : Match.t) =
let replacement_content, vars_substituted_for =
Rewrite_template.substitute
?metasyntax
?sequential
?fresh
rewrite_template
environment
in
@ -92,20 +93,20 @@ let substitute_in_rewrite_template ?sequential ?metasyntax rewrite_template ({ e
}
}
let all ?source ?metasyntax ?sequential ~rewrite_template matches : result option =
let all ?source ?metasyntax ?fresh ~rewrite_template matches : result option =
if List.is_empty matches then None else
match source with
(* in-place substitution *)
| Some source ->
let matches : Match.t list = List.rev matches in
matches
|> List.map ~f:(substitute_in_rewrite_template ?metasyntax ?sequential rewrite_template)
|> substitute_match_contexts ?metasyntax ?sequential matches source
|> List.map ~f:(substitute_in_rewrite_template ?metasyntax ?fresh rewrite_template)
|> substitute_match_contexts ?metasyntax ?fresh matches source
|> Option.some
(* no in place substitution, emit result separated by newlines *)
| None ->
matches
|> List.map ~f:(substitute_in_rewrite_template ?metasyntax ?sequential rewrite_template)
|> List.map ~f:(substitute_in_rewrite_template ?metasyntax ?fresh rewrite_template)
|> List.map ~f:(fun { replacement_content; _ } -> replacement_content)
|> String.concat ~sep:"\n"
|> (fun rewritten_source -> { rewritten_source; in_place_substitutions = [] })

2
lib/rewriter/rewrite.mli
View File

@ -3,7 +3,7 @@
val all
: ?source:string
-> ?metasyntax:Matchers.Metasyntax.t
-> ?sequential:bool
-> ?fresh:(unit -> string)
-> rewrite_template:string
-> Match.t list
-> Replacement.result option

36
lib/rewriter/rewrite_template.ml
View File

@ -7,8 +7,18 @@ let debug =
Sys.getenv "DEBUG_COMBY"
|> Option.is_some
let uuid_for_id_counter = ref 0
let uuid_for_sub_counter = ref 0
let counter =
let uuid_for_id_counter = ref 0 in
fun () ->
uuid_for_id_counter := !uuid_for_id_counter + 1;
Format.sprintf "%d" !uuid_for_id_counter
let sub_counter =
let uuid_for_sub_counter = ref 0 in
fun () ->
uuid_for_sub_counter := !uuid_for_sub_counter + 1;
Format.sprintf "sub_%d" !uuid_for_sub_counter
let replacement_sentinel metasyntax =
let open Matchers.Metasyntax in
@ -40,7 +50,7 @@ let parse_first_label ?(metasyntax = Matchers.Metasyntax.default_metasyntax) tem
| Ok label -> List.find_map label ~f:ident
| Error _ -> None
let substitute_fresh ?(metasyntax = Matchers.Metasyntax.default_metasyntax) ?(sequential = false) template =
let substitute_fresh ?(metasyntax = Matchers.Metasyntax.default_metasyntax) ?(fresh = counter) template =
let label_table = String.Table.create () in
let template_ref = ref template in
let current_label_ref = ref (parse_first_label ~metasyntax !template_ref) in
@ -50,18 +60,7 @@ let substitute_fresh ?(metasyntax = Matchers.Metasyntax.default_metasyntax) ?(se
match String.Table.find label_table label with
| Some id -> id
| None ->
let id =
if sequential then
(
uuid_for_id_counter := !uuid_for_id_counter + 1;
Format.sprintf "%d" !uuid_for_id_counter
)
else
(
let uuid = Uuid_unix.(Fn.compose Uuid.to_string create ()) in
String.suffix uuid 12
)
in
let id = fresh () in
if String.(label <> "") then
String.Table.add_exn label_table ~key:label ~data:id;
id
@ -83,9 +82,9 @@ let formats_of_metasyntax metasyntax =
| _ -> None)
|> List.concat
let substitute ?(metasyntax = Matchers.Metasyntax.default_metasyntax) ?sequential template env =
let substitute ?(metasyntax = Matchers.Metasyntax.default_metasyntax) ?fresh template env =
let substitution_formats = formats_of_metasyntax metasyntax.syntax in
let template = substitute_fresh ~metasyntax ?sequential template in
let template = substitute_fresh ~metasyntax ?fresh template in
Environment.vars env
|> List.fold ~init:(template, []) ~f:(fun (acc, vars) variable ->
match Environment.lookup env variable with
@ -101,6 +100,7 @@ let substitute ?(metasyntax = Matchers.Metasyntax.default_metasyntax) ?sequentia
let of_match_context
?(metasyntax = Matchers.Metasyntax.default_metasyntax)
?(fresh = sub_counter)
{ range =
{ match_start = { offset = start_index; _ }
; match_end = { offset = end_index; _ } }
@ -115,7 +115,7 @@ let of_match_context
String.slice source 0 start_index
in
let after_part = String.slice source end_index (String.length source) in
let hole_id = Uuid_unix.(Fn.compose Uuid.to_string create ()) in
let hole_id = fresh () in
let left, right = replacement_sentinel metasyntax in
let rewrite_template = String.concat [before_part; left; hole_id; right; after_part] in
hole_id, rewrite_template

14
lib/rewriter/rewrite_template.mli
View File

@ -1,16 +1,16 @@
open Matchers
open Match
(** if [sequential] is true, then substitute the pattern :[id()] starting at 1,
and incrementing subsequent IDs. if [sequential] is false, then substitute
the pattern :[id()] with a fresh hex string based on the last 48-bit part of
a UUID v3 identifier *)
val substitute_fresh : ?metasyntax:Metasyntax.t -> ?sequential:bool -> string -> string
(** if [fresh] is set, then substitute the pattern :[id()] starting at 1, and
incrementing subsequent IDs. If [fresh] is unset, then by default substitute
the pattern :[id()] starting at 1, and increment for each occurence of
:[id()], left to right. *)
val substitute_fresh : ?metasyntax:Metasyntax.t -> ?fresh:(unit -> string) -> string -> string
(** substitute returns the result and variables substituted for *)
val substitute : ?metasyntax:Metasyntax.t -> ?sequential:bool -> string -> Environment.t -> (string * string list)
val substitute : ?metasyntax:Metasyntax.t -> ?fresh:(unit -> string) -> string -> Environment.t -> (string * string list)
val of_match_context : ?metasyntax:Metasyntax.t -> Match.t -> source:string -> (string * string)
val of_match_context : ?metasyntax:Metasyntax.t -> ?fresh:(unit -> string) -> Match.t -> source:string -> (string * string)
val get_offsets_for_holes : ?metasyntax:Metasyntax.t -> string -> string list -> (string * int) list