From ed872d4b073d2e9886f7b2deb5639d17c87ae295 Mon Sep 17 00:00:00 2001 From: Mattias Wadman Date: Wed, 20 Nov 2024 11:50:28 +0100 Subject: [PATCH] yaml,toml: Add indent option for to_{toml,yaml} --- doc/usage.md | 48 ++++++++++++++++-------------- format/toml/testdata/indent.fqtest | 29 ++++++++++++++++++ format/toml/toml.go | 13 ++++++-- format/toml/toml.jq | 2 ++ format/yaml/testdata/indent.fqtest | 29 ++++++++++++++++++ format/yaml/yaml.go | 21 +++++++++---- format/yaml/yaml.jq | 2 ++ 7 files changed, 114 insertions(+), 30 deletions(-) create mode 100644 format/toml/testdata/indent.fqtest create mode 100644 format/yaml/testdata/indent.fqtest diff --git a/doc/usage.md b/doc/usage.md index e7f30256..e1e079a9 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -682,18 +682,20 @@ name: Afghanistan zip> ^D ``` -- `from_xml`/`from_xml($opts)` Parse XML into jq value.
- `{seq: true}` preserve element ordering if more than one sibling.
- `{array: true}` use nested `[name, attributes, children]` arrays to represent elements. Attributes will be `null` if none and children will be `[]` if none, this is to make it easier to work with as the array as 3 values. `to_xml` does not require this.
+- `from_xml`/`from_xml($opts)` Parse XML into jq value. `$opts` are: + - `{seq: true}` preserve element ordering if more than one sibling.
+ - `{array: true}` use nested `[name, attributes, children]` arrays to represent elements. Attributes will be `null` if none and children will be `[]` if none, this is to make it easier to work with as the array as 3 values. `to_xml` does not require this.
- `from_html`/`from_html($opts)` Parse HTML into jq value.
Similar to `from_xml` but parses html5 in non-script mode. Will always have a `html` root with `head` and `body` elements.
- `{array: true}` use nested arrays to represent elements.
- `{seq: true}` preserve element ordering if more than one sibling.
+ `$opts` are: + - `{array: true}` use nested arrays to represent elements.
+ - `{seq: true}` preserve element ordering if more than one sibling.
- `to_xml`/`to_xml($opts})` Serialize jq value into XML.
- `{indent: number}` indent child elements.
Assumes object representation if input is an object, and nested arrays if input is an array.
Will automatically add a root `doc` element if jq value has more then one root element.
If a `#seq` is found on at least one element all siblings will be sort by sequence number. Attributes are always sorted.
+ `$opts` are: + - `{indent: number}` indent child elements.
XML elements can be represented as jq value in two ways, as objects (inspired by [mxj](https://github.com/clbanning/mxj) and [xml.com's Converting Between XML and JSON ](https://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html)) or nested arrays. Both representations are lossy and might lose ordering of elements, text nodes and comments. In object representation `from_xml`, `from_html` and `to_xml` support `{seq:true}` option to parse/serialize `{"#seq"=}` attributes to preserve element sibling ordering. @@ -801,12 +803,11 @@ zip> ^D JSON and jq-flavoured JSON - `fromjson` Parse JSON into jq value. -- `tojson`/`tojson($opt)` Serialize jq value into JSON.
- `{indent: number}` indent array/object values.
+- `tojson`/`tojson($opts)` Serialize jq value into JSON. `$opts` are: + - `{indent: number}` Indent depth. - `from_jq` Parse jq-flavoured JSON into jq value. -- `to_jq`/`to_jq($opt)` Serialize jq value into jq-flavoured JSON
- `{indent: number}` indent array/object values.
- jq-flavoured JSON has optional key quotes, `#` comments and can have trailing comma in arrays. +- `to_jq`/`to_jq($opts)` Serialize jq value into jq-flavoured JSON. jq-flavoured JSON has optional key quotes, `#` comments and can have trailing comma in arrays. `$opts` are: + - `{indent: number}` Indent depth. - `from_jsonl` Parse JSON lines into jq array. - `to_jsonl` Serialize jq array into JSONL. @@ -814,19 +815,22 @@ Note that `fromjson` and `tojson` use different naming conventions as they origi YAML - `from_yaml` Parse YAML into jq value. -- `to_yaml` Serialize jq value into YAML. +- `to_yaml`/`to_yaml($opts)` Serialize jq value into YAML. `$opts` are: + - `{indent: number}` Indent depth. TOML - `from_toml` Parse TOML into jq value. -- `to_toml` Serialize jq value into TOML. +- `to_toml`/`to_toml($opts)` Serialize jq value into TOML. `$opts` are: + - `{indent: number}` Indent depth. CSV - `from_csv`/`from_cvs($opts)` Parse CSV into jq value.
- `{comma: string}` field separator, default ",".
- `{comment: string}` comment line character, default "#".
- To work with tab separated values you can use `fromcvs({comma: "\t"})` or `fq -d csv -o 'comma="\t"'` -- `to_csv`/`to_csv($opts)` Serialize jq value into CSV.
- `{comma: string}` field separator, default ",".
+ To work with tab separated values you can use `fromcvs({comma: "\t"})` or `fq -d csv -o 'comma="\t"'`
+ `$opts` are: + - `{comma: string}` field separator, default ",".
+ - `{comment: string}` comment line character, default "#".
+- `to_csv`/`to_csv($opts)` Serialize jq value into CSV. `$opts` are: + - `{comma: string}` field separator, default ",".
XML encoding - `from_xmlentities` Decode XML entities. @@ -862,10 +866,10 @@ URL parts and XML encodings Binary encodings like hex and base64 - `from_hex` Decode hex string to binary. - `to_hex` Encode binary into hex string. -- `from_base64`/`from_base64($opts)` Decode base64 encodings into binary.
- `{encoding:string}` encoding variant: `std` (default), `url`, `rawstd` or `rawurl` -- `to_base64`/`to_base64($opts)` Encode binary into base64 encodings.
- `{encoding:string}` encoding variant: `std` (default), `url`, `rawstd` or `rawurl` +- `from_base64`/`from_base64($opts)` Decode base64 encodings into binary. `$opts` are: + - `{encoding:string}` encoding variant: `std` (default), `url`, `rawstd` or `rawurl` +- `to_base64`/`to_base64($opts)` Encode binary into base64 encodings. `$opts` are: + - `{encoding:string}` encoding variant: `std` (default), `url`, `rawstd` or `rawurl` Hash functions - `to_md4` Hash binary using md4. diff --git a/format/toml/testdata/indent.fqtest b/format/toml/testdata/indent.fqtest new file mode 100644 index 00000000..6f79ce85 --- /dev/null +++ b/format/toml/testdata/indent.fqtest @@ -0,0 +1,29 @@ +$ fq -rn "{a: {b: 123}} | to_toml" +[a] + b = 123 + +$ fq -rn "{a: {b: 123}} | to_toml({indent: range(8)})" +[a] +b = 123 + +[a] + b = 123 + +[a] + b = 123 + +[a] + b = 123 + +[a] + b = 123 + +[a] + b = 123 + +[a] + b = 123 + +[a] + b = 123 + diff --git a/format/toml/toml.go b/format/toml/toml.go index 2d70fb72..2a937dfd 100644 --- a/format/toml/toml.go +++ b/format/toml/toml.go @@ -6,6 +6,7 @@ import ( "embed" "fmt" "io" + "strings" "unicode/utf8" "github.com/BurntSushi/toml" @@ -31,7 +32,7 @@ func init() { Functions: []string{"_todisplay"}, }) interp.RegisterFS(tomlFS) - interp.RegisterFunc0("to_toml", toTOML) + interp.RegisterFunc1("_to_toml", toTOML) } func decodeTOMLSeekFirstValidRune(br io.ReadSeeker) error { @@ -88,13 +89,19 @@ func decodeTOML(d *decode.D) any { return nil } -func toTOML(_ *interp.Interp, c any) any { +type ToTOMLOpts struct { + Indent int `default:"2"` // 2 is default for BurntSushi/toml +} + +func toTOML(_ *interp.Interp, c any, opts ToTOMLOpts) any { if c == nil { return gojqx.FuncTypeError{Name: "to_toml", V: c} } b := &bytes.Buffer{} - if err := toml.NewEncoder(b).Encode(gojqx.Normalize(c)); err != nil { + e := toml.NewEncoder(b) + e.Indent = strings.Repeat(" ", opts.Indent) + if err := e.Encode(gojqx.Normalize(c)); err != nil { return err } return b.String() diff --git a/format/toml/toml.jq b/format/toml/toml.jq index a15e7f61..7f6c822e 100644 --- a/format/toml/toml.jq +++ b/format/toml/toml.jq @@ -1 +1,3 @@ def _toml__todisplay: tovalue; +def to_toml($opts): _to_toml($opts); +def to_toml: _to_toml(null); diff --git a/format/yaml/testdata/indent.fqtest b/format/yaml/testdata/indent.fqtest new file mode 100644 index 00000000..6dd8d178 --- /dev/null +++ b/format/yaml/testdata/indent.fqtest @@ -0,0 +1,29 @@ +$ fq -rn "{a: {b: 123}} | to_yaml" +a: + b: 123 + +$ fq -rn "{a: {b: 123}} | to_yaml({indent: range(8)})" +a: + b: 123 + +a: + b: 123 + +a: + b: 123 + +a: + b: 123 + +a: + b: 123 + +a: + b: 123 + +a: + b: 123 + +a: + b: 123 + diff --git a/format/yaml/yaml.go b/format/yaml/yaml.go index 981469ee..107fad4d 100644 --- a/format/yaml/yaml.go +++ b/format/yaml/yaml.go @@ -3,6 +3,7 @@ package yaml // TODO: yaml type eval? walk eval? import ( + "bytes" "embed" "errors" "io" @@ -30,7 +31,7 @@ func init() { Functions: []string{"_todisplay"}, }) interp.RegisterFS(yamlFS) - interp.RegisterFunc0("to_yaml", toYAML) + interp.RegisterFunc1("_to_yaml", toYAML) } func decodeYAML(d *decode.D) any { @@ -61,10 +62,20 @@ func decodeYAML(d *decode.D) any { return nil } -func toYAML(_ *interp.Interp, c any) any { - b, err := yaml.Marshal(gojqx.Normalize(c)) - if err != nil { +type ToYAMLOpts struct { + Indent int `default:"4"` // 4 is default for gopkg.in/yaml.v3 +} + +func toYAML(_ *interp.Interp, c any, opts ToYAMLOpts) any { + b := &bytes.Buffer{} + e := yaml.NewEncoder(b) + // yaml.SetIndent panics if < 0 + if opts.Indent >= 0 { + e.SetIndent(opts.Indent) + } + if err := e.Encode(gojqx.Normalize(c)); err != nil { return err } - return string(b) + + return b.String() } diff --git a/format/yaml/yaml.jq b/format/yaml/yaml.jq index 3acee7c8..5cbcea44 100644 --- a/format/yaml/yaml.jq +++ b/format/yaml/yaml.jq @@ -1 +1,3 @@ def _yaml__todisplay: tovalue; +def to_yaml($opts): _to_yaml($opts); +def to_yaml: _to_yaml(null);