Merge pull request #666 from wader/bits-bytes-binary

bits,bytes: Behave as binary instead of raw decode value

format/bits/testdata/test.fqtest

@@ -1,14 +1,22 @@
 /hello:
 hello
-$ fq -d bits '., .size, .[8:-8]' hello
+$ fq -d bits '., .size, .[8:-8], scan("he|lo")' hello
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|68 65 6c 6c 6f 0a|                             |hello.|         |.: raw bits (bits)
+0x0|68 65 6c 6c 6f 0a|                             |hello.|         |.: raw bits 0x0-0x5.7 (6)
 48
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
 0x0|   65 6c 6c 6f                                 | ello           |.: raw bits 0x1-0x4.7 (4)
-$ fq -d bytes '., .size, .[1:-1]' hello
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|68 65 6c 6c 6f 0a|                             |hello.|         |.: raw bits (bytes)
+0x0|68 65                                          |he              |.: raw bits 0x0-0x1.7 (2)
+   |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
+0x0|         6c 6f                                 |   lo           |.: raw bits 0x3-0x4.7 (2)
+$ fq -d bytes '., .size, .[1:-1], scan("he|lo")' hello
+   |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
+0x0|68 65 6c 6c 6f 0a|                             |hello.|         |.: raw bits 0x0-0x5.7 (6)
 6
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
 0x0|   65 6c 6c 6f                                 | ello           |.: raw bits 0x1-0x4.7 (4)
+   |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
+0x0|68 65                                          |he              |.: raw bits 0x0-0x1.7 (2)
+   |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
+0x0|         6c 6f                                 |   lo           |.: raw bits 0x3-0x4.7 (2)

pkg/interp/binary.go

@@ -299,6 +299,7 @@ func (Binary) ExtKeys() []string {
 		"size",
 		"start",
 		"stop",
+		"unit",
 	}
 }
 
@@ -368,6 +369,8 @@ func (b Binary) JQValueKey(name string) any {
 			stopUnits++
 		}
 		return new(big.Int).SetInt64(stopUnits)
+	case "unit":
+		return b.unit
 	}
 	return nil
 }

pkg/interp/binary.jq

@@ -18,6 +18,14 @@ def _binary_or_orig(bfn; fn):
   if _exttype == "binary" then bfn
   else fn
   end;
+def _bytes_or_orig(bfn; fn):
+  _binary_or_orig(
+    # convert to bytes if bits
+    ( if .unit != 8 then tobytesrange end
+    | bfn
+    );
+    fn
+  );
 
 def _orig_explode: explode;
 def explode: _binary_or_orig([.[range(.size)]]; _orig_explode);
@@ -38,8 +46,8 @@ def _splits_binary($regex; $flags):
       end
     )
   );
-def splits($val): _binary_or_orig(_splits_binary($val; "g"); _orig_splits($val));
+def splits($val): _bytes_or_orig(_splits_binary($val; "g"); _orig_splits($val));
-def splits($regex; $flags): _binary_or_orig(_splits_binary($regex; "g"+$flags); _orig_splits($regex; $flags));
+def splits($regex; $flags): _bytes_or_orig(_splits_binary($regex; "g"+$flags); _orig_splits($regex; $flags));
 
 def _orig_split($val): split($val);
 def _orig_split($regex; $flags): split($regex; $flags);
@@ -53,13 +61,13 @@ def _test_binary($regex; $flags):
   ( isempty(_match_binary($regex; $flags))
   | not
   );
-def test($val): _binary_or_orig(_test_binary($val; ""); _orig_test($val));
+def test($val): _bytes_or_orig(_test_binary($val; ""); _orig_test($val));
-def test($regex; $flags): _binary_or_orig(_test_binary($regex; $flags); _orig_test($regex; $flags));
+def test($regex; $flags): _bytes_or_orig(_test_binary($regex; $flags); _orig_test($regex; $flags));
 
 def _orig_match($val): match($val);
 def _orig_match($regex; $flags): match($regex; $flags);
-def match($val): _binary_or_orig(_match_binary($val; ""); _orig_match($val));
+def match($val): _bytes_or_orig(_match_binary($val; ""); _orig_match($val));
-def match($regex; $flags): _binary_or_orig(_match_binary($regex; $flags); _orig_match($regex; $flags));
+def match($regex; $flags): _bytes_or_orig(_match_binary($regex; $flags); _orig_match($regex; $flags));
 
 def _orig_capture($val): capture($val);
 def _orig_capture($regex; $flags): capture($regex; $flags);
@@ -74,8 +82,8 @@ def _capture_binary($regex; $flags):
     )
   | from_entries
   );
-def capture($val): _binary_or_orig(_capture_binary($val; ""); _orig_capture($val));
+def capture($val): _bytes_or_orig(_capture_binary($val; ""); _orig_capture($val));
-def capture($regex; $flags): _binary_or_orig(_capture_binary($regex; $flags); _orig_capture($regex; $flags));
+def capture($regex; $flags): _bytes_or_orig(_capture_binary($regex; $flags); _orig_capture($regex; $flags));
 
 def _orig_scan($val): scan($val);
 def _orig_scan($regex; $flags): scan($regex; $flags);
@@ -84,5 +92,5 @@ def _scan_binary($regex; $flags):
   | _match_binary($regex; $flags)
   | $b[.offset:.offset+.length]
   );
-def scan($val): _binary_or_orig(_scan_binary($val; "g"); _orig_scan($val));
+def scan($val): _bytes_or_orig(_scan_binary($val; "g"); _orig_scan($val));
-def scan($regex; $flags): _binary_or_orig(_scan_binary($regex; "g"+$flags); _orig_scan($regex; $flags));
+def scan($regex; $flags): _bytes_or_orig(_scan_binary($regex; "g"+$flags); _orig_scan($regex; $flags));

pkg/interp/decode.go

@@ -434,10 +434,7 @@ func makeDecodeValueOut(dv *decode.Value, kind decodeValueKind, out any) any {
 				decodeValueBase: decodeValueBase{dv: dv},
 			}
 		case Binary:
-			return decodeValue{
+			return vvv
-				JQValue:         vvv,
-				decodeValueBase: decodeValueBase{dv: dv},
-			}
 
 		default:
 			panic(fmt.Sprintf("unreachable vv %#+v", vvv))

pkg/interp/repl.jq

@@ -152,7 +152,11 @@ def _prompt($opts):
           + if $c._error then "!" else "" end
           )
         else
-          ($c | type)
+          ( $c
+          | if _is_decode_value then type
+            else (_exttype // type)
+            end
+          )
         end
       ) | _ansi_if($opts; "prompt_value")
     end;

pkg/interp/testdata/repl.fqtest

@@ -108,7 +108,7 @@ json!> ^D
 $ fq -i -n '"[]" | json'
 json> ^D
 $ fq -i -d bytes . test.mp3
-bytes> ^D
+binary> ^D
 $ fq -n repl
 exitcode: 3
 stderr: