82ca880 · bat · git.iko.soy

git.iko.soy/bat

on master help sign in

Handle non-unicode characters in the preprocessor

82ca880sharkdp7 years ago

assets/syntaxes/show-nonprintable.sublime-syntax+4

compare with master

unifiedsplit

@@ -25,3 +25,7 @@ contexts:

25 scope: entity.other.attribute-name.show-nonprintable.escape

26 - match: "␈"

27 scope: entity.other.attribute-name.show-nonprintable.backspace

28+ - match: "\\\\x[A-Z0-9][A-Z0-9]"

29+ scope: comment.block.show-nonprintable.backspace

30+ - match: "\\\\u\\{[a-z0-9]+\\}"

31+ scope: comment.block.show-nonprintable.backspace

src/clap_app.rs+2−1

@@ -172,7 +172,8 @@ pub fn build_app(interactive_output: bool) -> ClapApp<'static, 'static> {

172 .help("Show non-printable characters (space, tab, newline, ..).")

173 .long_help(

174 "Show non-printable characters like space, tab or newline. \

175− Use '--tabs' to control the width of the tab-placeholders.",

175+ Use '--tabs' to control the width of the tab-placeholders. \

176+ This option can also be used to print binary files.",

176 ),

177 ),

177 )

178 )

178 .arg(

179 .arg(

src/preprocessor.rs+101−27

@@ -33,41 +33,115 @@ pub fn expand_tabs(line: &str, width: usize, cursor: &mut usize) -> String {

33 buffer

34}

36+fn try_parse_utf8_char(input: &[u8]) -> Option<(char, usize)> {

37+ let str_from_utf8 = |seq| std::str::from_utf8(seq).ok();

38+

39+ let decoded = None

40+ .or(input.get(0..1).and_then(str_from_utf8).map(|c| (c, 1)))

41+ .or(input.get(0..2).and_then(str_from_utf8).map(|c| (c, 2)))

42+ .or(input.get(0..3).and_then(str_from_utf8).map(|c| (c, 3)))

43+ .or(input.get(0..4).and_then(str_from_utf8).map(|c| (c, 4)));

44+

45+ let decoded_char = decoded.map(|(seq, n)| (seq.chars().next().unwrap(), n));

46+

47+ decoded_char

48+}

49+

36−pub fn replace_nonprintable(input: &str, tab_width: usize) -> String {

50+pub fn replace_nonprintable(input: &[u8], tab_width: usize) -> String {

37 let mut output = String::new();

51 let mut output = String::new();

39 let tab_width = if tab_width == 0 { 4 } else { tab_width };

53 let tab_width = if tab_width == 0 { 4 } else { tab_width };

41− for chr in input.chars() {

55+ let mut idx = 0;

56+ let len = input.len();

57+ while idx < len {

58+ if let Some((chr, skip_ahead)) = try_parse_utf8_char(&input[idx..]) {

59+ idx += skip_ahead;

60+

42− match chr {

61+ match chr {

43− // space

62+ // space

44− ' ' => output.push('•'),

63+ ' ' => output.push('•'),

45− // tab

64+ // tab

46− '\t' => {

65+ '\t' => {

47− if tab_width == 1 {

66+ if tab_width == 1 {

48− output.push('↹');

67+ output.push('↹');

49− } else {

68+ } else {

50− output.push('├');

69+ output.push('├');

51− output.push_str(&"─".repeat(tab_width - 2));

70+ output.push_str(&"─".repeat(tab_width - 2));

52− output.push('┤');

71+ output.push('┤');

72+ }

73+ }

74+ // line feed

75+ '\x0A' => output.push('␊'),

76+ // carriage return

77+ '\x0D' => output.push('␍'),

78+ // null

79+ '\x00' => output.push('␀'),

80+ // bell

81+ '\x07' => output.push('␇'),

82+ // backspace

83+ '\x08' => output.push('␈'),

84+ // escape

85+ '\x1B' => output.push('␛'),

86+ // printable ASCII

87+ c if c.is_ascii_alphanumeric()

88+ || c.is_ascii_punctuation()

89+ || c.is_ascii_graphic() =>

90+ {

91+ output.push(c)

53 }

92 }

93+ // everything else

94+ c => output.push_str(&c.escape_unicode().collect::<String>()),

54 }

95 }

55− // line feed

56− '\x0A' => output.push('␊'),

57− // carriage return

58− '\x0D' => output.push('␍'),

59− // null

60− '\x00' => output.push('␀'),

61− // bell

62− '\x07' => output.push('␇'),

63− // backspace

64− '\x08' => output.push('␈'),

65− // escape

66− '\x1B' => output.push('␛'),

67− // anything else

96+ } else {

68− _ => output.push(chr),

97+ output.push_str(&format!("\\x{:02X}", input[idx]));

98+ idx += 1;

69 }

99 }

70 }

100 }

101

72 output

102 output

73}

103}

104+

105+#[test]

106+fn test_try_parse_utf8_char() {

107+ assert_eq!(try_parse_utf8_char(&[0x20]), Some((' ', 1)));

108+ assert_eq!(try_parse_utf8_char(&[0x20, 0x20]), Some((' ', 1)));

109+ assert_eq!(try_parse_utf8_char(&[0x20, 0xef]), Some((' ', 1)));

110+

111+ assert_eq!(try_parse_utf8_char(&[0x00]), Some(('\x00', 1)));

112+ assert_eq!(try_parse_utf8_char(&[0x1b]), Some(('\x1b', 1)));

113+

114+ assert_eq!(try_parse_utf8_char(&[0xc3, 0xa4]), Some(('ä', 2)));

115+ assert_eq!(try_parse_utf8_char(&[0xc3, 0xa4, 0xef]), Some(('ä', 2)));

116+ assert_eq!(try_parse_utf8_char(&[0xc3, 0xa4, 0x20]), Some(('ä', 2)));

117+

118+ assert_eq!(try_parse_utf8_char(&[0xe2, 0x82, 0xac]), Some(('€', 3)));

119+ assert_eq!(

120+ try_parse_utf8_char(&[0xe2, 0x82, 0xac, 0xef]),

121+ Some(('€', 3))

122+ );

123+ assert_eq!(

124+ try_parse_utf8_char(&[0xe2, 0x82, 0xac, 0x20]),

125+ Some(('€', 3))

126+ );

127+

128+ assert_eq!(try_parse_utf8_char(&[0xe2, 0x88, 0xb0]), Some(('∰', 3)));

129+

130+ assert_eq!(

131+ try_parse_utf8_char(&[0xf0, 0x9f, 0x8c, 0x82]),

132+ Some(('🌂', 4))

133+ );

134+ assert_eq!(

135+ try_parse_utf8_char(&[0xf0, 0x9f, 0x8c, 0x82, 0xef]),

136+ Some(('🌂', 4))

137+ );

138+ assert_eq!(

139+ try_parse_utf8_char(&[0xf0, 0x9f, 0x8c, 0x82, 0x20]),

140+ Some(('🌂', 4))

141+ );

142+

143+ assert_eq!(try_parse_utf8_char(&[]), None);

144+ assert_eq!(try_parse_utf8_char(&[0xef]), None);

145+ assert_eq!(try_parse_utf8_char(&[0xef, 0x20]), None);

146+ assert_eq!(try_parse_utf8_char(&[0xf0, 0xf0]), None);

147+}

src/printer.rs+19−27

@@ -1,4 +1,3 @@

1−use std::ascii;

2use std::io::Write;

1use std::io::Write;

3use std::vec::Vec;

2use std::vec::Vec;

@@ -232,7 +231,8 @@ impl<'a> Printer for InteractivePrinter<'a> {

232 writeln!(

231 writeln!(

233 handle,

232 handle,

234 "{}: Binary content from {} will not be printed to the terminal \

233 "{}: Binary content from {} will not be printed to the terminal \

235− (but will be present if the output of 'bat' is piped).",

234+ (but will be present if the output of 'bat' is piped). You can use 'bat -A' \

235+ to show the binary file contents.",

236 Yellow.paint("[bat warning]"),

237 input

238 )?;

@@ -281,7 +281,7 @@ impl<'a> Printer for InteractivePrinter<'a> {

281 )?;

282

283 if self.config.output_components.grid() {

284− if self.content_type.map_or(false, |c| c.is_text()) {

284+ if self.content_type.map_or(false, |c| c.is_text()) || self.config.show_nonprintable {

285 self.print_horizontal_line(handle, '┼')?;

286 } else {

287 self.print_horizontal_line(handle, '┴')?;

@@ -292,7 +292,8 @@ impl<'a> Printer for InteractivePrinter<'a> {

292 }

293

294 fn print_footer(&mut self, handle: &mut dyn Write) -> Result<()> {

295− if self.config.output_components.grid() && self.content_type.map_or(false, |c| c.is_text())

295+ if self.config.output_components.grid()

296+ && (self.content_type.map_or(false, |c| c.is_text()) || self.config.show_nonprintable)

296 {

297 {

297 self.print_horizontal_line(handle, '┴')

298 self.print_horizontal_line(handle, '┴')

298 } else {

299 } else {

@@ -331,32 +332,23 @@ impl<'a> Printer for InteractivePrinter<'a> {

331 line_number: usize,

332 line_number: usize,

332 line_buffer: &[u8],

333 line_buffer: &[u8],

333 ) -> Result<()> {

334 ) -> Result<()> {

335+ let line = if self.config.show_nonprintable {

336+ replace_nonprintable(&line_buffer, self.config.tab_width)

337+ } else {

334− let mut line = match self.content_type {

338+ match self.content_type {

335− None => {

339+ Some(ContentType::BINARY) | None => {

336− return Ok(());

340+ return Ok(());

341+ }

342+ Some(ContentType::UTF_16LE) => UTF_16LE

343+ .decode(&line_buffer, DecoderTrap::Replace)

344+ .map_err(|_| "Invalid UTF-16LE")?,

345+ Some(ContentType::UTF_16BE) => UTF_16BE

346+ .decode(&line_buffer, DecoderTrap::Replace)

347+ .map_err(|_| "Invalid UTF-16BE")?,

348+ _ => String::from_utf8_lossy(&line_buffer).to_string(),

337 }

349 }

338− Some(ContentType::BINARY) => String::from_utf8(

339− line_buffer

340− .as_ref()

341− .iter()

342− .map(|b| ascii::escape_default(*b))

343− .flatten()

344− .collect(),

345− )

346− .unwrap(),

347− Some(ContentType::UTF_16LE) => UTF_16LE

348− .decode(&line_buffer, DecoderTrap::Replace)

349− .map_err(|_| "Invalid UTF-16LE")?,

350− Some(ContentType::UTF_16BE) => UTF_16BE

351− .decode(&line_buffer, DecoderTrap::Replace)

352− .map_err(|_| "Invalid UTF-16BE")?,

353− _ => String::from_utf8_lossy(&line_buffer).to_string(),

354 };

350 };

355

351

356− if self.config.show_nonprintable {

357− line = replace_nonprintable(&line, self.config.tab_width);

358− }

359−

360 let regions = {

352 let regions = {

361 let highlighter = match self.highlighter {

353 let highlighter = match self.highlighter {

362 Some(ref mut highlighter) => highlighter,

354 Some(ref mut highlighter) => highlighter,