b0b3f45 · helix · git.iko.soy

git.iko.soy/helix

on master help sign in

Conserve BOM and properly support UTF16 (#6497)

b0b3f45Alexis-Lapierre4 years ago

helix-term/src/commands.rs+2−1

compare with master

unifiedsplit

@@ -5111,7 +5111,8 @@ async fn shell_impl_async(

5111 let output = if let Some(mut stdin) = process.stdin.take() {

5112 let input_task = tokio::spawn(async move {

5113 if let Some(input) = input {

5114− helix_view::document::to_writer(&mut stdin, encoding::UTF_8, &input).await?;

5114+ helix_view::document::to_writer(&mut stdin, (encoding::UTF_8, false), &input)

5115+ .await?;

5115 }

5116 }

5116 Ok::<_, anyhow::Error>(())

5117 Ok::<_, anyhow::Error>(())

5117 });

5118 });

helix-term/tests/test/commands/write.rs+38

@@ -407,3 +407,41 @@ async fn test_write_fail_new_path() -> anyhow::Result<()> {

407

408 Ok(())

409}

410+

411+#[tokio::test(flavor = "multi_thread")]

412+async fn test_write_utf_bom_file() -> anyhow::Result<()> {

413+ // "ABC" with utf8 bom

414+ const UTF8_FILE: [u8; 6] = [0xef, 0xbb, 0xbf, b'A', b'B', b'C'];

415+

416+ // "ABC" in UTF16 with bom

417+ const UTF16LE_FILE: [u8; 8] = [0xff, 0xfe, b'A', 0x00, b'B', 0x00, b'C', 0x00];

418+ const UTF16BE_FILE: [u8; 8] = [0xfe, 0xff, 0x00, b'A', 0x00, b'B', 0x00, b'C'];

419+

420+ edit_file_with_content(&UTF8_FILE).await?;

421+ edit_file_with_content(&UTF16LE_FILE).await?;

422+ edit_file_with_content(&UTF16BE_FILE).await?;

423+

424+ Ok(())

425+}

426+

427+async fn edit_file_with_content(file_content: &[u8]) -> anyhow::Result<()> {

428+ let mut file = tempfile::NamedTempFile::new()?;

429+

430+ file.as_file_mut().write_all(&file_content)?;

431+

432+ helpers::test_key_sequence(

433+ &mut helpers::AppBuilder::new().build()?,

434+ Some(&format!(":o {}<ret>:x<ret>", file.path().to_string_lossy())),

435+ None,

436+ true,

437+ )

438+ .await?;

439+

440+ file.rewind()?;

441+ let mut new_file_content: Vec<u8> = Vec::new();

442+ file.read_to_end(&mut new_file_content)?;

443+

444+ assert_eq!(file_content, new_file_content);

445+

446+ Ok(())

447+}

helix-view/src/document.rs+144−34

@@ -5,6 +5,7 @@ use futures_util::future::BoxFuture;

5use futures_util::FutureExt;

6use helix_core::auto_pairs::AutoPairs;

7use helix_core::doc_formatter::TextFormat;

8+use helix_core::encoding::Encoding;

8use helix_core::syntax::Highlight;

9use helix_core::syntax::Highlight;

9use helix_core::text_annotations::{InlineAnnotation, TextAnnotations};

10use helix_core::text_annotations::{InlineAnnotation, TextAnnotations};

10use helix_core::Range;

11use helix_core::Range;

@@ -130,6 +131,7 @@ pub struct Document {

130

131

131 path: Option<PathBuf>,

132 path: Option<PathBuf>,

132 encoding: &'static encoding::Encoding,

133 encoding: &'static encoding::Encoding,

134+ has_bom: bool,

133

135

134 pub restore_cursor: bool,

136 pub restore_cursor: bool,

135

137

@@ -277,16 +279,104 @@ impl fmt::Debug for DocumentInlayHintsId {

277 }

279 }

278}

280}

279

281

282+enum Encoder {

283+ Utf16Be,

284+ Utf16Le,

285+ EncodingRs(encoding::Encoder),

286+}

287+

288+impl Encoder {

289+ fn from_encoding(encoding: &'static encoding::Encoding) -> Self {

290+ if encoding == encoding::UTF_16BE {

291+ Self::Utf16Be

292+ } else if encoding == encoding::UTF_16LE {

293+ Self::Utf16Le

294+ } else {

295+ Self::EncodingRs(encoding.new_encoder())

296+ }

297+ }

298+

299+ fn encode_from_utf8(

300+ &mut self,

301+ src: &str,

302+ dst: &mut [u8],

303+ is_empty: bool,

304+ ) -> (encoding::CoderResult, usize, usize) {

305+ if src.is_empty() {

306+ return (encoding::CoderResult::InputEmpty, 0, 0);

307+ }

308+ let mut write_to_buf = |convert: fn(u16) -> [u8; 2]| {

309+ let to_write = src.char_indices().map(|(indice, char)| {

310+ let mut encoded: [u16; 2] = [0, 0];

311+ (

312+ indice,

313+ char.encode_utf16(&mut encoded)

314+ .iter_mut()

315+ .flat_map(|char| convert(*char))

316+ .collect::<Vec<u8>>(),

317+ )

318+ });

319+

320+ let mut total_written = 0usize;

321+

322+ for (indice, utf16_bytes) in to_write {

323+ let character_size = utf16_bytes.len();

324+

325+ if dst.len() <= (total_written + character_size) {

326+ return (encoding::CoderResult::OutputFull, indice, total_written);

327+ }

328+

329+ for character in utf16_bytes {

330+ dst[total_written] = character;

331+ total_written += 1;

332+ }

333+ }

334+

335+ (encoding::CoderResult::InputEmpty, src.len(), total_written)

336+ };

337+

338+ match self {

339+ Self::Utf16Be => write_to_buf(u16::to_be_bytes),

340+ Self::Utf16Le => write_to_buf(u16::to_le_bytes),

341+ Self::EncodingRs(encoder) => {

342+ let (code_result, read, written, ..) = encoder.encode_from_utf8(src, dst, is_empty);

343+

344+ (code_result, read, written)

345+ }

346+ }

347+ }

348+}

349+

350+// Apply BOM if encoding permit it, return the number of bytes written at the start of buf

351+fn apply_bom(encoding: &'static encoding::Encoding, buf: &mut [u8; BUF_SIZE]) -> usize {

352+ if encoding == encoding::UTF_8 {

353+ buf[0] = 0xef;

354+ buf[1] = 0xbb;

355+ buf[2] = 0xbf;

356+ 3

357+ } else if encoding == encoding::UTF_16BE {

358+ buf[0] = 0xfe;

359+ buf[1] = 0xff;

360+ 2

361+ } else if encoding == encoding::UTF_16LE {

362+ buf[0] = 0xff;

363+ buf[1] = 0xfe;

364+ 2

365+ } else {

366+ 0

367+ }

368+}

369+

280// The documentation and implementation of this function should be up-to-date with

370// The documentation and implementation of this function should be up-to-date with

281// its sibling function, `to_writer()`.

371// its sibling function, `to_writer()`.

282//

372//

283/// Decodes a stream of bytes into UTF-8, returning a `Rope` and the

373/// Decodes a stream of bytes into UTF-8, returning a `Rope` and the

284−/// encoding it was decoded as. The optional `encoding` parameter can

374+/// encoding it was decoded as with BOM information. The optional `encoding`

285−/// be used to override encoding auto-detection.

375+/// parameter can be used to override encoding auto-detection.

286pub fn from_reader<R: std::io::Read + ?Sized>(

376pub fn from_reader<R: std::io::Read + ?Sized>(

287 reader: &mut R,

377 reader: &mut R,

288− encoding: Option<&'static encoding::Encoding>,

378+ encoding: Option<&'static Encoding>,

289−) -> Result<(Rope, &'static encoding::Encoding), Error> {

379+) -> Result<(Rope, &'static Encoding, bool), Error> {

290 // These two buffers are 8192 bytes in size each and are used as

380 // These two buffers are 8192 bytes in size each and are used as

291 // intermediaries during the decoding process. Text read into `buf`

381 // intermediaries during the decoding process. Text read into `buf`

292 // from `reader` is decoded into `buf_out` as UTF-8. Once either

382 // from `reader` is decoded into `buf_out` as UTF-8. Once either

@@ -296,25 +386,32 @@ pub fn from_reader<R: std::io::Read + ?Sized>(

296 let mut buf_out = [0u8; BUF_SIZE];

386 let mut buf_out = [0u8; BUF_SIZE];

297 let mut builder = RopeBuilder::new();

387 let mut builder = RopeBuilder::new();

298

388

299− // By default, the encoding of the text is auto-detected via the

389+ // By default, the encoding of the text is auto-detected by

390+ // `encoding_rs` for_bom, and if it fails, from `chardetng`

300− // `chardetng` crate which requires sample data from the reader.

391+ // crate which requires sample data from the reader.

301 // As a manual override to this auto-detection is possible, the

392 // As a manual override to this auto-detection is possible, the

302 // same data is read into `buf` to ensure symmetry in the upcoming

393 // same data is read into `buf` to ensure symmetry in the upcoming

303 // loop.

394 // loop.

304− let (encoding, mut decoder, mut slice, mut is_empty) = {

395+ let (encoding, has_bom, mut decoder, mut slice, mut is_empty) = {

305 let read = reader.read(&mut buf)?;

396 let read = reader.read(&mut buf)?;

306 let is_empty = read == 0;

397 let is_empty = read == 0;

307− let encoding = encoding.unwrap_or_else(|| {

398+ let (encoding, has_bom) = encoding

399+ .map(|encoding| (encoding, false))

400+ .or_else(|| {

401+ encoding::Encoding::for_bom(&buf).map(|(encoding, _bom_size)| (encoding, true))

402+ })

403+ .unwrap_or_else(|| {

308− let mut encoding_detector = chardetng::EncodingDetector::new();

404+ let mut encoding_detector = chardetng::EncodingDetector::new();

309− encoding_detector.feed(&buf, is_empty);

405+ encoding_detector.feed(&buf, is_empty);

310− encoding_detector.guess(None, true)

406+ (encoding_detector.guess(None, true), false)

311− });

407+ });

408+

312 let decoder = encoding.new_decoder();

409 let decoder = encoding.new_decoder();

313

410

314 // If the amount of bytes read from the reader is less than

411 // If the amount of bytes read from the reader is less than

315 // `buf.len()`, it is undesirable to read the bytes afterwards.

412 // `buf.len()`, it is undesirable to read the bytes afterwards.

316 let slice = &buf[..read];

413 let slice = &buf[..read];

317− (encoding, decoder, slice, is_empty)

414+ (encoding, has_bom, decoder, slice, is_empty)

318 };

415 };

319

416

320 // `RopeBuilder::append()` expects a `&str`, so this is the "real"

417 // `RopeBuilder::append()` expects a `&str`, so this is the "real"

@@ -382,7 +479,7 @@ pub fn from_reader<R: std::io::Read + ?Sized>(

382 is_empty = read == 0;

479 is_empty = read == 0;

383 }

480 }

384 let rope = builder.finish();

481 let rope = builder.finish();

385− Ok((rope, encoding))

482+ Ok((rope, encoding, has_bom))

386}

483}

387

484

388// The documentation and implementation of this function should be up-to-date with

485// The documentation and implementation of this function should be up-to-date with

@@ -393,7 +490,7 @@ pub fn from_reader<R: std::io::Read + ?Sized>(

393/// replacement characters may appear in the encoded text.

490/// replacement characters may appear in the encoded text.

394pub async fn to_writer<'a, W: tokio::io::AsyncWriteExt + Unpin + ?Sized>(

491pub async fn to_writer<'a, W: tokio::io::AsyncWriteExt + Unpin + ?Sized>(

395 writer: &'a mut W,

492 writer: &'a mut W,

396− encoding: &'static encoding::Encoding,

493+ encoding_with_bom_info: (&'static Encoding, bool),

397 rope: &'a Rope,

494 rope: &'a Rope,

398) -> Result<(), Error> {

495) -> Result<(), Error> {

399 // Text inside a `Rope` is stored as non-contiguous blocks of data called

496 // Text inside a `Rope` is stored as non-contiguous blocks of data called

@@ -402,13 +499,22 @@ pub async fn to_writer<'a, W: tokio::io::AsyncWriteExt + Unpin + ?Sized>(

402 // determined by filtering the iterator to remove all empty chunks and then

499 // determined by filtering the iterator to remove all empty chunks and then

403 // appending an empty chunk to it. This is valuable for detecting when all

500 // appending an empty chunk to it. This is valuable for detecting when all

404 // chunks in the `Rope` have been iterated over in the subsequent loop.

501 // chunks in the `Rope` have been iterated over in the subsequent loop.

502+ let (encoding, has_bom) = encoding_with_bom_info;

503+

405 let iter = rope

504 let iter = rope

406 .chunks()

505 .chunks()

407 .filter(|c| !c.is_empty())

506 .filter(|c| !c.is_empty())

408 .chain(std::iter::once(""));

507 .chain(std::iter::once(""));

409 let mut buf = [0u8; BUF_SIZE];

508 let mut buf = [0u8; BUF_SIZE];

410− let mut encoder = encoding.new_encoder();

509+

411− let mut total_written = 0usize;

510+ let mut total_written = if has_bom {

511+ apply_bom(encoding, &mut buf)

512+ } else {

513+ 0

514+ };

515+

516+ let mut encoder = Encoder::from_encoding(encoding);

517+

412 for chunk in iter {

518 for chunk in iter {

413 let is_empty = chunk.is_empty();

519 let is_empty = chunk.is_empty();

414 let mut total_read = 0usize;

520 let mut total_read = 0usize;

@@ -449,6 +555,7 @@ pub async fn to_writer<'a, W: tokio::io::AsyncWriteExt + Unpin + ?Sized>(

449 break;

555 break;

450 }

556 }

451 }

557 }

558+

452 Ok(())

559 Ok(())

453}

560}

454

561

@@ -466,10 +573,10 @@ use url::Url;

466impl Document {

573impl Document {

467 pub fn from(

574 pub fn from(

468 text: Rope,

575 text: Rope,

469− encoding: Option<&'static encoding::Encoding>,

576+ encoding_with_bom_info: Option<(&'static Encoding, bool)>,

470 config: Arc<dyn DynAccess<Config>>,

577 config: Arc<dyn DynAccess<Config>>,

471 ) -> Self {

578 ) -> Self {

472− let encoding = encoding.unwrap_or(encoding::UTF_8);

579+ let (encoding, has_bom) = encoding_with_bom_info.unwrap_or((encoding::UTF_8, false));

473 let changes = ChangeSet::new(&text);

580 let changes = ChangeSet::new(&text);

474 let old_state = None;

581 let old_state = None;

475

582

@@ -477,6 +584,7 @@ impl Document {

477 id: DocumentId::default(),

584 id: DocumentId::default(),

478 path: None,

585 path: None,

479 encoding,

586 encoding,

587+ has_bom,

480 text,

588 text,

481 selections: HashMap::default(),

589 selections: HashMap::default(),

482 inlay_hints: HashMap::default(),

590 inlay_hints: HashMap::default(),

@@ -511,21 +619,21 @@ impl Document {

511 /// overwritten with the `encoding` parameter.

619 /// overwritten with the `encoding` parameter.

512 pub fn open(

620 pub fn open(

513 path: &Path,

621 path: &Path,

514− encoding: Option<&'static encoding::Encoding>,

622+ encoding: Option<&'static Encoding>,

515 config_loader: Option<Arc<syntax::Loader>>,

623 config_loader: Option<Arc<syntax::Loader>>,

516 config: Arc<dyn DynAccess<Config>>,

624 config: Arc<dyn DynAccess<Config>>,

517 ) -> Result<Self, Error> {

625 ) -> Result<Self, Error> {

518 // Open the file if it exists, otherwise assume it is a new file (and thus empty).

626 // Open the file if it exists, otherwise assume it is a new file (and thus empty).

519− let (rope, encoding) = if path.exists() {

627+ let (rope, encoding, has_bom) = if path.exists() {

520 let mut file =

628 let mut file =

521 std::fs::File::open(path).context(format!("unable to open {:?}", path))?;

629 std::fs::File::open(path).context(format!("unable to open {:?}", path))?;

522 from_reader(&mut file, encoding)?

630 from_reader(&mut file, encoding)?

523 } else {

631 } else {

524 let encoding = encoding.unwrap_or(encoding::UTF_8);

632 let encoding = encoding.unwrap_or(encoding::UTF_8);

525− (Rope::from(DEFAULT_LINE_ENDING.as_str()), encoding)

633+ (Rope::from(DEFAULT_LINE_ENDING.as_str()), encoding, false)

526 };

634 };

527

635

528− let mut doc = Self::from(rope, Some(encoding), config);

636+ let mut doc = Self::from(rope, Some((encoding, has_bom)), config);

529

637

530 // set the path and try detecting the language

638 // set the path and try detecting the language

531 doc.set_path(Some(path))?;

639 doc.set_path(Some(path))?;

@@ -576,7 +684,7 @@ impl Document {

576 })?;

684 })?;

577 {

685 {

578 let mut stdin = process.stdin.take().ok_or(FormatterError::BrokenStdin)?;

686 let mut stdin = process.stdin.take().ok_or(FormatterError::BrokenStdin)?;

579− to_writer(&mut stdin, encoding::UTF_8, &text)

687+ to_writer(&mut stdin, (encoding::UTF_8, false), &text)

580 .await

688 .await

581 .map_err(|_| FormatterError::BrokenStdin)?;

689 .map_err(|_| FormatterError::BrokenStdin)?;

582 }

690 }

@@ -688,8 +796,7 @@ impl Document {

688 let current_rev = self.get_current_revision();

796 let current_rev = self.get_current_revision();

689 let doc_id = self.id();

797 let doc_id = self.id();

690

798

691− let encoding = self.encoding;

799+ let encoding_with_bom_info = (self.encoding, self.has_bom);

692−

693 let last_saved_time = self.last_saved_time;

800 let last_saved_time = self.last_saved_time;

694

801

695 // We encode the file according to the `Document`'s encoding.

802 // We encode the file according to the `Document`'s encoding.

@@ -718,7 +825,7 @@ impl Document {

718 }

825 }

719

826

720 let mut file = File::create(&path).await?;

827 let mut file = File::create(&path).await?;

721− to_writer(&mut file, encoding, &text).await?;

828+ to_writer(&mut file, encoding_with_bom_info, &text).await?;

722

829

723 let event = DocumentSavedEvent {

830 let event = DocumentSavedEvent {

724 revision: current_rev,

831 revision: current_rev,

@@ -776,7 +883,7 @@ impl Document {

776 provider_registry: &DiffProviderRegistry,

883 provider_registry: &DiffProviderRegistry,

777 redraw_handle: RedrawHandle,

884 redraw_handle: RedrawHandle,

778 ) -> Result<(), Error> {

885 ) -> Result<(), Error> {

779− let encoding = &self.encoding;

886+ let encoding = self.encoding;

780 let path = self

887 let path = self

781 .path()

888 .path()

782 .filter(|path| path.exists())

889 .filter(|path| path.exists())

@@ -810,13 +917,16 @@ impl Document {

810

917

811 /// Sets the [`Document`]'s encoding with the encoding correspondent to `label`.

918 /// Sets the [`Document`]'s encoding with the encoding correspondent to `label`.

812 pub fn set_encoding(&mut self, label: &str) -> Result<(), Error> {

919 pub fn set_encoding(&mut self, label: &str) -> Result<(), Error> {

813− self.encoding = encoding::Encoding::for_label(label.as_bytes())

920+ let encoding =

814− .ok_or_else(|| anyhow!("unknown encoding"))?;

921+ Encoding::for_label(label.as_bytes()).ok_or_else(|| anyhow!("unknown encoding"))?;

922+

923+ self.encoding = encoding;

924+

815 Ok(())

925 Ok(())

816 }

926 }

817

927

818 /// Returns the [`Document`]'s current encoding.

928 /// Returns the [`Document`]'s current encoding.

819− pub fn encoding(&self) -> &'static encoding::Encoding {

929+ pub fn encoding(&self) -> &'static Encoding {

820 self.encoding

930 self.encoding

821 }

931 }

822

932

@@ -1280,7 +1390,7 @@ impl Document {

1280

1390

1281 /// Intialize/updates the differ for this document with a new base.

1391 /// Intialize/updates the differ for this document with a new base.

1282 pub fn set_diff_base(&mut self, diff_base: Vec<u8>, redraw_handle: RedrawHandle) {

1392 pub fn set_diff_base(&mut self, diff_base: Vec<u8>, redraw_handle: RedrawHandle) {

1283− if let Ok((diff_base, _)) = from_reader(&mut diff_base.as_slice(), Some(self.encoding)) {

1393+ if let Ok((diff_base, ..)) = from_reader(&mut diff_base.as_slice(), Some(self.encoding)) {

1284 if let Some(differ) = &self.diff_handle {

1394 if let Some(differ) = &self.diff_handle {

1285 differ.update_diff_base(diff_base);

1395 differ.update_diff_base(diff_base);

1286 return;

1396 return;

@@ -1724,7 +1834,7 @@ mod test {

1724 assert!(ref_path.exists());

1834 assert!(ref_path.exists());

1725

1835

1726 let mut file = std::fs::File::open(path).unwrap();

1836 let mut file = std::fs::File::open(path).unwrap();

1727− let text = from_reader(&mut file, Some(encoding))

1837+ let text = from_reader(&mut file, Some(encoding.into()))

1728 .unwrap()

1838 .unwrap()

1729 .0

1839 .0

1730 .to_string();

1840 .to_string();

@@ -1750,7 +1860,7 @@ mod test {

1750

1860

1751 let text = Rope::from_str(&std::fs::read_to_string(path).unwrap());

1861 let text = Rope::from_str(&std::fs::read_to_string(path).unwrap());

1752 let mut buf: Vec<u8> = Vec::new();

1862 let mut buf: Vec<u8> = Vec::new();

1753− helix_lsp::block_on(to_writer(&mut buf, encoding, &text)).unwrap();

1863+ helix_lsp::block_on(to_writer(&mut buf, (encoding, false), &text)).unwrap();

1754

1864

1755 let expectation = std::fs::read(ref_path).unwrap();

1865 let expectation = std::fs::read(ref_path).unwrap();

1756 assert_eq!(buf, expectation);

1866 assert_eq!(buf, expectation);

helix-view/src/editor.rs+2−2

@@ -1305,10 +1305,10 @@ impl Editor {

1305 }

1306

1307 pub fn new_file_from_stdin(&mut self, action: Action) -> Result<DocumentId, Error> {

1308− let (rope, encoding) = crate::document::from_reader(&mut stdin(), None)?;

1308+ let (rope, encoding, has_bom) = crate::document::from_reader(&mut stdin(), None)?;

1309 Ok(self.new_file_from_document(

1310 action,

1311− Document::from(rope, Some(encoding), self.config.clone()),

1311+ Document::from(rope, Some((encoding, has_bom)), self.config.clone()),

1312 ))

1313 }

1314