@@ -5,6 +5,7 @@ use futures_util::future::BoxFuture;
55use futures_util::FutureExt;
66use helix_core::auto_pairs::AutoPairs;
77use helix_core::doc_formatter::TextFormat;
8+use helix_core::encoding::Encoding;
89use helix_core::syntax::Highlight;
910use helix_core::text_annotations::{InlineAnnotation, TextAnnotations};
1011use helix_core::Range;
@@ -130,6 +131,7 @@ pub struct Document {
130131
131132 path: Option<PathBuf>,
132133 encoding: &'static encoding::Encoding,
134+ has_bom: bool,
133135
134136 pub restore_cursor: bool,
135137
@@ -277,16 +279,104 @@ impl fmt::Debug for DocumentInlayHintsId {
277279 }
278280}
279281
282+enum Encoder {
283+ Utf16Be,
284+ Utf16Le,
285+ EncodingRs(encoding::Encoder),
286+}
287+
288+impl Encoder {
289+ fn from_encoding(encoding: &'static encoding::Encoding) -> Self {
290+ if encoding == encoding::UTF_16BE {
291+ Self::Utf16Be
292+ } else if encoding == encoding::UTF_16LE {
293+ Self::Utf16Le
294+ } else {
295+ Self::EncodingRs(encoding.new_encoder())
296+ }
297+ }
298+
299+ fn encode_from_utf8(
300+ &mut self,
301+ src: &str,
302+ dst: &mut [u8],
303+ is_empty: bool,
304+ ) -> (encoding::CoderResult, usize, usize) {
305+ if src.is_empty() {
306+ return (encoding::CoderResult::InputEmpty, 0, 0);
307+ }
308+ let mut write_to_buf = |convert: fn(u16) -> [u8; 2]| {
309+ let to_write = src.char_indices().map(|(indice, char)| {
310+ let mut encoded: [u16; 2] = [0, 0];
311+ (
312+ indice,
313+ char.encode_utf16(&mut encoded)
314+ .iter_mut()
315+ .flat_map(|char| convert(*char))
316+ .collect::<Vec<u8>>(),
317+ )
318+ });
319+
320+ let mut total_written = 0usize;
321+
322+ for (indice, utf16_bytes) in to_write {
323+ let character_size = utf16_bytes.len();
324+
325+ if dst.len() <= (total_written + character_size) {
326+ return (encoding::CoderResult::OutputFull, indice, total_written);
327+ }
328+
329+ for character in utf16_bytes {
330+ dst[total_written] = character;
331+ total_written += 1;
332+ }
333+ }
334+
335+ (encoding::CoderResult::InputEmpty, src.len(), total_written)
336+ };
337+
338+ match self {
339+ Self::Utf16Be => write_to_buf(u16::to_be_bytes),
340+ Self::Utf16Le => write_to_buf(u16::to_le_bytes),
341+ Self::EncodingRs(encoder) => {
342+ let (code_result, read, written, ..) = encoder.encode_from_utf8(src, dst, is_empty);
343+
344+ (code_result, read, written)
345+ }
346+ }
347+ }
348+}
349+
350+// Apply BOM if encoding permit it, return the number of bytes written at the start of buf
351+fn apply_bom(encoding: &'static encoding::Encoding, buf: &mut [u8; BUF_SIZE]) -> usize {
352+ if encoding == encoding::UTF_8 {
353+ buf[0] = 0xef;
354+ buf[1] = 0xbb;
355+ buf[2] = 0xbf;
356+ 3
357+ } else if encoding == encoding::UTF_16BE {
358+ buf[0] = 0xfe;
359+ buf[1] = 0xff;
360+ 2
361+ } else if encoding == encoding::UTF_16LE {
362+ buf[0] = 0xff;
363+ buf[1] = 0xfe;
364+ 2
365+ } else {
366+ 0
367+ }
368+}
369+
280370// The documentation and implementation of this function should be up-to-date with
281371// its sibling function, `to_writer()`.
282372//
283373/// Decodes a stream of bytes into UTF-8, returning a `Rope` and the
284−/// encoding it was decoded as. The optional `encoding` parameter can
285−/// be used to override encoding auto-detection.
374+/// encoding it was decoded as with BOM information. The optional `encoding`
375+/// parameter can be used to override encoding auto-detection.
286376pub fn from_reader<R: std::io::Read + ?Sized>(
287377 reader: &mut R,
288− encoding: Option<&'static encoding::Encoding>,
289−) -> Result<(Rope, &'static encoding::Encoding), Error> {
378+ encoding: Option<&'static Encoding>,
379+) -> Result<(Rope, &'static Encoding, bool), Error> {
290380 // These two buffers are 8192 bytes in size each and are used as
291381 // intermediaries during the decoding process. Text read into `buf`
292382 // from `reader` is decoded into `buf_out` as UTF-8. Once either
@@ -296,25 +386,32 @@ pub fn from_reader<R: std::io::Read + ?Sized>(
296386 let mut buf_out = [0u8; BUF_SIZE];
297387 let mut builder = RopeBuilder::new();
298388
299− // By default, the encoding of the text is auto-detected via the
300− // `chardetng` crate which requires sample data from the reader.
389+ // By default, the encoding of the text is auto-detected by
390+ // `encoding_rs` for_bom, and if it fails, from `chardetng`
391+ // crate which requires sample data from the reader.
301392 // As a manual override to this auto-detection is possible, the
302393 // same data is read into `buf` to ensure symmetry in the upcoming
303394 // loop.
304− let (encoding, mut decoder, mut slice, mut is_empty) = {
395+ let (encoding, has_bom, mut decoder, mut slice, mut is_empty) = {
305396 let read = reader.read(&mut buf)?;
306397 let is_empty = read == 0;
307− let encoding = encoding.unwrap_or_else(|| {
308− let mut encoding_detector = chardetng::EncodingDetector::new();
309− encoding_detector.feed(&buf, is_empty);
310− encoding_detector.guess(None, true)
311− });
398+ let (encoding, has_bom) = encoding
399+ .map(|encoding| (encoding, false))
400+ .or_else(|| {
401+ encoding::Encoding::for_bom(&buf).map(|(encoding, _bom_size)| (encoding, true))
402+ })
403+ .unwrap_or_else(|| {
404+ let mut encoding_detector = chardetng::EncodingDetector::new();
405+ encoding_detector.feed(&buf, is_empty);
406+ (encoding_detector.guess(None, true), false)
407+ });
408+
312409 let decoder = encoding.new_decoder();
313410
314411 // If the amount of bytes read from the reader is less than
315412 // `buf.len()`, it is undesirable to read the bytes afterwards.
316413 let slice = &buf[..read];
317− (encoding, decoder, slice, is_empty)
414+ (encoding, has_bom, decoder, slice, is_empty)
318415 };
319416
320417 // `RopeBuilder::append()` expects a `&str`, so this is the "real"
@@ -382,7 +479,7 @@ pub fn from_reader<R: std::io::Read + ?Sized>(
382479 is_empty = read == 0;
383480 }
384481 let rope = builder.finish();
385− Ok((rope, encoding))
482+ Ok((rope, encoding, has_bom))
386483}
387484
388485// The documentation and implementation of this function should be up-to-date with
@@ -393,7 +490,7 @@ pub fn from_reader<R: std::io::Read + ?Sized>(
393490/// replacement characters may appear in the encoded text.
394491pub async fn to_writer<'a, W: tokio::io::AsyncWriteExt + Unpin + ?Sized>(
395492 writer: &'a mut W,
396− encoding: &'static encoding::Encoding,
493+ encoding_with_bom_info: (&'static Encoding, bool),
397494 rope: &'a Rope,
398495) -> Result<(), Error> {
399496 // Text inside a `Rope` is stored as non-contiguous blocks of data called
@@ -402,13 +499,22 @@ pub async fn to_writer<'a, W: tokio::io::AsyncWriteExt + Unpin + ?Sized>(
402499 // determined by filtering the iterator to remove all empty chunks and then
403500 // appending an empty chunk to it. This is valuable for detecting when all
404501 // chunks in the `Rope` have been iterated over in the subsequent loop.
502+ let (encoding, has_bom) = encoding_with_bom_info;
503+
405504 let iter = rope
406505 .chunks()
407506 .filter(|c| !c.is_empty())
408507 .chain(std::iter::once(""));
409508 let mut buf = [0u8; BUF_SIZE];
410− let mut encoder = encoding.new_encoder();
411− let mut total_written = 0usize;
509+
510+ let mut total_written = if has_bom {
511+ apply_bom(encoding, &mut buf)
512+ } else {
513+ 0
514+ };
515+
516+ let mut encoder = Encoder::from_encoding(encoding);
517+
412518 for chunk in iter {
413519 let is_empty = chunk.is_empty();
414520 let mut total_read = 0usize;
@@ -449,6 +555,7 @@ pub async fn to_writer<'a, W: tokio::io::AsyncWriteExt + Unpin + ?Sized>(
449555 break;
450556 }
451557 }
558+
452559 Ok(())
453560}
454561
@@ -466,10 +573,10 @@ use url::Url;
466573impl Document {
467574 pub fn from(
468575 text: Rope,
469− encoding: Option<&'static encoding::Encoding>,
576+ encoding_with_bom_info: Option<(&'static Encoding, bool)>,
470577 config: Arc<dyn DynAccess<Config>>,
471578 ) -> Self {
472− let encoding = encoding.unwrap_or(encoding::UTF_8);
579+ let (encoding, has_bom) = encoding_with_bom_info.unwrap_or((encoding::UTF_8, false));
473580 let changes = ChangeSet::new(&text);
474581 let old_state = None;
475582
@@ -477,6 +584,7 @@ impl Document {
477584 id: DocumentId::default(),
478585 path: None,
479586 encoding,
587+ has_bom,
480588 text,
481589 selections: HashMap::default(),
482590 inlay_hints: HashMap::default(),
@@ -511,21 +619,21 @@ impl Document {
511619 /// overwritten with the `encoding` parameter.
512620 pub fn open(
513621 path: &Path,
514− encoding: Option<&'static encoding::Encoding>,
622+ encoding: Option<&'static Encoding>,
515623 config_loader: Option<Arc<syntax::Loader>>,
516624 config: Arc<dyn DynAccess<Config>>,
517625 ) -> Result<Self, Error> {
518626 // Open the file if it exists, otherwise assume it is a new file (and thus empty).
519− let (rope, encoding) = if path.exists() {
627+ let (rope, encoding, has_bom) = if path.exists() {
520628 let mut file =
521629 std::fs::File::open(path).context(format!("unable to open {:?}", path))?;
522630 from_reader(&mut file, encoding)?
523631 } else {
524632 let encoding = encoding.unwrap_or(encoding::UTF_8);
525− (Rope::from(DEFAULT_LINE_ENDING.as_str()), encoding)
633+ (Rope::from(DEFAULT_LINE_ENDING.as_str()), encoding, false)
526634 };
527635
528− let mut doc = Self::from(rope, Some(encoding), config);
636+ let mut doc = Self::from(rope, Some((encoding, has_bom)), config);
529637
530638 // set the path and try detecting the language
531639 doc.set_path(Some(path))?;
@@ -576,7 +684,7 @@ impl Document {
576684 })?;
577685 {
578686 let mut stdin = process.stdin.take().ok_or(FormatterError::BrokenStdin)?;
579− to_writer(&mut stdin, encoding::UTF_8, &text)
687+ to_writer(&mut stdin, (encoding::UTF_8, false), &text)
580688 .await
581689 .map_err(|_| FormatterError::BrokenStdin)?;
582690 }
@@ -688,8 +796,7 @@ impl Document {
688796 let current_rev = self.get_current_revision();
689797 let doc_id = self.id();
690798
691− let encoding = self.encoding;
692−
799+ let encoding_with_bom_info = (self.encoding, self.has_bom);
693800 let last_saved_time = self.last_saved_time;
694801
695802 // We encode the file according to the `Document`'s encoding.
@@ -718,7 +825,7 @@ impl Document {
718825 }
719826
720827 let mut file = File::create(&path).await?;
721− to_writer(&mut file, encoding, &text).await?;
828+ to_writer(&mut file, encoding_with_bom_info, &text).await?;
722829
723830 let event = DocumentSavedEvent {
724831 revision: current_rev,
@@ -776,7 +883,7 @@ impl Document {
776883 provider_registry: &DiffProviderRegistry,
777884 redraw_handle: RedrawHandle,
778885 ) -> Result<(), Error> {
779− let encoding = &self.encoding;
886+ let encoding = self.encoding;
780887 let path = self
781888 .path()
782889 .filter(|path| path.exists())
@@ -810,13 +917,16 @@ impl Document {
810917
811918 /// Sets the [`Document`]'s encoding with the encoding correspondent to `label`.
812919 pub fn set_encoding(&mut self, label: &str) -> Result<(), Error> {
813− self.encoding = encoding::Encoding::for_label(label.as_bytes())
814− .ok_or_else(|| anyhow!("unknown encoding"))?;
920+ let encoding =
921+ Encoding::for_label(label.as_bytes()).ok_or_else(|| anyhow!("unknown encoding"))?;
922+
923+ self.encoding = encoding;
924+
815925 Ok(())
816926 }
817927
818928 /// Returns the [`Document`]'s current encoding.
819− pub fn encoding(&self) -> &'static encoding::Encoding {
929+ pub fn encoding(&self) -> &'static Encoding {
820930 self.encoding
821931 }
822932
@@ -1280,7 +1390,7 @@ impl Document {
12801390
12811391 /// Intialize/updates the differ for this document with a new base.
12821392 pub fn set_diff_base(&mut self, diff_base: Vec<u8>, redraw_handle: RedrawHandle) {
1283− if let Ok((diff_base, _)) = from_reader(&mut diff_base.as_slice(), Some(self.encoding)) {
1393+ if let Ok((diff_base, ..)) = from_reader(&mut diff_base.as_slice(), Some(self.encoding)) {
12841394 if let Some(differ) = &self.diff_handle {
12851395 differ.update_diff_base(diff_base);
12861396 return;
@@ -1724,7 +1834,7 @@ mod test {
17241834 assert!(ref_path.exists());
17251835
17261836 let mut file = std::fs::File::open(path).unwrap();
1727− let text = from_reader(&mut file, Some(encoding))
1837+ let text = from_reader(&mut file, Some(encoding.into()))
17281838 .unwrap()
17291839 .0
17301840 .to_string();
@@ -1750,7 +1860,7 @@ mod test {
17501860
17511861 let text = Rope::from_str(&std::fs::read_to_string(path).unwrap());
17521862 let mut buf: Vec<u8> = Vec::new();
1753− helix_lsp::block_on(to_writer(&mut buf, encoding, &text)).unwrap();
1863+ helix_lsp::block_on(to_writer(&mut buf, (encoding, false), &text)).unwrap();
17541864
17551865 let expectation = std::fs::read(ref_path).unwrap();
17561866 assert_eq!(buf, expectation);