@@ -11,7 +11,7 @@ use helix_core::{
11 history::History,
11 history::History,
12 line_ending::auto_detect_line_ending,
12 line_ending::auto_detect_line_ending,
13 syntax::{self, LanguageConfiguration},
13 syntax::{self, LanguageConfiguration},
14− ChangeSet, Diagnostic, LineEnding, Rope, Selection, State, Syntax, Transaction,
14+ ChangeSet, Diagnostic, LineEnding, Rope, RopeBuilder, Selection, State, Syntax, Transaction,
15 DEFAULT_LINE_ENDING,
15 DEFAULT_LINE_ENDING,
@@ -19,6 +19,8 @@ use crate::{DocumentId, Theme, ViewId};
20use std::collections::HashMap;
20use std::collections::HashMap;
22+const BUF_SIZE: usize = 8192;
22#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
24#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
23pub enum Mode {
25pub enum Mode {
@@ -39,6 +41,7 @@ pub struct Document {
39 pub(crate) selections: HashMap<ViewId, Selection>,
41 pub(crate) selections: HashMap<ViewId, Selection>,
41 path: Option<PathBuf>,
43 path: Option<PathBuf>,
44+ encoding: &'static encoding_rs::Encoding,
43 /// Current editing mode.
46 /// Current editing mode.
44 pub mode: Mode,
47 pub mode: Mode,
@@ -78,6 +81,7 @@ impl fmt::Debug for Document {
78 .field("text", &self.text)
81 .field("text", &self.text)
79 .field("selections", &self.selections)
82 .field("selections", &self.selections)
80 .field("path", &self.path)
83 .field("path", &self.path)
84+ .field("encoding", &self.encoding)
81 .field("mode", &self.mode)
85 .field("mode", &self.mode)
82 .field("restore_cursor", &self.restore_cursor)
86 .field("restore_cursor", &self.restore_cursor)
83 .field("syntax", &self.syntax)
87 .field("syntax", &self.syntax)
@@ -116,6 +120,173 @@ impl FromStr for Mode {
123+// The documentation and implementation of this function should be up-to-date with
124+// its sibling function, `to_writer()`.
126+/// Decodes a stream of bytes into UTF-8, returning a `Rope` and the
127+/// encoding it was decoded as. The optional `encoding` parameter can
128+/// be used to override encoding auto-detection.
129+pub fn from_reader<R: std::io::Read + ?Sized>(
131+ encoding: Option<&'static encoding_rs::Encoding>,
132+) -> Result<(Rope, &'static encoding_rs::Encoding), Error> {
133+ // These two buffers are 8192 bytes in size each and are used as
134+ // intermediaries during the decoding process. Text read into `buf`
135+ // from `reader` is decoded into `buf_out` as UTF-8. Once either
136+ // `buf_out` is full or the end of the reader was reached, the
137+ // contents are appended to `builder`.
138+ let mut buf = [0u8; BUF_SIZE];
139+ let mut buf_out = [0u8; BUF_SIZE];
140+ let mut builder = RopeBuilder::new();
142+ // By default, the encoding of the text is auto-detected via the
143+ // `chardetng` crate which requires sample data from the reader.
144+ // As a manual override to this auto-detection is possible, the
145+ // same data is read into `buf` to ensure symmetry in the upcoming
147+ let (encoding, mut decoder, mut slice, mut is_empty) = {
148+ let read = reader.read(&mut buf)?;
149+ let is_empty = read == 0;
150+ let encoding = encoding.unwrap_or_else(|| {
151+ let mut encoding_detector = chardetng::EncodingDetector::new();
152+ encoding_detector.feed(&buf, is_empty);
153+ encoding_detector.guess(None, true)
155+ let decoder = encoding.new_decoder();
157+ // If the amount of bytes read from the reader is less than
158+ // `buf.len()`, it is undesirable to read the bytes afterwards.
159+ let slice = &buf[..read];
160+ (encoding, decoder, slice, is_empty)
163+ // `RopeBuilder::append()` expects a `&str`, so this is the "real"
164+ // output buffer. When decoding, the number of bytes in the output
165+ // buffer will often exceed the number of bytes in the input buffer.
166+ // The `result` returned by `decode_to_str()` will state whether or
167+ // not that happened. The contents of `buf_str` is appended to
168+ // `builder` and it is reused for the next iteration of the decoding
171+ // As it is possible to read less than the buffer's maximum from `read()`
172+ // even when the end of the reader has yet to be reached, the end of
173+ // the reader is determined only when a `read()` call returns `0`.
175+ // SAFETY: `buf_out` is a zero-initialized array, thus it will always
176+ // contain valid UTF-8.
177+ let buf_str = unsafe { std::str::from_utf8_unchecked_mut(&mut buf_out[..]) };
178+ let mut total_written = 0usize;
180+ let mut total_read = 0usize;
183+ let (result, read, written, ..) = decoder.decode_to_str(
184+ &slice[total_read..],
185+ &mut buf_str[total_written..],
189+ // These variables act as the read and write cursors of `buf` and `buf_str` respectively.
190+ // They are necessary in case the output buffer fills before decoding of the entire input
191+ // loop is complete. Otherwise, the loop would endlessly iterate over the same `buf` and
192+ // the data inside the output buffer would be overwritten.
194+ total_written += written;
196+ encoding_rs::CoderResult::InputEmpty => {
197+ debug_assert_eq!(slice.len(), total_read);
200+ encoding_rs::CoderResult::OutputFull => {
201+ debug_assert!(slice.len() > total_read);
202+ builder.append(&buf_str[..total_written]);
207+ // Once the end of the stream is reached, the output buffer is
208+ // flushed and the loop terminates.
210+ debug_assert_eq!(reader.read(&mut buf)?, 0);
211+ builder.append(&buf_str[..total_written]);
215+ // Once the previous input has been processed and decoded, the next set of
216+ // data is fetched from the reader. The end of the reader is determined to
217+ // be when exactly `0` bytes were read from the reader, as per the invariants
218+ // of the `Read` trait.
219+ let read = reader.read(&mut buf)?;
220+ slice = &buf[..read];
221+ is_empty = read == 0;
223+ let rope = builder.finish();
224+ Ok((rope, encoding))
227+// The documentation and implementation of this function should be up-to-date with
228+// its sibling function, `from_reader()`.
230+/// Encodes the text inside `rope` into the given `encoding` and writes the
231+/// encoded output into `writer.` As a `Rope` can only contain valid UTF-8,
232+/// replacement characters may appear in the encoded text.
233+pub async fn to_writer<'a, W: tokio::io::AsyncWriteExt + Unpin + ?Sized>(
235+ encoding: &'static encoding_rs::Encoding,
237+) -> Result<(), Error> {
238+ // Text inside a `Rope` is stored as non-contiguous blocks of data called
239+ // chunks. The absolute size of each chunk is unknown, thus it is impossible
240+ // to predict the end of the chunk iterator ahead of time. Instead, it is
241+ // determined by filtering the iterator to remove all empty chunks and then
242+ // appending an empty chunk to it. This is valuable for detecting when all
243+ // chunks in the `Rope` have been iterated over in the subsequent loop.
246+ .filter(|c| !c.is_empty())
247+ .chain(std::iter::once(""));
248+ let mut buf = [0u8; BUF_SIZE];
249+ let mut encoder = encoding.new_encoder();
250+ let mut total_written = 0usize;
252+ let is_empty = chunk.is_empty();
253+ let mut total_read = 0usize;
256+ let (result, read, written, ..) =
257+ encoder.encode_from_utf8(&chunk[total_read..], &mut buf[total_written..], is_empty);
259+ // These variables act as the read and write cursors of `chunk` and `buf` respectively.
260+ // They are necessary in case the output buffer fills before encoding of the entire input
261+ // loop is complete. Otherwise, the loop would endlessly iterate over the same `chunk` and
262+ // the data inside the output buffer would be overwritten.
264+ total_written += written;
266+ encoding_rs::CoderResult::InputEmpty => {
267+ debug_assert_eq!(chunk.len(), total_read);
268+ debug_assert!(buf.len() >= total_written);
271+ encoding_rs::CoderResult::OutputFull => {
272+ debug_assert!(chunk.len() > total_read);
273+ writer.write_all(&buf[..total_written]).await?;
279+ // Once the end of the iterator is reached, the output buffer is
280+ // flushed and the outer loop terminates.
282+ writer.write_all(&buf[..total_written]).await?;
283+ writer.flush().await?;
119/// Like std::mem::replace() except it allows the replacement value to be mapped from the
290/// Like std::mem::replace() except it allows the replacement value to be mapped from the
120/// original value.
291/// original value.
121fn take_with<T, F>(mut_ref: &mut T, closure: F)
292fn take_with<T, F>(mut_ref: &mut T, closure: F)
@@ -216,13 +387,15 @@ use helix_lsp::lsp;
216use url::Url;
387use url::Url;
218impl Document {
389impl Document {
219− pub fn new(text: Rope) -> Self {
390+ pub fn from(text: Rope, encoding: Option<&'static encoding_rs::Encoding>) -> Self {
391+ let encoding = encoding.unwrap_or(encoding_rs::UTF_8);
220 let changes = ChangeSet::new(&text);
392 let changes = ChangeSet::new(&text);
221 let old_state = None;
393 let old_state = None;
224 id: DocumentId::default(),
396 id: DocumentId::default(),
225 path: None,
397 path: None,
227 selections: HashMap::default(),
400 selections: HashMap::default(),
228 indent_style: IndentStyle::Spaces(4),
401 indent_style: IndentStyle::Spaces(4),
@@ -242,29 +415,31 @@ impl Document {
244 // TODO: async fn?
417 // TODO: async fn?
418+ /// Create a new document from `path`. Encoding is auto-detected, but it can be manually
419+ /// overwritten with the `encoding` parameter.
245− pub fn load(
420+ pub fn open(
246 path: PathBuf,
421 path: PathBuf,
422+ encoding: Option<&'static encoding_rs::Encoding>,
247 theme: Option<&Theme>,
423 theme: Option<&Theme>,
248 config_loader: Option<&syntax::Loader>,
424 config_loader: Option<&syntax::Loader>,
249 ) -> Result<Self, Error> {
425 ) -> Result<Self, Error> {
250− use std::{fs::File, io::BufReader};
426+ if !path.exists() {
427+ return Ok(Self::default());
252− let mut doc = if !path.exists() {
253− Rope::from(DEFAULT_LINE_ENDING.as_str())
255− let file = File::open(&path).context(format!("unable to open {:?}", path))?;
430+ let mut file = std::fs::File::open(&path).context(format!("unable to open {:?}", path))?;
256− Rope::from_reader(BufReader::new(file))?
431+ let (mut rope, encoding) = from_reader(&mut file, encoding)?;
259 // search for line endings
433 // search for line endings
260− let line_ending = auto_detect_line_ending(&doc).unwrap_or(DEFAULT_LINE_ENDING);
434+ let line_ending = auto_detect_line_ending(&rope).unwrap_or(DEFAULT_LINE_ENDING);
262 // add missing newline at the end of file
436 // add missing newline at the end of file
263− if doc.len_bytes() == 0 || !char_is_line_ending(doc.char(doc.len_chars() - 1)) {
437+ if rope.len_bytes() == 0 || !char_is_line_ending(rope.char(rope.len_chars() - 1)) {
264− doc.insert(doc.len_chars(), line_ending.as_str());
438+ rope.insert(rope.len_chars(), line_ending.as_str());
267− let mut doc = Self::new(doc);
441+ let mut doc = Self::from(rope, Some(encoding));
268 // set the path and try detecting the language
443 // set the path and try detecting the language
269 doc.set_path(&path)?;
444 doc.set_path(&path)?;
270 doc.detect_indent_style();
445 doc.detect_indent_style();
@@ -303,6 +478,8 @@ impl Document {
304 // TODO: do we need some way of ensuring two save operations on the same doc can't run at once?
479 // TODO: do we need some way of ensuring two save operations on the same doc can't run at once?
305 // or is that handled by the OS/async layer
480 // or is that handled by the OS/async layer
481+ /// The `Document`'s text is encoded according to its encoding and written to the file located
482+ /// at its `path()`.
306 pub fn save(&mut self) -> impl Future<Output = Result<(), anyhow::Error>> {
483 pub fn save(&mut self) -> impl Future<Output = Result<(), anyhow::Error>> {
307 // we clone and move text + path into the future so that we asynchronously save the current
484 // we clone and move text + path into the future so that we asynchronously save the current
308 // state without blocking any further edits.
485 // state without blocking any further edits.
@@ -320,8 +497,11 @@ impl Document {
320 self.last_saved_revision = history.current_revision();
497 self.last_saved_revision = history.current_revision();
321 self.history.set(history);
498 self.history.set(history);
500+ let encoding = self.encoding;
502+ // We encode the file according to the `Document`'s encoding.
323 async move {
503 async move {
324− use tokio::{fs::File, io::AsyncWriteExt};
504+ use tokio::fs::File;
325 if let Some(parent) = path.parent() {
505 if let Some(parent) = path.parent() {
326 // TODO: display a prompt asking the user if the directories should be created
506 // TODO: display a prompt asking the user if the directories should be created
327 if !parent.exists() {
507 if !parent.exists() {
@@ -330,13 +510,9 @@ impl Document {
333− let mut file = File::create(path).await?;
335− // write all the rope chunks to file
514+ let mut file = File::create(path).await?;
336− for chunk in text.chunks() {
515+ to_writer(&mut file, encoding, &text).await?;
337− file.write_all(chunk.as_bytes()).await?;
341 if let Some(language_server) = language_server {
517 if let Some(language_server) = language_server {
342 language_server
518 language_server
@@ -531,7 +707,7 @@ impl Document {
531 self.selections.insert(view_id, selection);
707 self.selections.insert(view_id, selection);
534− fn _apply(&mut self, transaction: &Transaction, view_id: ViewId) -> bool {
710+ fn apply_impl(&mut self, transaction: &Transaction, view_id: ViewId) -> bool {
535 let old_doc = self.text().clone();
711 let old_doc = self.text().clone();
537 let success = transaction.changes().apply(&mut self.text);
713 let success = transaction.changes().apply(&mut self.text);
@@ -594,7 +770,7 @@ impl Document {
597− let success = self._apply(transaction, view_id);
773+ let success = self.apply_impl(transaction, view_id);
599 if !transaction.changes().is_empty() {
775 if !transaction.changes().is_empty() {
600 // Compose this transaction with the previous one
776 // Compose this transaction with the previous one
@@ -608,7 +784,7 @@ impl Document {
608 pub fn undo(&mut self, view_id: ViewId) {
784 pub fn undo(&mut self, view_id: ViewId) {
609 let mut history = self.history.take();
785 let mut history = self.history.take();
610 let success = if let Some(transaction) = history.undo() {
786 let success = if let Some(transaction) = history.undo() {
611− self._apply(&transaction, view_id)
787+ self.apply_impl(transaction, view_id)
@@ -623,7 +799,7 @@ impl Document {
623 pub fn redo(&mut self, view_id: ViewId) {
799 pub fn redo(&mut self, view_id: ViewId) {
624 let mut history = self.history.take();
800 let mut history = self.history.take();
625 let success = if let Some(transaction) = history.redo() {
801 let success = if let Some(transaction) = history.redo() {
626− self._apply(&transaction, view_id)
802+ self.apply_impl(transaction, view_id)
@@ -638,14 +814,14 @@ impl Document {
638 pub fn earlier(&mut self, view_id: ViewId, uk: helix_core::history::UndoKind) {
814 pub fn earlier(&mut self, view_id: ViewId, uk: helix_core::history::UndoKind) {
639 let txns = self.history.get_mut().earlier(uk);
815 let txns = self.history.get_mut().earlier(uk);
640 for txn in txns {
816 for txn in txns {
641− self._apply(&txn, view_id);
817+ self.apply_impl(&txn, view_id);
645 pub fn later(&mut self, view_id: ViewId, uk: helix_core::history::UndoKind) {
821 pub fn later(&mut self, view_id: ViewId, uk: helix_core::history::UndoKind) {
646 let txns = self.history.get_mut().later(uk);
822 let txns = self.history.get_mut().later(uk);
647 for txn in txns {
823 for txn in txns {
648− self._apply(&txn, view_id);
824+ self.apply_impl(&txn, view_id);
@@ -670,12 +846,10 @@ impl Document {
670 self.history.set(history);
846 self.history.set(history);
674 pub fn id(&self) -> DocumentId {
849 pub fn id(&self) -> DocumentId {
679 pub fn is_modified(&self) -> bool {
853 pub fn is_modified(&self) -> bool {
680 let history = self.history.take();
854 let history = self.history.take();
681 let current_revision = history.current_revision();
855 let current_revision = history.current_revision();
@@ -683,12 +857,10 @@ impl Document {
683 current_revision != self.last_saved_revision || !self.changes.is_empty()
857 current_revision != self.last_saved_revision || !self.changes.is_empty()
687 pub fn mode(&self) -> Mode {
860 pub fn mode(&self) -> Mode {
688 self.mode
861 self.mode
692 /// Corresponding language scope name. Usually `source.<lang>`.
864 /// Corresponding language scope name. Usually `source.<lang>`.
693 pub fn language(&self) -> Option<&str> {
865 pub fn language(&self) -> Option<&str> {
694 self.language
866 self.language
@@ -696,21 +868,21 @@ impl Document {
696 .map(|language| language.scope.as_str())
868 .map(|language| language.scope.as_str())
700 pub fn language_config(&self) -> Option<&LanguageConfiguration> {
871 pub fn language_config(&self) -> Option<&LanguageConfiguration> {
701 self.language.as_deref()
872 self.language.as_deref()
705 /// Current document version, incremented at each change.
875 /// Current document version, incremented at each change.
706 pub fn version(&self) -> i32 {
876 pub fn version(&self) -> i32 {
707 self.version
877 self.version
710 pub fn language_server(&self) -> Option<&helix_lsp::Client> {
881 pub fn language_server(&self) -> Option<&helix_lsp::Client> {
711 self.language_server.as_deref()
882 self.language_server.as_deref()
714 /// Tree-sitter AST tree
886 /// Tree-sitter AST tree
715 pub fn syntax(&self) -> Option<&Syntax> {
887 pub fn syntax(&self) -> Option<&Syntax> {
716 self.syntax.as_ref()
888 self.syntax.as_ref()
@@ -756,10 +928,12 @@ impl Document {
756 self.path().map(|path| Url::from_file_path(path).unwrap())
928 self.path().map(|path| Url::from_file_path(path).unwrap())
759 pub fn text(&self) -> &Rope {
932 pub fn text(&self) -> &Rope {
760 &self.text
933 &self.text
763 pub fn selection(&self, view_id: ViewId) -> &Selection {
937 pub fn selection(&self, view_id: ViewId) -> &Selection {
764 &self.selections[&view_id]
938 &self.selections[&view_id]
@@ -787,6 +961,7 @@ impl Document {
788 // -- LSP methods
962 // -- LSP methods
790 pub fn identifier(&self) -> lsp::TextDocumentIdentifier {
965 pub fn identifier(&self) -> lsp::TextDocumentIdentifier {
791 lsp::TextDocumentIdentifier::new(self.url().unwrap())
966 lsp::TextDocumentIdentifier::new(self.url().unwrap())
@@ -795,6 +970,7 @@ impl Document {
795 lsp::VersionedTextDocumentIdentifier::new(self.url().unwrap(), self.version)
970 lsp::VersionedTextDocumentIdentifier::new(self.url().unwrap(), self.version)
798 pub fn diagnostics(&self) -> &[Diagnostic] {
974 pub fn diagnostics(&self) -> &[Diagnostic] {
799 &self.diagnostics
975 &self.diagnostics
@@ -804,6 +980,13 @@ impl Document {
983+impl Default for Document {
984+ fn default() -> Self {
985+ let text = Rope::from(DEFAULT_LINE_ENDING.as_str());
986+ Self::from(text, None)
807#[cfg(test)]
990#[cfg(test)]
808mod test {
991mod test {
809 use super::*;
992 use super::*;
@@ -812,7 +995,7 @@ mod test {
812 fn changeset_to_changes() {
995 fn changeset_to_changes() {
813 use helix_lsp::{lsp, Client, OffsetEncoding};
996 use helix_lsp::{lsp, Client, OffsetEncoding};
814 let text = Rope::from("hello");
997 let text = Rope::from("hello");
815− let mut doc = Document::new(text);
998+ let mut doc = Document::from(text, None);
816 let view = ViewId::default();
999 let view = ViewId::default();
817 doc.set_selection(view, Selection::single(5, 5));
1000 doc.set_selection(view, Selection::single(5, 5));
@@ -921,4 +1104,94 @@ mod test {
1109+ fn test_line_ending() {
1111+ assert_eq!(Document::default().text().to_string(), "\r\n");
1113+ assert_eq!(Document::default().text().to_string(), "\n");
1117+ macro_rules! test_decode {
1118+ ($label:expr, $label_override:expr) => {
1119+ let encoding = encoding_rs::Encoding::for_label($label_override.as_bytes()).unwrap();
1120+ let base_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests");
1121+ let path = base_path.join(format!("{}_in.txt", $label));
1122+ let ref_path = base_path.join(format!("{}_in_ref.txt", $label));
1123+ assert!(path.exists());
1124+ assert!(ref_path.exists());
1126+ let mut file = std::fs::File::open(path).unwrap();
1127+ let text = from_reader(&mut file, Some(encoding))
1131+ let expectation = std::fs::read_to_string(ref_path).unwrap();
1132+ assert_eq!(text[..], expectation[..]);
1136+ macro_rules! test_encode {
1137+ ($label:expr, $label_override:expr) => {
1138+ let encoding = encoding_rs::Encoding::for_label($label_override.as_bytes()).unwrap();
1139+ let base_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests");
1140+ let path = base_path.join(format!("{}_out.txt", $label));
1141+ let ref_path = base_path.join(format!("{}_out_ref.txt", $label));
1142+ assert!(path.exists());
1143+ assert!(ref_path.exists());
1145+ let text = Rope::from_str(&std::fs::read_to_string(path).unwrap());
1146+ let mut buf: Vec<u8> = Vec::new();
1147+ helix_lsp::block_on(to_writer(&mut buf, encoding, &text)).unwrap();
1149+ let expectation = std::fs::read(ref_path).unwrap();
1150+ assert_eq!(buf, expectation);
1154+ macro_rules! test_decode_fn {
1155+ ($name:ident, $label:expr, $label_override:expr) => {
1158+ test_decode!($label, $label_override);
1161+ ($name:ident, $label:expr) => {
1164+ test_decode!($label, $label);
1169+ macro_rules! test_encode_fn {
1170+ ($name:ident, $label:expr, $label_override:expr) => {
1173+ test_encode!($label, $label_override);
1176+ ($name:ident, $label:expr) => {
1179+ test_encode!($label, $label);
1184+ test_decode_fn!(test_big5_decode, "big5");
1185+ test_encode_fn!(test_big5_encode, "big5");
1186+ test_decode_fn!(test_euc_kr_decode, "euc_kr", "EUC-KR");
1187+ test_encode_fn!(test_euc_kr_encode, "euc_kr", "EUC-KR");
1188+ test_decode_fn!(test_gb18030_decode, "gb18030");
1189+ test_encode_fn!(test_gb18030_encode, "gb18030");
1190+ test_decode_fn!(test_iso_2022_jp_decode, "iso_2022_jp", "ISO-2022-JP");
1191+ test_encode_fn!(test_iso_2022_jp_encode, "iso_2022_jp", "ISO-2022-JP");
1192+ test_decode_fn!(test_jis0208_decode, "jis0208", "EUC-JP");
1193+ test_encode_fn!(test_jis0208_encode, "jis0208", "EUC-JP");
1194+ test_decode_fn!(test_jis0212_decode, "jis0212", "EUC-JP");
1195+ test_decode_fn!(test_shift_jis_decode, "shift_jis");
1196+ test_encode_fn!(test_shift_jis_encode, "shift_jis");