mirror of
https://github.com/ilyakooo0/helix.git
synced 2024-09-12 00:07:42 +03:00
Incremental parsing: rough draft.
This commit is contained in:
parent
31999d6528
commit
088f8a82af
1
.gitignore
vendored
1
.gitignore
vendored
@ -1,2 +1,3 @@
|
||||
target
|
||||
helix-term/rustfmt.toml
|
||||
helix-syntax/languages/
|
||||
|
1
Cargo.lock
generated
1
Cargo.lock
generated
@ -369,6 +369,7 @@ name = "helix-core"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"helix-syntax",
|
||||
"ropey",
|
||||
"smallvec",
|
||||
"tendril",
|
||||
|
@ -6,5 +6,5 @@ members = [
|
||||
]
|
||||
|
||||
# Build helix-syntax in release mode to make the code path faster in development.
|
||||
[profile.dev.package."helix-syntax"]
|
||||
opt-level = 3
|
||||
# [profile.dev.package."helix-syntax"]
|
||||
# opt-level = 3
|
||||
|
@ -7,6 +7,8 @@ edition = "2018"
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
helix-syntax = { path = "../helix-syntax" }
|
||||
|
||||
ropey = "1.2.0"
|
||||
anyhow = "1.0.31"
|
||||
smallvec = "1.4.0"
|
||||
|
@ -1,6 +1,7 @@
|
||||
#![allow(unused)]
|
||||
pub mod commands;
|
||||
pub mod graphemes;
|
||||
mod position;
|
||||
mod selection;
|
||||
pub mod state;
|
||||
pub mod syntax;
|
||||
@ -9,8 +10,10 @@
|
||||
pub use ropey::{Rope, RopeSlice};
|
||||
pub use tendril::StrTendril as Tendril;
|
||||
|
||||
pub use position::Position;
|
||||
pub use selection::Range as SelectionRange;
|
||||
pub use selection::Selection;
|
||||
pub use syntax::Syntax;
|
||||
|
||||
pub use state::State;
|
||||
|
||||
|
57
helix-core/src/position.rs
Normal file
57
helix-core/src/position.rs
Normal file
@ -0,0 +1,57 @@
|
||||
/// Represents a single point in a text buffer. Zero indexed.
|
||||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct Position {
|
||||
pub row: usize,
|
||||
pub col: usize,
|
||||
}
|
||||
|
||||
impl Position {
|
||||
pub const fn new(row: usize, col: usize) -> Self {
|
||||
Self { row, col }
|
||||
}
|
||||
|
||||
pub const fn is_zero(self) -> bool {
|
||||
self.row == 0 && self.col == 0
|
||||
}
|
||||
|
||||
// TODO: generalize
|
||||
pub fn traverse(self, text: &crate::Tendril) -> Self {
|
||||
let Self { mut row, mut col } = self;
|
||||
// TODO: there should be a better way here
|
||||
for ch in text.chars() {
|
||||
if ch == '\n' {
|
||||
row += 1;
|
||||
col = 0;
|
||||
} else {
|
||||
col += 1;
|
||||
}
|
||||
}
|
||||
Self { row, col }
|
||||
}
|
||||
}
|
||||
|
||||
impl From<(usize, usize)> for Position {
|
||||
fn from(tuple: (usize, usize)) -> Self {
|
||||
Position {
|
||||
row: tuple.0,
|
||||
col: tuple.1,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Into<tree_sitter::Point> for Position {
|
||||
fn into(self) -> tree_sitter::Point {
|
||||
tree_sitter::Point::new(self.row, self.col)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_ordering() {
|
||||
// (0, 5) is less than (1, 0)
|
||||
assert!(Position::new(0, 5) < Position::new(1, 0));
|
||||
}
|
||||
}
|
@ -1,5 +1,5 @@
|
||||
use crate::graphemes::{nth_next_grapheme_boundary, nth_prev_grapheme_boundary, RopeGraphemes};
|
||||
use crate::{Rope, RopeSlice, Selection, SelectionRange};
|
||||
use crate::{Position, Rope, RopeSlice, Selection, SelectionRange, Syntax};
|
||||
use anyhow::Error;
|
||||
|
||||
use std::path::PathBuf;
|
||||
@ -17,6 +17,9 @@ pub struct State {
|
||||
pub(crate) doc: Rope,
|
||||
pub(crate) selection: Selection,
|
||||
pub(crate) mode: Mode,
|
||||
|
||||
//
|
||||
pub syntax: Option<Syntax>,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, PartialEq, Eq)]
|
||||
@ -40,6 +43,7 @@ pub fn new(doc: Rope) -> Self {
|
||||
doc,
|
||||
selection: Selection::single(0, 0),
|
||||
mode: Mode::Normal,
|
||||
syntax: None,
|
||||
}
|
||||
}
|
||||
|
||||
@ -54,6 +58,29 @@ pub fn load(path: PathBuf) -> Result<Self, Error> {
|
||||
let mut state = Self::new(doc);
|
||||
state.path = Some(path);
|
||||
|
||||
let language = helix_syntax::get_language(&helix_syntax::LANG::Rust);
|
||||
|
||||
let mut highlight_config = crate::syntax::HighlightConfiguration::new(
|
||||
language,
|
||||
&std::fs::read_to_string(
|
||||
"../helix-syntax/languages/tree-sitter-rust/queries/highlights.scm",
|
||||
)
|
||||
.unwrap(),
|
||||
&std::fs::read_to_string(
|
||||
"../helix-syntax/languages/tree-sitter-rust/queries/injections.scm",
|
||||
)
|
||||
.unwrap(),
|
||||
"", // locals.scm
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// TODO: config.configure(scopes) is now delayed, is that ok?
|
||||
|
||||
// TODO: get_language is called twice
|
||||
let syntax = Syntax::new(helix_syntax::LANG::Rust, &state.doc, highlight_config);
|
||||
|
||||
state.syntax = Some(syntax);
|
||||
|
||||
Ok(state)
|
||||
}
|
||||
|
||||
@ -175,29 +202,29 @@ pub fn extend_selection(
|
||||
}
|
||||
|
||||
/// Coordinates are a 0-indexed line and column pair.
|
||||
type Coords = (usize, usize); // line, col
|
||||
pub type Coords = (usize, usize); // line, col
|
||||
|
||||
/// Convert a character index to (line, column) coordinates.
|
||||
pub fn coords_at_pos(text: &RopeSlice, pos: usize) -> Coords {
|
||||
pub fn coords_at_pos(text: &RopeSlice, pos: usize) -> Position {
|
||||
let line = text.char_to_line(pos);
|
||||
let line_start = text.line_to_char(line);
|
||||
let col = text.slice(line_start..pos).len_chars();
|
||||
(line, col)
|
||||
Position::new(line, col)
|
||||
}
|
||||
|
||||
/// Convert (line, column) coordinates to a character index.
|
||||
pub fn pos_at_coords(text: &RopeSlice, coords: Coords) -> usize {
|
||||
let (line, col) = coords;
|
||||
let line_start = text.line_to_char(line);
|
||||
pub fn pos_at_coords(text: &RopeSlice, coords: Position) -> usize {
|
||||
let Position { row, col } = coords;
|
||||
let line_start = text.line_to_char(row);
|
||||
nth_next_grapheme_boundary(text, line_start, col)
|
||||
}
|
||||
|
||||
fn move_vertically(text: &RopeSlice, dir: Direction, pos: usize, count: usize) -> usize {
|
||||
let (line, col) = coords_at_pos(text, pos);
|
||||
let Position { row, col } = coords_at_pos(text, pos);
|
||||
|
||||
let new_line = match dir {
|
||||
Direction::Backward => line.saturating_sub(count),
|
||||
Direction::Forward => std::cmp::min(line.saturating_add(count), text.len_lines() - 1),
|
||||
Direction::Backward => row.saturating_sub(count),
|
||||
Direction::Forward => std::cmp::min(row.saturating_add(count), text.len_lines() - 1),
|
||||
};
|
||||
|
||||
// convert to 0-indexed, subtract another 1 because len_chars() counts \n
|
||||
@ -210,7 +237,7 @@ fn move_vertically(text: &RopeSlice, dir: Direction, pos: usize, count: usize) -
|
||||
col
|
||||
};
|
||||
|
||||
pos_at_coords(text, (new_line, new_col))
|
||||
pos_at_coords(text, Position::new(new_line, new_col))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@ -220,32 +247,32 @@ mod test {
|
||||
#[test]
|
||||
fn test_coords_at_pos() {
|
||||
let text = Rope::from("ḧëḷḷö\nẅöṛḷḋ");
|
||||
assert_eq!(coords_at_pos(&text.slice(..), 0), (0, 0));
|
||||
assert_eq!(coords_at_pos(&text.slice(..), 5), (0, 5)); // position on \n
|
||||
assert_eq!(coords_at_pos(&text.slice(..), 6), (1, 0)); // position on w
|
||||
assert_eq!(coords_at_pos(&text.slice(..), 7), (1, 1)); // position on o
|
||||
assert_eq!(coords_at_pos(&text.slice(..), 10), (1, 4)); // position on d
|
||||
assert_eq!(coords_at_pos(&text.slice(..), 0), (0, 0).into());
|
||||
assert_eq!(coords_at_pos(&text.slice(..), 5), (0, 5).into()); // position on \n
|
||||
assert_eq!(coords_at_pos(&text.slice(..), 6), (1, 0).into()); // position on w
|
||||
assert_eq!(coords_at_pos(&text.slice(..), 7), (1, 1).into()); // position on o
|
||||
assert_eq!(coords_at_pos(&text.slice(..), 10), (1, 4).into()); // position on d
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_pos_at_coords() {
|
||||
let text = Rope::from("ḧëḷḷö\nẅöṛḷḋ");
|
||||
assert_eq!(pos_at_coords(&text.slice(..), (0, 0)), 0);
|
||||
assert_eq!(pos_at_coords(&text.slice(..), (0, 5)), 5); // position on \n
|
||||
assert_eq!(pos_at_coords(&text.slice(..), (1, 0)), 6); // position on w
|
||||
assert_eq!(pos_at_coords(&text.slice(..), (1, 1)), 7); // position on o
|
||||
assert_eq!(pos_at_coords(&text.slice(..), (1, 4)), 10); // position on d
|
||||
assert_eq!(pos_at_coords(&text.slice(..), (0, 0).into()), 0);
|
||||
assert_eq!(pos_at_coords(&text.slice(..), (0, 5).into()), 5); // position on \n
|
||||
assert_eq!(pos_at_coords(&text.slice(..), (1, 0).into()), 6); // position on w
|
||||
assert_eq!(pos_at_coords(&text.slice(..), (1, 1).into()), 7); // position on o
|
||||
assert_eq!(pos_at_coords(&text.slice(..), (1, 4).into()), 10); // position on d
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_vertical_move() {
|
||||
let text = Rope::from("abcd\nefg\nwrs");
|
||||
let pos = pos_at_coords(&text.slice(..), (0, 4));
|
||||
let pos = pos_at_coords(&text.slice(..), (0, 4).into());
|
||||
let slice = text.slice(..);
|
||||
|
||||
assert_eq!(
|
||||
coords_at_pos(&slice, move_vertically(&slice, Direction::Forward, pos, 1)),
|
||||
(1, 2)
|
||||
(1, 2).into()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
@ -1,66 +1,397 @@
|
||||
// pub struct Syntax {
|
||||
// parser: Parser,
|
||||
// }
|
||||
use crate::{Change, Rope, RopeSlice, Transaction};
|
||||
pub use helix_syntax::LANG;
|
||||
pub use helix_syntax::{get_language, get_language_name};
|
||||
|
||||
//impl Syntax {
|
||||
// // buffer, grammar, config, grammars, sync_timeout?
|
||||
// pub fn new() -> Self {
|
||||
// unimplemented!()
|
||||
// // make a new root layer
|
||||
// // track markers of injections
|
||||
// //
|
||||
// // track scope_descriptor: a Vec of scopes for item in tree
|
||||
// //
|
||||
// // fetch grammar for parser based on language string
|
||||
// // update root layer
|
||||
// }
|
||||
pub struct Syntax {
|
||||
grammar: Language,
|
||||
parser: Parser,
|
||||
cursors: Vec<QueryCursor>,
|
||||
|
||||
// // fn buffer_changed -> call layer.update(range, new_text) on root layer and then all marker layers
|
||||
config: HighlightConfiguration,
|
||||
|
||||
// // call this on transaction.apply() -> buffer_changed(changes)
|
||||
// //
|
||||
// // fn parse(language, old_tree, ranges)
|
||||
// //
|
||||
// // fn tree() -> Tree
|
||||
// //
|
||||
// // <!--update_for_injection(grammar)-->
|
||||
root_layer: LanguageLayer,
|
||||
}
|
||||
|
||||
// // Highlighting
|
||||
// // fn highlight_iter() -> iterates over all the scopes
|
||||
// // on_tokenize
|
||||
// // on_change_highlighting
|
||||
impl Syntax {
|
||||
// buffer, grammar, config, grammars, sync_timeout?
|
||||
pub fn new(language: LANG, source: &Rope, config: HighlightConfiguration) -> Self {
|
||||
// fetch grammar for parser based on language string
|
||||
let grammar = get_language(&language);
|
||||
let parser = Parser::new();
|
||||
|
||||
// // Commenting
|
||||
// // comment_strings_for_pos
|
||||
// // is_commented
|
||||
let root_layer = LanguageLayer::new();
|
||||
|
||||
// // Indentation
|
||||
// // suggested_indent_for_line_at_buffer_row
|
||||
// // suggested_indent_for_buffer_row
|
||||
// // indent_level_for_line
|
||||
// track markers of injections
|
||||
// track scope_descriptor: a Vec of scopes for item in tree
|
||||
|
||||
// // TODO: Folding
|
||||
let mut syntax = Self {
|
||||
grammar,
|
||||
parser,
|
||||
cursors: Vec::new(),
|
||||
config,
|
||||
root_layer,
|
||||
};
|
||||
|
||||
// // Syntax APIs
|
||||
// // get_syntax_node_containing_range ->
|
||||
// // ...
|
||||
// // get_syntax_node_at_pos
|
||||
// // buffer_range_for_scope_at_pos
|
||||
//}
|
||||
// update root layer
|
||||
syntax.root_layer.parse(
|
||||
&mut syntax.parser,
|
||||
&syntax.config,
|
||||
source,
|
||||
0,
|
||||
vec![Range {
|
||||
start_byte: 0,
|
||||
end_byte: usize::MAX,
|
||||
start_point: Point::new(0, 0),
|
||||
end_point: Point::new(usize::MAX, usize::MAX),
|
||||
}],
|
||||
);
|
||||
syntax
|
||||
}
|
||||
|
||||
pub fn configure(&mut self, scopes: &[String]) {
|
||||
self.config.configure(scopes)
|
||||
}
|
||||
|
||||
pub fn update(&mut self, source: &Rope, changeset: &ChangeSet) -> Result<(), Error> {
|
||||
self.root_layer
|
||||
.update(&mut self.parser, &self.config, source, changeset)
|
||||
|
||||
// TODO: deal with injections and update them too
|
||||
}
|
||||
|
||||
// fn buffer_changed -> call layer.update(range, new_text) on root layer and then all marker layers
|
||||
|
||||
// call this on transaction.apply() -> buffer_changed(changes)
|
||||
//
|
||||
// fn parse(language, old_tree, ranges)
|
||||
//
|
||||
fn tree(&self) -> &Tree {
|
||||
self.root_layer.tree()
|
||||
}
|
||||
//
|
||||
// <!--update_for_injection(grammar)-->
|
||||
|
||||
// Highlighting
|
||||
|
||||
/// Iterate over the highlighted regions for a given slice of source code.
|
||||
pub fn highlight_iter<'a>(
|
||||
&'a mut self,
|
||||
source: &'a [u8],
|
||||
cancellation_flag: Option<&'a AtomicUsize>,
|
||||
mut injection_callback: impl FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
|
||||
) -> Result<impl Iterator<Item = Result<HighlightEvent, Error>> + 'a, Error> {
|
||||
// The `captures` iterator borrows the `Tree` and the `QueryCursor`, which
|
||||
// prevents them from being moved. But both of these values are really just
|
||||
// pointers, so it's actually ok to move them.
|
||||
|
||||
let mut cursor = QueryCursor::new(); // reuse a pool
|
||||
let tree_ref = unsafe { mem::transmute::<_, &'static Tree>(self.tree()) };
|
||||
let cursor_ref = unsafe { mem::transmute::<_, &'static mut QueryCursor>(&mut cursor) };
|
||||
let query_ref = unsafe { mem::transmute::<_, &'static mut Query>(&mut self.config.query) };
|
||||
let config_ref =
|
||||
unsafe { mem::transmute::<_, &'static HighlightConfiguration>(&self.config) };
|
||||
let captures = cursor_ref
|
||||
.captures(query_ref, tree_ref.root_node(), move |n: Node| {
|
||||
&source[n.byte_range()]
|
||||
})
|
||||
.peekable();
|
||||
|
||||
// manually craft the root layer based on the existing tree
|
||||
let layer = HighlightIterLayer {
|
||||
highlight_end_stack: Vec::new(),
|
||||
scope_stack: vec![LocalScope {
|
||||
inherits: false,
|
||||
range: 0..usize::MAX,
|
||||
local_defs: Vec::new(),
|
||||
}],
|
||||
cursor,
|
||||
depth: 0,
|
||||
_tree: None,
|
||||
captures,
|
||||
config: config_ref,
|
||||
ranges: vec![Range {
|
||||
start_byte: 0,
|
||||
end_byte: usize::MAX,
|
||||
start_point: Point::new(0, 0),
|
||||
end_point: Point::new(usize::MAX, usize::MAX),
|
||||
}],
|
||||
};
|
||||
|
||||
let mut result = HighlightIter {
|
||||
source,
|
||||
byte_offset: 0,
|
||||
injection_callback,
|
||||
cancellation_flag,
|
||||
highlighter: self,
|
||||
iter_count: 0,
|
||||
layers: vec![layer],
|
||||
next_event: None,
|
||||
last_highlight_range: None,
|
||||
};
|
||||
result.sort_layers();
|
||||
Ok(result)
|
||||
}
|
||||
// on_tokenize
|
||||
// on_change_highlighting
|
||||
|
||||
// Commenting
|
||||
// comment_strings_for_pos
|
||||
// is_commented
|
||||
|
||||
// Indentation
|
||||
// suggested_indent_for_line_at_buffer_row
|
||||
// suggested_indent_for_buffer_row
|
||||
// indent_level_for_line
|
||||
|
||||
// TODO: Folding
|
||||
|
||||
// Syntax APIs
|
||||
// get_syntax_node_containing_range ->
|
||||
// ...
|
||||
// get_syntax_node_at_pos
|
||||
// buffer_range_for_scope_at_pos
|
||||
}
|
||||
|
||||
pub struct LanguageLayer {
|
||||
// mode
|
||||
// grammar
|
||||
// depth
|
||||
// tree: Tree,
|
||||
// grammar
|
||||
// depth
|
||||
tree: Option<Tree>,
|
||||
}
|
||||
|
||||
// impl LanguageLayer {
|
||||
// // fn highlight_iter() -> same as Mode but for this layer. Mode composits these
|
||||
// // fn buffer_changed
|
||||
// // fn update(range)
|
||||
// // fn update_injections()
|
||||
// }
|
||||
use crate::state::{coords_at_pos, Coords};
|
||||
use crate::transaction::{ChangeSet, Operation};
|
||||
use crate::Tendril;
|
||||
|
||||
impl LanguageLayer {
|
||||
pub fn new() -> Self {
|
||||
Self { tree: None }
|
||||
}
|
||||
|
||||
fn tree(&self) -> &Tree {
|
||||
// TODO: no unwrap
|
||||
self.tree.as_ref().unwrap()
|
||||
}
|
||||
|
||||
fn parse<'a>(
|
||||
&mut self,
|
||||
parser: &mut Parser,
|
||||
config: &HighlightConfiguration,
|
||||
source: &Rope,
|
||||
mut depth: usize,
|
||||
mut ranges: Vec<Range>,
|
||||
) -> Result<(), Error> {
|
||||
if parser.set_included_ranges(&ranges).is_ok() {
|
||||
parser
|
||||
.set_language(config.language)
|
||||
.map_err(|_| Error::InvalidLanguage)?;
|
||||
|
||||
// unsafe { syntax.parser.set_cancellation_flag(cancellation_flag) };
|
||||
let tree = parser
|
||||
.parse_with(
|
||||
&mut |byte, _| {
|
||||
if byte <= source.len_bytes() {
|
||||
let (chunk, start_byte, _, _) = source.chunk_at_byte(byte);
|
||||
chunk[byte - start_byte..].as_bytes()
|
||||
} else {
|
||||
// out of range
|
||||
&[]
|
||||
}
|
||||
},
|
||||
self.tree.as_ref(),
|
||||
)
|
||||
.ok_or(Error::Cancelled)?;
|
||||
// unsafe { syntax.parser.set_cancellation_flag(None) };
|
||||
// let mut cursor = syntax.cursors.pop().unwrap_or_else(QueryCursor::new);
|
||||
|
||||
// Process combined injections. (ERB, EJS, etc https://github.com/tree-sitter/tree-sitter/pull/526)
|
||||
// if let Some(combined_injections_query) = &config.combined_injections_query {
|
||||
// let mut injections_by_pattern_index =
|
||||
// vec![(None, Vec::new(), false); combined_injections_query.pattern_count()];
|
||||
// let matches =
|
||||
// cursor.matches(combined_injections_query, tree.root_node(), |n: Node| {
|
||||
// &source[n.byte_range()]
|
||||
// });
|
||||
// for mat in matches {
|
||||
// let entry = &mut injections_by_pattern_index[mat.pattern_index];
|
||||
// let (language_name, content_node, include_children) =
|
||||
// injection_for_match(config, combined_injections_query, &mat, source);
|
||||
// if language_name.is_some() {
|
||||
// entry.0 = language_name;
|
||||
// }
|
||||
// if let Some(content_node) = content_node {
|
||||
// entry.1.push(content_node);
|
||||
// }
|
||||
// entry.2 = include_children;
|
||||
// }
|
||||
// for (lang_name, content_nodes, includes_children) in injections_by_pattern_index {
|
||||
// if let (Some(lang_name), false) = (lang_name, content_nodes.is_empty()) {
|
||||
// if let Some(next_config) = (injection_callback)(lang_name) {
|
||||
// let ranges =
|
||||
// Self::intersect_ranges(&ranges, &content_nodes, includes_children);
|
||||
// if !ranges.is_empty() {
|
||||
// queue.push((next_config, depth + 1, ranges));
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
self.tree = Some(tree)
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn generate_edits(
|
||||
text: &RopeSlice,
|
||||
changeset: &ChangeSet,
|
||||
) -> Vec<tree_sitter::InputEdit> {
|
||||
use Operation::*;
|
||||
let mut old_pos = 0;
|
||||
let mut new_pos = 0;
|
||||
|
||||
let mut edits = Vec::new();
|
||||
|
||||
let mut iter = changeset.changes.iter().peekable();
|
||||
|
||||
// TODO; this is a lot easier with Change instead of Operation.
|
||||
|
||||
fn point_at_pos(text: &RopeSlice, pos: usize) -> (usize, Point) {
|
||||
let byte = text.char_to_byte(pos);
|
||||
let line = text.char_to_line(pos);
|
||||
let line_start_byte = text.line_to_byte(line);
|
||||
let col = byte - line_start_byte;
|
||||
|
||||
(byte, Point::new(line, col))
|
||||
}
|
||||
|
||||
fn traverse(point: Point, text: &Tendril) -> Point {
|
||||
let Point {
|
||||
mut row,
|
||||
mut column,
|
||||
} = point;
|
||||
|
||||
// TODO: there should be a better way here
|
||||
for ch in text.bytes() {
|
||||
if ch == b'\n' {
|
||||
row += 1;
|
||||
column = 0;
|
||||
} else {
|
||||
column += 1;
|
||||
}
|
||||
}
|
||||
Point { row, column }
|
||||
}
|
||||
|
||||
while let Some(change) = iter.next() {
|
||||
let len = match change {
|
||||
Delete(i) | Retain(i) => *i,
|
||||
Insert(_) => 0,
|
||||
};
|
||||
let old_end = old_pos + len;
|
||||
|
||||
match change {
|
||||
Retain(_) => {
|
||||
new_pos += len;
|
||||
}
|
||||
Delete(_) => {
|
||||
let (start_byte, start_position) = point_at_pos(&text, old_pos);
|
||||
let (old_end_byte, old_end_position) = point_at_pos(&text, old_end);
|
||||
|
||||
// TODO: Position also needs to be byte based...
|
||||
// let byte = char_to_byte(old_pos)
|
||||
// let line = char_to_line(old_pos)
|
||||
// let line_start_byte = line_to_byte()
|
||||
// Position::new(line, line_start_byte - byte)
|
||||
|
||||
// a subsequent ins means a replace, consume it
|
||||
if let Some(Insert(s)) = iter.peek() {
|
||||
iter.next();
|
||||
let ins = s.chars().count();
|
||||
|
||||
// replacement
|
||||
edits.push(tree_sitter::InputEdit {
|
||||
start_byte, // old_pos to byte
|
||||
old_end_byte, // old_end to byte
|
||||
new_end_byte: start_byte + s.len(), // old_pos to byte + s.len()
|
||||
start_position, // old pos to coords
|
||||
old_end_position, // old_end to coords
|
||||
new_end_position: traverse(start_position, s), // old pos + chars, newlines matter too (iter over)
|
||||
});
|
||||
|
||||
new_pos += ins;
|
||||
} else {
|
||||
// deletion
|
||||
edits.push(tree_sitter::InputEdit {
|
||||
start_byte, // old_pos to byte
|
||||
old_end_byte, // old_end to byte
|
||||
new_end_byte: start_byte, // old_pos to byte
|
||||
start_position, // old pos to coords
|
||||
old_end_position, // old_end to coords
|
||||
new_end_position: start_position, // old pos to coords
|
||||
});
|
||||
};
|
||||
}
|
||||
Insert(s) => {
|
||||
let (start_byte, start_position) = point_at_pos(&text, old_pos);
|
||||
|
||||
let ins = s.chars().count();
|
||||
|
||||
// insert
|
||||
edits.push(tree_sitter::InputEdit {
|
||||
start_byte, // old_pos to byte
|
||||
old_end_byte: start_byte, // same
|
||||
new_end_byte: start_byte + s.len(), // old_pos + s.len()
|
||||
start_position, // old pos to coords
|
||||
old_end_position: start_position, // same
|
||||
new_end_position: traverse(start_position, s), // old pos + chars, newlines matter too (iter over)
|
||||
});
|
||||
|
||||
new_pos += ins;
|
||||
}
|
||||
}
|
||||
old_pos = old_end;
|
||||
}
|
||||
edits
|
||||
}
|
||||
|
||||
fn update(
|
||||
&mut self,
|
||||
parser: &mut Parser,
|
||||
config: &HighlightConfiguration,
|
||||
source: &Rope,
|
||||
changeset: &ChangeSet,
|
||||
) -> Result<(), Error> {
|
||||
if changeset.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let edits = Self::generate_edits(&source.slice(..), changeset);
|
||||
|
||||
// Notify the tree about all the changes
|
||||
for edit in edits {
|
||||
self.tree.as_mut().unwrap().edit(&edit);
|
||||
}
|
||||
|
||||
self.parse(
|
||||
parser,
|
||||
config,
|
||||
source,
|
||||
0,
|
||||
// TODO: what to do about this range on update
|
||||
vec![Range {
|
||||
start_byte: 0,
|
||||
end_byte: usize::MAX,
|
||||
start_point: Point::new(0, 0),
|
||||
end_point: Point::new(usize::MAX, usize::MAX),
|
||||
}],
|
||||
)
|
||||
}
|
||||
|
||||
// fn highlight_iter() -> same as Mode but for this layer. Mode composits these
|
||||
// fn buffer_changed
|
||||
// fn update(range)
|
||||
// fn update_injections()
|
||||
}
|
||||
|
||||
// -- refactored from tree-sitter-highlight to be able to retain state
|
||||
// TODO: add seek() to iter
|
||||
@ -169,7 +500,7 @@ struct HighlightIter<'a, F>
|
||||
{
|
||||
source: &'a [u8],
|
||||
byte_offset: usize,
|
||||
highlighter: &'a mut Highlighter,
|
||||
highlighter: &'a mut Syntax,
|
||||
injection_callback: F,
|
||||
cancellation_flag: Option<&'a AtomicUsize>,
|
||||
layers: Vec<HighlightIterLayer<'a>>,
|
||||
@ -179,7 +510,7 @@ struct HighlightIter<'a, F>
|
||||
}
|
||||
|
||||
struct HighlightIterLayer<'a> {
|
||||
_tree: Tree,
|
||||
_tree: Option<Tree>,
|
||||
cursor: QueryCursor,
|
||||
captures: iter::Peekable<QueryCaptures<'a, &'a [u8]>>,
|
||||
config: &'a HighlightConfiguration,
|
||||
@ -207,43 +538,43 @@ pub fn parser(&mut self) -> &mut Parser {
|
||||
&mut self.parser
|
||||
}
|
||||
|
||||
/// Iterate over the highlighted regions for a given slice of source code.
|
||||
pub fn highlight<'a>(
|
||||
&'a mut self,
|
||||
config: &'a HighlightConfiguration,
|
||||
source: &'a [u8],
|
||||
cancellation_flag: Option<&'a AtomicUsize>,
|
||||
mut injection_callback: impl FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
|
||||
) -> Result<impl Iterator<Item = Result<HighlightEvent, Error>> + 'a, Error> {
|
||||
let layers = HighlightIterLayer::new(
|
||||
source,
|
||||
self,
|
||||
cancellation_flag,
|
||||
&mut injection_callback,
|
||||
config,
|
||||
0,
|
||||
vec![Range {
|
||||
start_byte: 0,
|
||||
end_byte: usize::MAX,
|
||||
start_point: Point::new(0, 0),
|
||||
end_point: Point::new(usize::MAX, usize::MAX),
|
||||
}],
|
||||
)?;
|
||||
assert_ne!(layers.len(), 0);
|
||||
let mut result = HighlightIter {
|
||||
source,
|
||||
byte_offset: 0,
|
||||
injection_callback,
|
||||
cancellation_flag,
|
||||
highlighter: self,
|
||||
iter_count: 0,
|
||||
layers,
|
||||
next_event: None,
|
||||
last_highlight_range: None,
|
||||
};
|
||||
result.sort_layers();
|
||||
Ok(result)
|
||||
}
|
||||
// /// Iterate over the highlighted regions for a given slice of source code.
|
||||
// pub fn highlight<'a>(
|
||||
// &'a mut self,
|
||||
// config: &'a HighlightConfiguration,
|
||||
// source: &'a [u8],
|
||||
// cancellation_flag: Option<&'a AtomicUsize>,
|
||||
// mut injection_callback: impl FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
|
||||
// ) -> Result<impl Iterator<Item = Result<HighlightEvent, Error>> + 'a, Error> {
|
||||
// let layers = HighlightIterLayer::new(
|
||||
// source,
|
||||
// self,
|
||||
// cancellation_flag,
|
||||
// &mut injection_callback,
|
||||
// config,
|
||||
// 0,
|
||||
// vec![Range {
|
||||
// start_byte: 0,
|
||||
// end_byte: usize::MAX,
|
||||
// start_point: Point::new(0, 0),
|
||||
// end_point: Point::new(usize::MAX, usize::MAX),
|
||||
// }],
|
||||
// )?;
|
||||
// assert_ne!(layers.len(), 0);
|
||||
// let mut result = HighlightIter {
|
||||
// source,
|
||||
// byte_offset: 0,
|
||||
// injection_callback,
|
||||
// cancellation_flag,
|
||||
// highlighter: self,
|
||||
// iter_count: 0,
|
||||
// layers,
|
||||
// next_event: None,
|
||||
// last_highlight_range: None,
|
||||
// };
|
||||
// result.sort_layers();
|
||||
// Ok(result)
|
||||
// }
|
||||
}
|
||||
|
||||
impl HighlightConfiguration {
|
||||
@ -413,7 +744,7 @@ impl<'a> HighlightIterLayer<'a> {
|
||||
/// added to the returned vector.
|
||||
fn new<F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a>(
|
||||
source: &'a [u8],
|
||||
highlighter: &mut Highlighter,
|
||||
highlighter: &mut Syntax,
|
||||
cancellation_flag: Option<&'a AtomicUsize>,
|
||||
injection_callback: &mut F,
|
||||
mut config: &'a HighlightConfiguration,
|
||||
@ -423,6 +754,8 @@ fn new<F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a>(
|
||||
let mut result = Vec::with_capacity(1);
|
||||
let mut queue = Vec::new();
|
||||
loop {
|
||||
// --> Tree parsing part
|
||||
|
||||
if highlighter.parser.set_included_ranges(&ranges).is_ok() {
|
||||
highlighter
|
||||
.parser
|
||||
@ -474,6 +807,8 @@ fn new<F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a>(
|
||||
}
|
||||
}
|
||||
|
||||
// --> Highlighting query part
|
||||
|
||||
// The `captures` iterator borrows the `Tree` and the `QueryCursor`, which
|
||||
// prevents them from being moved. But both of these values are really just
|
||||
// pointers, so it's actually ok to move them.
|
||||
@ -495,7 +830,7 @@ fn new<F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a>(
|
||||
}],
|
||||
cursor,
|
||||
depth,
|
||||
_tree: tree,
|
||||
_tree: Some(tree),
|
||||
captures,
|
||||
config,
|
||||
ranges,
|
||||
@ -1016,3 +1351,106 @@ fn shrink_and_clear<T>(vec: &mut Vec<T>, capacity: usize) {
|
||||
}
|
||||
vec.clear();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parser() {
|
||||
let highlight_names: Vec<String> = [
|
||||
"attribute",
|
||||
"constant",
|
||||
"function.builtin",
|
||||
"function",
|
||||
"keyword",
|
||||
"operator",
|
||||
"property",
|
||||
"punctuation",
|
||||
"punctuation.bracket",
|
||||
"punctuation.delimiter",
|
||||
"string",
|
||||
"string.special",
|
||||
"tag",
|
||||
"type",
|
||||
"type.builtin",
|
||||
"variable",
|
||||
"variable.builtin",
|
||||
"variable.parameter",
|
||||
]
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(String::from)
|
||||
.collect();
|
||||
|
||||
let language = get_language(&LANG::Rust);
|
||||
let mut config = HighlightConfiguration::new(
|
||||
language,
|
||||
&std::fs::read_to_string(
|
||||
"../helix-syntax/languages/tree-sitter-rust/queries/highlights.scm",
|
||||
)
|
||||
.unwrap(),
|
||||
&std::fs::read_to_string(
|
||||
"../helix-syntax/languages/tree-sitter-rust/queries/injections.scm",
|
||||
)
|
||||
.unwrap(),
|
||||
"", // locals.scm
|
||||
)
|
||||
.unwrap();
|
||||
config.configure(&highlight_names);
|
||||
|
||||
let source = Rope::from_str(
|
||||
"
|
||||
struct Stuff {}
|
||||
fn main() {}
|
||||
",
|
||||
);
|
||||
let syntax = Syntax::new(LANG::Rust, &source, config);
|
||||
let tree = syntax.root_layer.tree.unwrap();
|
||||
let root = tree.root_node();
|
||||
assert_eq!(root.kind(), "source_file");
|
||||
|
||||
assert_eq!(
|
||||
root.to_sexp(),
|
||||
concat!(
|
||||
"(source_file ",
|
||||
"(struct_item name: (type_identifier) body: (field_declaration_list)) ",
|
||||
"(function_item name: (identifier) parameters: (parameters) body: (block)))"
|
||||
)
|
||||
);
|
||||
|
||||
let struct_node = root.child(0).unwrap();
|
||||
assert_eq!(struct_node.kind(), "struct_item");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_input_edits() {
|
||||
use crate::State;
|
||||
use tree_sitter::InputEdit;
|
||||
|
||||
let mut state = State::new("hello world!\ntest 123".into());
|
||||
let transaction = Transaction::change(
|
||||
&state,
|
||||
vec![(6, 11, Some("test".into())), (12, 17, None)].into_iter(),
|
||||
);
|
||||
let edits = LanguageLayer::generate_edits(&state.doc.slice(..), &transaction.changes);
|
||||
// transaction.apply(&mut state);
|
||||
|
||||
assert_eq!(
|
||||
edits,
|
||||
&[
|
||||
InputEdit {
|
||||
start_byte: 6,
|
||||
old_end_byte: 11,
|
||||
new_end_byte: 10,
|
||||
start_position: Point { row: 0, column: 6 },
|
||||
old_end_position: Point { row: 0, column: 11 },
|
||||
new_end_position: Point { row: 0, column: 10 }
|
||||
},
|
||||
InputEdit {
|
||||
start_byte: 12,
|
||||
old_end_byte: 17,
|
||||
new_end_byte: 12,
|
||||
start_position: Point { row: 0, column: 12 },
|
||||
old_end_position: Point { row: 1, column: 4 },
|
||||
new_end_position: Point { row: 0, column: 12 }
|
||||
}
|
||||
]
|
||||
);
|
||||
}
|
||||
|
@ -4,7 +4,7 @@
|
||||
pub type Change = (usize, usize, Option<Tendril>);
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
enum Operation {
|
||||
pub(crate) enum Operation {
|
||||
/// Move cursor by n characters.
|
||||
Retain(usize),
|
||||
/// Delete n characters.
|
||||
@ -22,7 +22,7 @@ pub enum Assoc {
|
||||
// ChangeSpec = Change | ChangeSet | Vec<Change>
|
||||
#[derive(Debug)]
|
||||
pub struct ChangeSet {
|
||||
changes: Vec<Operation>,
|
||||
pub(crate) changes: Vec<Operation>,
|
||||
/// The required document length. Will refuse to apply changes unless it matches.
|
||||
len: usize,
|
||||
}
|
||||
@ -307,7 +307,7 @@ pub fn map_pos(&self, pos: usize, assoc: Assoc) -> usize {
|
||||
/// a single transaction.
|
||||
pub struct Transaction {
|
||||
/// Changes made to the buffer.
|
||||
changes: ChangeSet,
|
||||
pub(crate) changes: ChangeSet,
|
||||
/// When set, explicitly updates the selection.
|
||||
selection: Option<Selection>,
|
||||
// effects, annotations
|
||||
@ -337,6 +337,14 @@ pub fn apply(&self, state: &mut State) -> bool {
|
||||
.clone()
|
||||
.unwrap_or_else(|| state.selection.clone().map(&self.changes));
|
||||
|
||||
// TODO: no unwrap
|
||||
state
|
||||
.syntax
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.update(&state.doc, &self.changes)
|
||||
.unwrap();
|
||||
|
||||
true
|
||||
}
|
||||
|
||||
@ -359,9 +367,15 @@ pub fn change<I>(state: &State, changes: I) -> Self
|
||||
for (from, to, tendril) in changes {
|
||||
// Retain from last "to" to current "from"
|
||||
acc.push(Operation::Retain(from - last));
|
||||
let span = to - from;
|
||||
match tendril {
|
||||
Some(text) => acc.push(Operation::Insert(text)),
|
||||
None => acc.push(Operation::Delete(to - from)),
|
||||
Some(text) => {
|
||||
if span > 0 {
|
||||
acc.push(Operation::Delete(span));
|
||||
}
|
||||
acc.push(Operation::Insert(text));
|
||||
}
|
||||
None => acc.push(Operation::Delete(span)),
|
||||
}
|
||||
last = to;
|
||||
}
|
||||
@ -466,4 +480,15 @@ fn map_pos() {
|
||||
assert_eq!(cs.map_pos(2, Assoc::Before), 2);
|
||||
assert_eq!(cs.map_pos(2, Assoc::After), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transaction_change() {
|
||||
let mut state = State::new("hello world!\ntest 123".into());
|
||||
let transaction = Transaction::change(
|
||||
&state,
|
||||
vec![(6, 11, Some("void".into())), (12, 17, None)].into_iter(),
|
||||
);
|
||||
transaction.apply(&mut state);
|
||||
assert_eq!(state.doc, Rope::from_str("hello void! 123"));
|
||||
}
|
||||
}
|
||||
|
@ -11,11 +11,11 @@ name = "hx"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
helix-core = { path = "../helix-core" }
|
||||
helix-syntax = { path = "../helix-syntax" }
|
||||
|
||||
anyhow = "1"
|
||||
argh = "0.1.3"
|
||||
helix-core = { path = "../helix-core" }
|
||||
crossterm = { version = "0.17", features = ["event-stream"] }
|
||||
|
||||
smol = "1"
|
||||
|
@ -38,8 +38,6 @@ pub struct Editor {
|
||||
size: (u16, u16),
|
||||
surface: Surface,
|
||||
theme: Theme,
|
||||
highlighter: Highlighter,
|
||||
highlight_config: HighlightConfiguration,
|
||||
}
|
||||
|
||||
impl Editor {
|
||||
@ -49,33 +47,8 @@ pub fn new(mut args: Args) -> Result<Self, Error> {
|
||||
let mut terminal = Terminal::new(backend)?;
|
||||
let size = terminal::size().unwrap();
|
||||
let area = Rect::new(0, 0, size.0, size.1);
|
||||
|
||||
let theme = Theme::default();
|
||||
|
||||
// let mut parser = tree_sitter::Parser::new();
|
||||
// parser.set_language(language).unwrap();
|
||||
// let tree = parser.parse(source_code, None).unwrap();
|
||||
|
||||
let language = helix_syntax::get_language(&helix_syntax::LANG::Rust);
|
||||
|
||||
let highlighter = Highlighter::new();
|
||||
|
||||
let mut highlight_config = HighlightConfiguration::new(
|
||||
language,
|
||||
&std::fs::read_to_string(
|
||||
"../helix-syntax/languages/tree-sitter-rust/queries/highlights.scm",
|
||||
)
|
||||
.unwrap(),
|
||||
&std::fs::read_to_string(
|
||||
"../helix-syntax/languages/tree-sitter-rust/queries/injections.scm",
|
||||
)
|
||||
.unwrap(),
|
||||
"", // locals.scm
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
highlight_config.configure(theme.scopes());
|
||||
|
||||
let mut editor = Editor {
|
||||
terminal,
|
||||
state: None,
|
||||
@ -84,8 +57,6 @@ pub fn new(mut args: Args) -> Result<Self, Error> {
|
||||
surface: Surface::empty(area),
|
||||
theme,
|
||||
// TODO; move to state
|
||||
highlighter,
|
||||
highlight_config,
|
||||
};
|
||||
|
||||
if let Some(file) = args.files.pop() {
|
||||
@ -96,12 +67,19 @@ pub fn new(mut args: Args) -> Result<Self, Error> {
|
||||
}
|
||||
|
||||
pub fn open(&mut self, path: PathBuf) -> Result<(), Error> {
|
||||
self.state = Some(State::load(path)?);
|
||||
let mut state = State::load(path)?;
|
||||
state
|
||||
.syntax
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.configure(self.theme.scopes());
|
||||
self.state = Some(state);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn render(&mut self) {
|
||||
match &self.state {
|
||||
// TODO: ideally not mut but highlights require it because of cursor cache
|
||||
match &mut self.state {
|
||||
Some(state) => {
|
||||
let area = Rect::new(0, 0, self.size.0, self.size.1);
|
||||
let mut surface = Surface::empty(area);
|
||||
@ -112,12 +90,13 @@ fn render(&mut self) {
|
||||
|
||||
// TODO: cache highlight results
|
||||
// TODO: only recalculate when state.doc is actually modified
|
||||
let highlights = self
|
||||
.highlighter
|
||||
.highlight(&self.highlight_config, source_code.as_bytes(), None, |_| {
|
||||
None
|
||||
})
|
||||
.unwrap();
|
||||
let highlights: Vec<_> = state
|
||||
.syntax
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.highlight_iter(source_code.as_bytes(), None, |_| None)
|
||||
.unwrap()
|
||||
.collect(); // TODO: we collect here to avoid double borrow, fix later
|
||||
|
||||
let mut spans = Vec::new();
|
||||
|
||||
@ -235,7 +214,7 @@ fn render(&mut self) {
|
||||
let coords = coords_at_pos(&state.doc().slice(..), pos);
|
||||
execute!(
|
||||
stdout,
|
||||
cursor::MoveTo((coords.1 + 2) as u16, coords.0 as u16)
|
||||
cursor::MoveTo((coords.col + 2) as u16, coords.row as u16)
|
||||
);
|
||||
}
|
||||
None => (),
|
||||
|
Loading…
Reference in New Issue
Block a user