088f8a8 · helix · git.iko.soy

git.iko.soy/helix

on master help sign in

Incremental parsing: rough draft.

088f8a8Blaž Hrastnik6 years ago

.gitignore+1

compare with master

unifiedsplit

@@ -1,2 +1,3 @@

1target

2helix-term/rustfmt.toml

3+helix-syntax/languages/

Cargo.lock+1

@@ -369,6 +369,7 @@ name = "helix-core"

369version = "0.1.0"

370dependencies = [

371 "anyhow",

372+ "helix-syntax",

372 "ropey",

373 "ropey",

373 "smallvec",

374 "smallvec",

374 "tendril",

375 "tendril",

Cargo.toml+2−2

@@ -6,5 +6,5 @@ members = [

8# Build helix-syntax in release mode to make the code path faster in development.

9−[profile.dev.package."helix-syntax"]

9+# [profile.dev.package."helix-syntax"]

10−opt-level = 3

10+# opt-level = 3

helix-core/Cargo.toml+2

@@ -7,6 +7,8 @@ edition = "2018"

7# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

9[dependencies]

10+helix-syntax = { path = "../helix-syntax" }

11+

10ropey = "1.2.0"

12ropey = "1.2.0"

11anyhow = "1.0.31"

13anyhow = "1.0.31"

12smallvec = "1.4.0"

14smallvec = "1.4.0"

helix-core/src/lib.rs+3

@@ -1,6 +1,7 @@

1#![allow(unused)]

2pub mod commands;

3pub mod graphemes;

4+mod position;

4mod selection;

5mod selection;

5pub mod state;

6pub mod state;

6pub mod syntax;

7pub mod syntax;

@@ -9,8 +10,10 @@ mod transaction;

9pub use ropey::{Rope, RopeSlice};

10pub use ropey::{Rope, RopeSlice};

10pub use tendril::StrTendril as Tendril;

11pub use tendril::StrTendril as Tendril;

13+pub use position::Position;

12pub use selection::Range as SelectionRange;

14pub use selection::Range as SelectionRange;

13pub use selection::Selection;

15pub use selection::Selection;

16+pub use syntax::Syntax;

15pub use state::State;

18pub use state::State;

helix-core/src/position.rsnew file+57

@@ -0,0 +1,57 @@

1+/// Represents a single point in a text buffer. Zero indexed.

2+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]

3+pub struct Position {

4+ pub row: usize,

5+ pub col: usize,

6+}

8+impl Position {

9+ pub const fn new(row: usize, col: usize) -> Self {

10+ Self { row, col }

11+ }

12+

13+ pub const fn is_zero(self) -> bool {

14+ self.row == 0 && self.col == 0

15+ }

16+

17+ // TODO: generalize

18+ pub fn traverse(self, text: &crate::Tendril) -> Self {

19+ let Self { mut row, mut col } = self;

20+ // TODO: there should be a better way here

21+ for ch in text.chars() {

22+ if ch == '\n' {

23+ row += 1;

24+ col = 0;

25+ } else {

26+ col += 1;

27+ }

28+ }

29+ Self { row, col }

30+ }

31+}

32+

33+impl From<(usize, usize)> for Position {

34+ fn from(tuple: (usize, usize)) -> Self {

35+ Position {

36+ row: tuple.0,

37+ col: tuple.1,

38+ }

39+ }

40+}

41+

42+impl Into<tree_sitter::Point> for Position {

43+ fn into(self) -> tree_sitter::Point {

44+ tree_sitter::Point::new(self.row, self.col)

45+ }

46+}

47+

48+#[cfg(test)]

49+mod test {

50+ use super::*;

51+

52+ #[test]

53+ fn test_ordering() {

54+ // (0, 5) is less than (1, 0)

55+ assert!(Position::new(0, 5) < Position::new(1, 0));

56+ }

57+}

helix-core/src/state.rs+50−23

@@ -1,5 +1,5 @@

1use crate::graphemes::{nth_next_grapheme_boundary, nth_prev_grapheme_boundary, RopeGraphemes};

2−use crate::{Rope, RopeSlice, Selection, SelectionRange};

2+use crate::{Position, Rope, RopeSlice, Selection, SelectionRange, Syntax};

3use anyhow::Error;

5use std::path::PathBuf;

@@ -17,6 +17,9 @@ pub struct State {

17 pub(crate) doc: Rope,

18 pub(crate) selection: Selection,

19 pub(crate) mode: Mode,

20+

21+ //

22+ pub syntax: Option<Syntax>,

20}

23}

22#[derive(Copy, Clone, PartialEq, Eq)]

25#[derive(Copy, Clone, PartialEq, Eq)]

@@ -40,6 +43,7 @@ impl State {

40 doc,

43 doc,

41 selection: Selection::single(0, 0),

44 selection: Selection::single(0, 0),

42 mode: Mode::Normal,

45 mode: Mode::Normal,

46+ syntax: None,

43 }

47 }

44 }

48 }

@@ -54,6 +58,29 @@ impl State {

54 let mut state = Self::new(doc);

58 let mut state = Self::new(doc);

55 state.path = Some(path);

59 state.path = Some(path);

61+ let language = helix_syntax::get_language(&helix_syntax::LANG::Rust);

62+

63+ let mut highlight_config = crate::syntax::HighlightConfiguration::new(

64+ language,

65+ &std::fs::read_to_string(

66+ "../helix-syntax/languages/tree-sitter-rust/queries/highlights.scm",

67+ )

68+ .unwrap(),

69+ &std::fs::read_to_string(

70+ "../helix-syntax/languages/tree-sitter-rust/queries/injections.scm",

71+ )

72+ .unwrap(),

73+ "", // locals.scm

74+ )

75+ .unwrap();

76+

77+ // TODO: config.configure(scopes) is now delayed, is that ok?

78+

79+ // TODO: get_language is called twice

80+ let syntax = Syntax::new(helix_syntax::LANG::Rust, &state.doc, highlight_config);

81+

82+ state.syntax = Some(syntax);

83+

57 Ok(state)

84 Ok(state)

58 }

85 }

@@ -175,29 +202,29 @@ impl State {

175}

202}

176

203

177/// Coordinates are a 0-indexed line and column pair.

204/// Coordinates are a 0-indexed line and column pair.

178−type Coords = (usize, usize); // line, col

205+pub type Coords = (usize, usize); // line, col

179

206

180/// Convert a character index to (line, column) coordinates.

207/// Convert a character index to (line, column) coordinates.

181−pub fn coords_at_pos(text: &RopeSlice, pos: usize) -> Coords {

208+pub fn coords_at_pos(text: &RopeSlice, pos: usize) -> Position {

182 let line = text.char_to_line(pos);

209 let line = text.char_to_line(pos);

183 let line_start = text.line_to_char(line);

210 let line_start = text.line_to_char(line);

184 let col = text.slice(line_start..pos).len_chars();

211 let col = text.slice(line_start..pos).len_chars();

185− (line, col)

212+ Position::new(line, col)

186}

213}

187

214

188/// Convert (line, column) coordinates to a character index.

215/// Convert (line, column) coordinates to a character index.

189−pub fn pos_at_coords(text: &RopeSlice, coords: Coords) -> usize {

216+pub fn pos_at_coords(text: &RopeSlice, coords: Position) -> usize {

190− let (line, col) = coords;

217+ let Position { row, col } = coords;

191− let line_start = text.line_to_char(line);

218+ let line_start = text.line_to_char(row);

192 nth_next_grapheme_boundary(text, line_start, col)

219 nth_next_grapheme_boundary(text, line_start, col)

193}

220}

194

221

195fn move_vertically(text: &RopeSlice, dir: Direction, pos: usize, count: usize) -> usize {

222fn move_vertically(text: &RopeSlice, dir: Direction, pos: usize, count: usize) -> usize {

196− let (line, col) = coords_at_pos(text, pos);

223+ let Position { row, col } = coords_at_pos(text, pos);

197

224

198 let new_line = match dir {

225 let new_line = match dir {

199− Direction::Backward => line.saturating_sub(count),

226+ Direction::Backward => row.saturating_sub(count),

200− Direction::Forward => std::cmp::min(line.saturating_add(count), text.len_lines() - 1),

227+ Direction::Forward => std::cmp::min(row.saturating_add(count), text.len_lines() - 1),

201 };

228 };

202

229

203 // convert to 0-indexed, subtract another 1 because len_chars() counts \n

230 // convert to 0-indexed, subtract another 1 because len_chars() counts \n

@@ -210,7 +237,7 @@ fn move_vertically(text: &RopeSlice, dir: Direction, pos: usize, count: usize) -

210 col

237 col

211 };

238 };

212

239

213− pos_at_coords(text, (new_line, new_col))

240+ pos_at_coords(text, Position::new(new_line, new_col))

214}

241}

215

242

216#[cfg(test)]

243#[cfg(test)]

@@ -220,32 +247,32 @@ mod test {

220 #[test]

247 #[test]

221 fn test_coords_at_pos() {

248 fn test_coords_at_pos() {

222 let text = Rope::from("ḧëḷḷö\nẅöṛḷḋ");

249 let text = Rope::from("ḧëḷḷö\nẅöṛḷḋ");

223− assert_eq!(coords_at_pos(&text.slice(..), 0), (0, 0));

250+ assert_eq!(coords_at_pos(&text.slice(..), 0), (0, 0).into());

224− assert_eq!(coords_at_pos(&text.slice(..), 5), (0, 5)); // position on \n

251+ assert_eq!(coords_at_pos(&text.slice(..), 5), (0, 5).into()); // position on \n

225− assert_eq!(coords_at_pos(&text.slice(..), 6), (1, 0)); // position on w

252+ assert_eq!(coords_at_pos(&text.slice(..), 6), (1, 0).into()); // position on w

226− assert_eq!(coords_at_pos(&text.slice(..), 7), (1, 1)); // position on o

253+ assert_eq!(coords_at_pos(&text.slice(..), 7), (1, 1).into()); // position on o

227− assert_eq!(coords_at_pos(&text.slice(..), 10), (1, 4)); // position on d

254+ assert_eq!(coords_at_pos(&text.slice(..), 10), (1, 4).into()); // position on d

228 }

255 }

229

256

230 #[test]

257 #[test]

231 fn test_pos_at_coords() {

258 fn test_pos_at_coords() {

232 let text = Rope::from("ḧëḷḷö\nẅöṛḷḋ");

259 let text = Rope::from("ḧëḷḷö\nẅöṛḷḋ");

233− assert_eq!(pos_at_coords(&text.slice(..), (0, 0)), 0);

260+ assert_eq!(pos_at_coords(&text.slice(..), (0, 0).into()), 0);

234− assert_eq!(pos_at_coords(&text.slice(..), (0, 5)), 5); // position on \n

261+ assert_eq!(pos_at_coords(&text.slice(..), (0, 5).into()), 5); // position on \n

235− assert_eq!(pos_at_coords(&text.slice(..), (1, 0)), 6); // position on w

262+ assert_eq!(pos_at_coords(&text.slice(..), (1, 0).into()), 6); // position on w

236− assert_eq!(pos_at_coords(&text.slice(..), (1, 1)), 7); // position on o

263+ assert_eq!(pos_at_coords(&text.slice(..), (1, 1).into()), 7); // position on o

237− assert_eq!(pos_at_coords(&text.slice(..), (1, 4)), 10); // position on d

264+ assert_eq!(pos_at_coords(&text.slice(..), (1, 4).into()), 10); // position on d

238 }

265 }

239

266

240 #[test]

267 #[test]

241 fn test_vertical_move() {

268 fn test_vertical_move() {

242 let text = Rope::from("abcd\nefg\nwrs");

269 let text = Rope::from("abcd\nefg\nwrs");

243− let pos = pos_at_coords(&text.slice(..), (0, 4));

270+ let pos = pos_at_coords(&text.slice(..), (0, 4).into());

244 let slice = text.slice(..);

271 let slice = text.slice(..);

245

272

246 assert_eq!(

273 assert_eq!(

247 coords_at_pos(&slice, move_vertically(&slice, Direction::Forward, pos, 1)),

274 coords_at_pos(&slice, move_vertically(&slice, Direction::Forward, pos, 1)),

248− (1, 2)

275+ (1, 2).into()

249 );

276 );

250 }

277 }

251}

278}

helix-core/src/syntax.rs+536−98

@@ -1,66 +1,397 @@

1+use crate::{Change, Rope, RopeSlice, Transaction};

2+pub use helix_syntax::LANG;

3+pub use helix_syntax::{get_language, get_language_name};

1−// pub struct Syntax {

5+pub struct Syntax {

6+ grammar: Language,

2−// parser: Parser,

7+ parser: Parser,

3−// }

8+ cursors: Vec<QueryCursor>,

10+ config: HighlightConfiguration,

12+ root_layer: LanguageLayer,

13+}

14+

5−//impl Syntax {

15+impl Syntax {

6−// // buffer, grammar, config, grammars, sync_timeout?

16+ // buffer, grammar, config, grammars, sync_timeout?

7−// pub fn new() -> Self {

17+ pub fn new(language: LANG, source: &Rope, config: HighlightConfiguration) -> Self {

8−// unimplemented!()

18+ // fetch grammar for parser based on language string

9−// // make a new root layer

19+ let grammar = get_language(&language);

20+ let parser = Parser::new();

21+

22+ let root_layer = LanguageLayer::new();

23+

10−// // track markers of injections

24+ // track markers of injections

11−// //

12−// // track scope_descriptor: a Vec of scopes for item in tree

25+ // track scope_descriptor: a Vec of scopes for item in tree

13−// //

26+

14−// // fetch grammar for parser based on language string

27+ let mut syntax = Self {

15−// // update root layer

28+ grammar,

29+ parser,

30+ cursors: Vec::new(),

31+ config,

32+ root_layer,

33+ };

34+

35+ // update root layer

36+ syntax.root_layer.parse(

37+ &mut syntax.parser,

38+ &syntax.config,

39+ source,

40+ 0,

41+ vec![Range {

42+ start_byte: 0,

43+ end_byte: usize::MAX,

44+ start_point: Point::new(0, 0),

45+ end_point: Point::new(usize::MAX, usize::MAX),

46+ }],

47+ );

48+ syntax

49+ }

50+

51+ pub fn configure(&mut self, scopes: &[String]) {

52+ self.config.configure(scopes)

53+ }

54+

55+ pub fn update(&mut self, source: &Rope, changeset: &ChangeSet) -> Result<(), Error> {

56+ self.root_layer

57+ .update(&mut self.parser, &self.config, source, changeset)

58+

59+ // TODO: deal with injections and update them too

16−// }

60+ }

17−

61+

18−// // fn buffer_changed -> call layer.update(range, new_text) on root layer and then all marker layers

62+ // fn buffer_changed -> call layer.update(range, new_text) on root layer and then all marker layers

19−

63+

20−// // call this on transaction.apply() -> buffer_changed(changes)

64+ // call this on transaction.apply() -> buffer_changed(changes)

21−// //

65+ //

22−// // fn parse(language, old_tree, ranges)

66+ // fn parse(language, old_tree, ranges)

23−// //

67+ //

24−// // fn tree() -> Tree

68+ fn tree(&self) -> &Tree {

25−// //

69+ self.root_layer.tree()

70+ }

71+ //

26−// //

72+ //

27−

73+

28−// // Highlighting

74+ // Highlighting

29−// // fn highlight_iter() -> iterates over all the scopes

75+

76+ /// Iterate over the highlighted regions for a given slice of source code.

77+ pub fn highlight_iter<'a>(

78+ &'a mut self,

79+ source: &'a [u8],

80+ cancellation_flag: Option<&'a AtomicUsize>,

81+ mut injection_callback: impl FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,

82+ ) -> Result<impl Iterator<Item = Result<HighlightEvent, Error>> + 'a, Error> {

83+ // The `captures` iterator borrows the `Tree` and the `QueryCursor`, which

84+ // prevents them from being moved. But both of these values are really just

85+ // pointers, so it's actually ok to move them.

86+

87+ let mut cursor = QueryCursor::new(); // reuse a pool

88+ let tree_ref = unsafe { mem::transmute::<_, &'static Tree>(self.tree()) };

89+ let cursor_ref = unsafe { mem::transmute::<_, &'static mut QueryCursor>(&mut cursor) };

90+ let query_ref = unsafe { mem::transmute::<_, &'static mut Query>(&mut self.config.query) };

91+ let config_ref =

92+ unsafe { mem::transmute::<_, &'static HighlightConfiguration>(&self.config) };

93+ let captures = cursor_ref

94+ .captures(query_ref, tree_ref.root_node(), move |n: Node| {

95+ &source[n.byte_range()]

96+ })

97+ .peekable();

98+

99+ // manually craft the root layer based on the existing tree

100+ let layer = HighlightIterLayer {

101+ highlight_end_stack: Vec::new(),

102+ scope_stack: vec![LocalScope {

103+ inherits: false,

104+ range: 0..usize::MAX,

105+ local_defs: Vec::new(),

106+ }],

107+ cursor,

108+ depth: 0,

109+ _tree: None,

110+ captures,

111+ config: config_ref,

112+ ranges: vec![Range {

113+ start_byte: 0,

114+ end_byte: usize::MAX,

115+ start_point: Point::new(0, 0),

116+ end_point: Point::new(usize::MAX, usize::MAX),

117+ }],

118+ };

119+

120+ let mut result = HighlightIter {

121+ source,

122+ byte_offset: 0,

123+ injection_callback,

124+ cancellation_flag,

125+ highlighter: self,

126+ iter_count: 0,

127+ layers: vec![layer],

128+ next_event: None,

129+ last_highlight_range: None,

130+ };

131+ result.sort_layers();

132+ Ok(result)

133+ }

30−// // on_tokenize

134+ // on_tokenize

31−// // on_change_highlighting

135+ // on_change_highlighting

32−

136+

33−// // Commenting

137+ // Commenting

34−// // comment_strings_for_pos

138+ // comment_strings_for_pos

35−// // is_commented

139+ // is_commented

36−

140+

37−// // Indentation

141+ // Indentation

38−// // suggested_indent_for_line_at_buffer_row

142+ // suggested_indent_for_line_at_buffer_row

39−// // suggested_indent_for_buffer_row

143+ // suggested_indent_for_buffer_row

40−// // indent_level_for_line

144+ // indent_level_for_line

41−

145+

42−// // TODO: Folding

146+ // TODO: Folding

43−

147+

44−// // Syntax APIs

148+ // Syntax APIs

45−// // get_syntax_node_containing_range ->

149+ // get_syntax_node_containing_range ->

46−// // ...

150+ // ...

47−// // get_syntax_node_at_pos

151+ // get_syntax_node_at_pos

48−// // buffer_range_for_scope_at_pos

152+ // buffer_range_for_scope_at_pos

49−//}

153+}

154

51pub struct LanguageLayer {

155pub struct LanguageLayer {

52 // mode

156 // mode

53−// grammar

157+ // grammar

54−// depth

158+ // depth

55−// tree: Tree,

159+ tree: Option<Tree>,

56}

160}

161

162+use crate::state::{coords_at_pos, Coords};

163+use crate::transaction::{ChangeSet, Operation};

164+use crate::Tendril;

165+

58−// impl LanguageLayer {

166+impl LanguageLayer {

167+ pub fn new() -> Self {

168+ Self { tree: None }

169+ }

170+

171+ fn tree(&self) -> &Tree {

172+ // TODO: no unwrap

173+ self.tree.as_ref().unwrap()

174+ }

175+

176+ fn parse<'a>(

177+ &mut self,

178+ parser: &mut Parser,

179+ config: &HighlightConfiguration,

180+ source: &Rope,

181+ mut depth: usize,

182+ mut ranges: Vec<Range>,

183+ ) -> Result<(), Error> {

184+ if parser.set_included_ranges(&ranges).is_ok() {

185+ parser

186+ .set_language(config.language)

187+ .map_err(|_| Error::InvalidLanguage)?;

188+

189+ // unsafe { syntax.parser.set_cancellation_flag(cancellation_flag) };

190+ let tree = parser

191+ .parse_with(

192+ &mut |byte, _| {

193+ if byte <= source.len_bytes() {

194+ let (chunk, start_byte, _, _) = source.chunk_at_byte(byte);

195+ chunk[byte - start_byte..].as_bytes()

196+ } else {

197+ // out of range

198+ &[]

199+ }

200+ },

201+ self.tree.as_ref(),

202+ )

203+ .ok_or(Error::Cancelled)?;

204+ // unsafe { syntax.parser.set_cancellation_flag(None) };

205+ // let mut cursor = syntax.cursors.pop().unwrap_or_else(QueryCursor::new);

206+

207+ // Process combined injections. (ERB, EJS, etc https://github.com/tree-sitter/tree-sitter/pull/526)

208+ // if let Some(combined_injections_query) = &config.combined_injections_query {

209+ // let mut injections_by_pattern_index =

210+ // vec![(None, Vec::new(), false); combined_injections_query.pattern_count()];

211+ // let matches =

212+ // cursor.matches(combined_injections_query, tree.root_node(), |n: Node| {

213+ // &source[n.byte_range()]

214+ // });

215+ // for mat in matches {

216+ // let entry = &mut injections_by_pattern_index[mat.pattern_index];

217+ // let (language_name, content_node, include_children) =

218+ // injection_for_match(config, combined_injections_query, &mat, source);

219+ // if language_name.is_some() {

220+ // entry.0 = language_name;

221+ // }

222+ // if let Some(content_node) = content_node {

223+ // entry.1.push(content_node);

224+ // }

225+ // entry.2 = include_children;

226+ // }

227+ // for (lang_name, content_nodes, includes_children) in injections_by_pattern_index {

228+ // if let (Some(lang_name), false) = (lang_name, content_nodes.is_empty()) {

229+ // if let Some(next_config) = (injection_callback)(lang_name) {

230+ // let ranges =

231+ // Self::intersect_ranges(&ranges, &content_nodes, includes_children);

232+ // if !ranges.is_empty() {

233+ // queue.push((next_config, depth + 1, ranges));

234+ // }

235+ // }

236+ // }

237+ // }

238+ // }

239+ self.tree = Some(tree)

240+ }

241+ Ok(())

242+ }

243+

244+ pub(crate) fn generate_edits(

245+ text: &RopeSlice,

246+ changeset: &ChangeSet,

247+ ) -> Vec<tree_sitter::InputEdit> {

248+ use Operation::*;

249+ let mut old_pos = 0;

250+ let mut new_pos = 0;

251+

252+ let mut edits = Vec::new();

253+

254+ let mut iter = changeset.changes.iter().peekable();

255+

256+ // TODO; this is a lot easier with Change instead of Operation.

257+

258+ fn point_at_pos(text: &RopeSlice, pos: usize) -> (usize, Point) {

259+ let byte = text.char_to_byte(pos);

260+ let line = text.char_to_line(pos);

261+ let line_start_byte = text.line_to_byte(line);

262+ let col = byte - line_start_byte;

263+

264+ (byte, Point::new(line, col))

265+ }

266+

267+ fn traverse(point: Point, text: &Tendril) -> Point {

268+ let Point {

269+ mut row,

270+ mut column,

271+ } = point;

272+

273+ // TODO: there should be a better way here

274+ for ch in text.bytes() {

275+ if ch == b'\n' {

276+ row += 1;

277+ column = 0;

278+ } else {

279+ column += 1;

280+ }

281+ }

282+ Point { row, column }

283+ }

284+

285+ while let Some(change) = iter.next() {

286+ let len = match change {

287+ Delete(i) | Retain(i) => *i,

288+ Insert(_) => 0,

289+ };

290+ let old_end = old_pos + len;

291+

292+ match change {

293+ Retain(_) => {

294+ new_pos += len;

295+ }

296+ Delete(_) => {

297+ let (start_byte, start_position) = point_at_pos(&text, old_pos);

298+ let (old_end_byte, old_end_position) = point_at_pos(&text, old_end);

299+

300+ // TODO: Position also needs to be byte based...

301+ // let byte = char_to_byte(old_pos)

302+ // let line = char_to_line(old_pos)

303+ // let line_start_byte = line_to_byte()

304+ // Position::new(line, line_start_byte - byte)

305+

306+ // a subsequent ins means a replace, consume it

307+ if let Some(Insert(s)) = iter.peek() {

308+ iter.next();

309+ let ins = s.chars().count();

310+

311+ // replacement

312+ edits.push(tree_sitter::InputEdit {

313+ start_byte, // old_pos to byte

314+ old_end_byte, // old_end to byte

315+ new_end_byte: start_byte + s.len(), // old_pos to byte + s.len()

316+ start_position, // old pos to coords

317+ old_end_position, // old_end to coords

318+ new_end_position: traverse(start_position, s), // old pos + chars, newlines matter too (iter over)

319+ });

320+

321+ new_pos += ins;

322+ } else {

323+ // deletion

324+ edits.push(tree_sitter::InputEdit {

325+ start_byte, // old_pos to byte

326+ old_end_byte, // old_end to byte

327+ new_end_byte: start_byte, // old_pos to byte

328+ start_position, // old pos to coords

329+ old_end_position, // old_end to coords

330+ new_end_position: start_position, // old pos to coords

331+ });

332+ };

333+ }

334+ Insert(s) => {

335+ let (start_byte, start_position) = point_at_pos(&text, old_pos);

336+

337+ let ins = s.chars().count();

338+

339+ // insert

340+ edits.push(tree_sitter::InputEdit {

341+ start_byte, // old_pos to byte

342+ old_end_byte: start_byte, // same

343+ new_end_byte: start_byte + s.len(), // old_pos + s.len()

344+ start_position, // old pos to coords

345+ old_end_position: start_position, // same

346+ new_end_position: traverse(start_position, s), // old pos + chars, newlines matter too (iter over)

347+ });

348+

349+ new_pos += ins;

350+ }

351+ }

352+ old_pos = old_end;

353+ }

354+ edits

355+ }

356+

357+ fn update(

358+ &mut self,

359+ parser: &mut Parser,

360+ config: &HighlightConfiguration,

361+ source: &Rope,

362+ changeset: &ChangeSet,

363+ ) -> Result<(), Error> {

364+ if changeset.is_empty() {

365+ return Ok(());

366+ }

367+

368+ let edits = Self::generate_edits(&source.slice(..), changeset);

369+

370+ // Notify the tree about all the changes

371+ for edit in edits {

372+ self.tree.as_mut().unwrap().edit(&edit);

373+ }

374+

375+ self.parse(

376+ parser,

377+ config,

378+ source,

379+ 0,

380+ // TODO: what to do about this range on update

381+ vec![Range {

382+ start_byte: 0,

383+ end_byte: usize::MAX,

384+ start_point: Point::new(0, 0),

385+ end_point: Point::new(usize::MAX, usize::MAX),

386+ }],

387+ )

388+ }

389+

59−// // fn highlight_iter() -> same as Mode but for this layer. Mode composits these

390+ // fn highlight_iter() -> same as Mode but for this layer. Mode composits these

60−// // fn buffer_changed

391+ // fn buffer_changed

61−// // fn update(range)

392+ // fn update(range)

62−// // fn update_injections()

393+ // fn update_injections()

63−// }

394+}

395

65// -- refactored from tree-sitter-highlight to be able to retain state

396// -- refactored from tree-sitter-highlight to be able to retain state

66// TODO: add seek() to iter

397// TODO: add seek() to iter

@@ -169,7 +500,7 @@ where

169{

500{

170 source: &'a [u8],

501 source: &'a [u8],

171 byte_offset: usize,

502 byte_offset: usize,

172− highlighter: &'a mut Highlighter,

503+ highlighter: &'a mut Syntax,

173 injection_callback: F,

504 injection_callback: F,

174 cancellation_flag: Option<&'a AtomicUsize>,

505 cancellation_flag: Option<&'a AtomicUsize>,

175 layers: Vec<HighlightIterLayer<'a>>,

506 layers: Vec<HighlightIterLayer<'a>>,

@@ -179,7 +510,7 @@ where

179}

510}

180

511

181struct HighlightIterLayer<'a> {

512struct HighlightIterLayer<'a> {

182− _tree: Tree,

513+ _tree: Option<Tree>,

183 cursor: QueryCursor,

514 cursor: QueryCursor,

184 captures: iter::Peekable<QueryCaptures<'a, &'a [u8]>>,

515 captures: iter::Peekable<QueryCaptures<'a, &'a [u8]>>,

185 config: &'a HighlightConfiguration,

516 config: &'a HighlightConfiguration,

@@ -207,43 +538,43 @@ impl Highlighter {

207 &mut self.parser

538 &mut self.parser

208 }

539 }

209

540

210− /// Iterate over the highlighted regions for a given slice of source code.

541+ // /// Iterate over the highlighted regions for a given slice of source code.

211− pub fn highlight<'a>(

542+ // pub fn highlight<'a>(

212− &'a mut self,

543+ // &'a mut self,

213− config: &'a HighlightConfiguration,

544+ // config: &'a HighlightConfiguration,

214− source: &'a [u8],

545+ // source: &'a [u8],

215− cancellation_flag: Option<&'a AtomicUsize>,

546+ // cancellation_flag: Option<&'a AtomicUsize>,

216− mut injection_callback: impl FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,

547+ // mut injection_callback: impl FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,

217− ) -> Result<impl Iterator<Item = Result<HighlightEvent, Error>> + 'a, Error> {

548+ // ) -> Result<impl Iterator<Item = Result<HighlightEvent, Error>> + 'a, Error> {

218− let layers = HighlightIterLayer::new(

549+ // let layers = HighlightIterLayer::new(

219− source,

550+ // source,

220− self,

551+ // self,

221− cancellation_flag,

552+ // cancellation_flag,

222− &mut injection_callback,

553+ // &mut injection_callback,

223− config,

554+ // config,

224− 0,

555+ // 0,

225− vec![Range {

556+ // vec![Range {

226− start_byte: 0,

557+ // start_byte: 0,

227− end_byte: usize::MAX,

558+ // end_byte: usize::MAX,

228− start_point: Point::new(0, 0),

559+ // start_point: Point::new(0, 0),

229− end_point: Point::new(usize::MAX, usize::MAX),

560+ // end_point: Point::new(usize::MAX, usize::MAX),

230− }],

561+ // }],

231− )?;

562+ // )?;

232− assert_ne!(layers.len(), 0);

563+ // assert_ne!(layers.len(), 0);

233− let mut result = HighlightIter {

564+ // let mut result = HighlightIter {

234− source,

565+ // source,

235− byte_offset: 0,

566+ // byte_offset: 0,

236− injection_callback,

567+ // injection_callback,

237− cancellation_flag,

568+ // cancellation_flag,

238− highlighter: self,

569+ // highlighter: self,

239− iter_count: 0,

570+ // iter_count: 0,

240− layers,

571+ // layers,

241− next_event: None,

572+ // next_event: None,

242− last_highlight_range: None,

573+ // last_highlight_range: None,

243− };

574+ // };

244− result.sort_layers();

575+ // result.sort_layers();

245− Ok(result)

576+ // Ok(result)

246− }

577+ // }

247}

578}

248

579

249impl HighlightConfiguration {

580impl HighlightConfiguration {

@@ -413,7 +744,7 @@ impl<'a> HighlightIterLayer<'a> {

413 /// added to the returned vector.

744 /// added to the returned vector.

414 fn new<F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a>(

745 fn new<F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a>(

415 source: &'a [u8],

746 source: &'a [u8],

416− highlighter: &mut Highlighter,

747+ highlighter: &mut Syntax,

417 cancellation_flag: Option<&'a AtomicUsize>,

748 cancellation_flag: Option<&'a AtomicUsize>,

418 injection_callback: &mut F,

749 injection_callback: &mut F,

419 mut config: &'a HighlightConfiguration,

750 mut config: &'a HighlightConfiguration,

@@ -423,6 +754,8 @@ impl<'a> HighlightIterLayer<'a> {

423 let mut result = Vec::with_capacity(1);

754 let mut result = Vec::with_capacity(1);

424 let mut queue = Vec::new();

755 let mut queue = Vec::new();

425 loop {

756 loop {

757+ // --> Tree parsing part

758+

426 if highlighter.parser.set_included_ranges(&ranges).is_ok() {

759 if highlighter.parser.set_included_ranges(&ranges).is_ok() {

427 highlighter

760 highlighter

428 .parser

761 .parser

@@ -474,6 +807,8 @@ impl<'a> HighlightIterLayer<'a> {

474 }

807 }

475 }

808 }

476

809

810+ // --> Highlighting query part

811+

477 // The `captures` iterator borrows the `Tree` and the `QueryCursor`, which

812 // The `captures` iterator borrows the `Tree` and the `QueryCursor`, which

478 // prevents them from being moved. But both of these values are really just

813 // prevents them from being moved. But both of these values are really just

479 // pointers, so it's actually ok to move them.

814 // pointers, so it's actually ok to move them.

@@ -495,7 +830,7 @@ impl<'a> HighlightIterLayer<'a> {

495 }],

830 }],

496 cursor,

831 cursor,

497 depth,

832 depth,

498− _tree: tree,

833+ _tree: Some(tree),

499 captures,

834 captures,

500 config,

835 config,

501 ranges,

836 ranges,

@@ -1016,3 +1351,106 @@ fn shrink_and_clear<T>(vec: &mut Vec<T>, capacity: usize) {

1016 }

1351 }

1017 vec.clear();

1352 vec.clear();

1018}

1353}

1354+

1355+#[test]

1356+fn test_parser() {

1357+ let highlight_names: Vec<String> = [

1358+ "attribute",

1359+ "constant",

1360+ "function.builtin",

1361+ "function",

1362+ "keyword",

1363+ "operator",

1364+ "property",

1365+ "punctuation",

1366+ "punctuation.bracket",

1367+ "punctuation.delimiter",

1368+ "string",

1369+ "string.special",

1370+ "tag",

1371+ "type",

1372+ "type.builtin",

1373+ "variable",

1374+ "variable.builtin",

1375+ "variable.parameter",

1376+ ]

1377+ .iter()

1378+ .cloned()

1379+ .map(String::from)

1380+ .collect();

1381+

1382+ let language = get_language(&LANG::Rust);

1383+ let mut config = HighlightConfiguration::new(

1384+ language,

1385+ &std::fs::read_to_string(

1386+ "../helix-syntax/languages/tree-sitter-rust/queries/highlights.scm",

1387+ )

1388+ .unwrap(),

1389+ &std::fs::read_to_string(

1390+ "../helix-syntax/languages/tree-sitter-rust/queries/injections.scm",

1391+ )

1392+ .unwrap(),

1393+ "", // locals.scm

1394+ )

1395+ .unwrap();

1396+ config.configure(&highlight_names);

1397+

1398+ let source = Rope::from_str(

1399+ "

1400+ struct Stuff {}

1401+ fn main() {}

1402+ ",

1403+ );

1404+ let syntax = Syntax::new(LANG::Rust, &source, config);

1405+ let tree = syntax.root_layer.tree.unwrap();

1406+ let root = tree.root_node();

1407+ assert_eq!(root.kind(), "source_file");

1408+

1409+ assert_eq!(

1410+ root.to_sexp(),

1411+ concat!(

1412+ "(source_file ",

1413+ "(struct_item name: (type_identifier) body: (field_declaration_list)) ",

1414+ "(function_item name: (identifier) parameters: (parameters) body: (block)))"

1415+ )

1416+ );

1417+

1418+ let struct_node = root.child(0).unwrap();

1419+ assert_eq!(struct_node.kind(), "struct_item");

1420+}

1421+

1422+#[test]

1423+fn test_input_edits() {

1424+ use crate::State;

1425+ use tree_sitter::InputEdit;

1426+

1427+ let mut state = State::new("hello world!\ntest 123".into());

1428+ let transaction = Transaction::change(

1429+ &state,

1430+ vec![(6, 11, Some("test".into())), (12, 17, None)].into_iter(),

1431+ );

1432+ let edits = LanguageLayer::generate_edits(&state.doc.slice(..), &transaction.changes);

1433+ // transaction.apply(&mut state);

1434+

1435+ assert_eq!(

1436+ edits,

1437+ &[

1438+ InputEdit {

1439+ start_byte: 6,

1440+ old_end_byte: 11,

1441+ new_end_byte: 10,

1442+ start_position: Point { row: 0, column: 6 },

1443+ old_end_position: Point { row: 0, column: 11 },

1444+ new_end_position: Point { row: 0, column: 10 }

1445+ },

1446+ InputEdit {

1447+ start_byte: 12,

1448+ old_end_byte: 17,

1449+ new_end_byte: 12,

1450+ start_position: Point { row: 0, column: 12 },

1451+ old_end_position: Point { row: 1, column: 4 },

1452+ new_end_position: Point { row: 0, column: 12 }

1453+ }

1454+ ]

1455+ );

1456+}

helix-core/src/transaction.rs+30−5

@@ -4,7 +4,7 @@ use crate::{Rope, Selection, SelectionRange, State, Tendril};

4pub type Change = (usize, usize, Option<Tendril>);

6#[derive(Debug, Clone, PartialEq, Eq)]

7−enum Operation {

7+pub(crate) enum Operation {

8 /// Move cursor by n characters.

9 Retain(usize),

10 /// Delete n characters.

@@ -22,7 +22,7 @@ pub enum Assoc {

22// ChangeSpec = Change | ChangeSet | Vec<Change>

23#[derive(Debug)]

24pub struct ChangeSet {

25− changes: Vec<Operation>,

25+ pub(crate) changes: Vec<Operation>,

26 /// The required document length. Will refuse to apply changes unless it matches.

27 len: usize,

28}

@@ -307,7 +307,7 @@ impl ChangeSet {

307/// a single transaction.

308pub struct Transaction {

309 /// Changes made to the buffer.

310− changes: ChangeSet,

310+ pub(crate) changes: ChangeSet,

311 /// When set, explicitly updates the selection.

312 selection: Option<Selection>,

313 // effects, annotations

@@ -337,6 +337,14 @@ impl Transaction {

337 .clone()

338 .unwrap_or_else(|| state.selection.clone().map(&self.changes));

339

340+ // TODO: no unwrap

341+ state

342+ .syntax

343+ .as_mut()

344+ .unwrap()

345+ .update(&state.doc, &self.changes)

346+ .unwrap();

347+

340 true

348 true

341 }

349 }

342

350

@@ -359,9 +367,15 @@ impl Transaction {

359 for (from, to, tendril) in changes {

367 for (from, to, tendril) in changes {

360 // Retain from last "to" to current "from"

368 // Retain from last "to" to current "from"

361 acc.push(Operation::Retain(from - last));

369 acc.push(Operation::Retain(from - last));

370+ let span = to - from;

362 match tendril {

371 match tendril {

372+ Some(text) => {

373+ if span > 0 {

374+ acc.push(Operation::Delete(span));

375+ }

363− Some(text) => acc.push(Operation::Insert(text)),

376+ acc.push(Operation::Insert(text));

377+ }

364− None => acc.push(Operation::Delete(to - from)),

378+ None => acc.push(Operation::Delete(span)),

365 }

379 }

366 last = to;

380 last = to;

367 }

381 }

@@ -466,4 +480,15 @@ mod test {

466 assert_eq!(cs.map_pos(2, Assoc::Before), 2);

480 assert_eq!(cs.map_pos(2, Assoc::Before), 2);

467 assert_eq!(cs.map_pos(2, Assoc::After), 2);

481 assert_eq!(cs.map_pos(2, Assoc::After), 2);

468 }

482 }

483+

484+ #[test]

485+ fn transaction_change() {

486+ let mut state = State::new("hello world!\ntest 123".into());

487+ let transaction = Transaction::change(

488+ &state,

489+ vec![(6, 11, Some("void".into())), (12, 17, None)].into_iter(),

490+ );

491+ transaction.apply(&mut state);

492+ assert_eq!(state.doc, Rope::from_str("hello void! 123"));

493+ }

469}

494}

helix-term/Cargo.toml+1−1

@@ -11,11 +11,11 @@ name = "hx"

11path = "src/main.rs"

13[dependencies]

14+helix-core = { path = "../helix-core" }

14helix-syntax = { path = "../helix-syntax" }

15helix-syntax = { path = "../helix-syntax" }

16anyhow = "1"

17anyhow = "1"

17argh = "0.1.3"

18argh = "0.1.3"

18−helix-core = { path = "../helix-core" }

19crossterm = { version = "0.17", features = ["event-stream"] }

21smol = "1"

helix-term/src/editor.rs+17−38

@@ -38,8 +38,6 @@ pub struct Editor {

38 size: (u16, u16),

39 surface: Surface,

40 theme: Theme,

41− highlighter: Highlighter,

42− highlight_config: HighlightConfiguration,

43}

41}

45impl Editor {

43impl Editor {

@@ -49,33 +47,8 @@ impl Editor {

49 let mut terminal = Terminal::new(backend)?;

47 let mut terminal = Terminal::new(backend)?;

50 let size = terminal::size().unwrap();

48 let size = terminal::size().unwrap();

51 let area = Rect::new(0, 0, size.0, size.1);

49 let area = Rect::new(0, 0, size.0, size.1);

52−

53 let theme = Theme::default();

50 let theme = Theme::default();

55− // let mut parser = tree_sitter::Parser::new();

56− // parser.set_language(language).unwrap();

57− // let tree = parser.parse(source_code, None).unwrap();

58−

59− let language = helix_syntax::get_language(&helix_syntax::LANG::Rust);

60−

61− let highlighter = Highlighter::new();

62−

63− let mut highlight_config = HighlightConfiguration::new(

64− language,

65− &std::fs::read_to_string(

66− "../helix-syntax/languages/tree-sitter-rust/queries/highlights.scm",

67− )

68− .unwrap(),

69− &std::fs::read_to_string(

70− "../helix-syntax/languages/tree-sitter-rust/queries/injections.scm",

71− )

72− .unwrap(),

73− "", // locals.scm

74− )

75− .unwrap();

76−

77− highlight_config.configure(theme.scopes());

78−

79 let mut editor = Editor {

52 let mut editor = Editor {

80 terminal,

53 terminal,

81 state: None,

54 state: None,

@@ -84,8 +57,6 @@ impl Editor {

84 surface: Surface::empty(area),

57 surface: Surface::empty(area),

85 theme,

58 theme,

86 // TODO; move to state

59 // TODO; move to state

87− highlighter,

88− highlight_config,

89 };

60 };

91 if let Some(file) = args.files.pop() {

62 if let Some(file) = args.files.pop() {

@@ -96,12 +67,19 @@ impl Editor {

96 }

67 }

98 pub fn open(&mut self, path: PathBuf) -> Result<(), Error> {

69 pub fn open(&mut self, path: PathBuf) -> Result<(), Error> {

99− self.state = Some(State::load(path)?);

70+ let mut state = State::load(path)?;

71+ state

72+ .syntax

73+ .as_mut()

74+ .unwrap()

75+ .configure(self.theme.scopes());

76+ self.state = Some(state);

100 Ok(())

77 Ok(())

101 }

78 }

102

103 fn render(&mut self) {

80 fn render(&mut self) {

81+ // TODO: ideally not mut but highlights require it because of cursor cache

104− match &self.state {

82+ match &mut self.state {

105 Some(state) => {

83 Some(state) => {

106 let area = Rect::new(0, 0, self.size.0, self.size.1);

84 let area = Rect::new(0, 0, self.size.0, self.size.1);

107 let mut surface = Surface::empty(area);

85 let mut surface = Surface::empty(area);

@@ -112,12 +90,13 @@ impl Editor {

112

113 // TODO: cache highlight results

91 // TODO: cache highlight results

114 // TODO: only recalculate when state.doc is actually modified

92 // TODO: only recalculate when state.doc is actually modified

115− let highlights = self

93+ let highlights: Vec<_> = state

116− .highlighter

94+ .syntax

95+ .as_mut()

96+ .unwrap()

117− .highlight(&self.highlight_config, source_code.as_bytes(), None, |_| {

97+ .highlight_iter(source_code.as_bytes(), None, |_| None)

118− None

98+ .unwrap()

119− })

99+ .collect(); // TODO: we collect here to avoid double borrow, fix later

120− .unwrap();

121

100

122 let mut spans = Vec::new();

101 let mut spans = Vec::new();

123

102

@@ -235,7 +214,7 @@ impl Editor {

235 let coords = coords_at_pos(&state.doc().slice(..), pos);

214 let coords = coords_at_pos(&state.doc().slice(..), pos);

236 execute!(

215 execute!(

237 stdout,

216 stdout,

238− cursor::MoveTo((coords.1 + 2) as u16, coords.0 as u16)

217+ cursor::MoveTo((coords.col + 2) as u16, coords.row as u16)

239 );

218 );

240 }

219 }

241 None => (),

220 None => (),