6728e44 · helix · git.iko.soy

git.iko.soy/helix

on master help sign in

syntax: Split parsing and highlighting

6728e44Blaž Hrastnik5 years ago

Cargo.lock+1

compare with master

unifiedsplit

@@ -381,6 +381,7 @@ dependencies = [

381 "serde",

382 "serde_json",

383 "similar",

384+ "slotmap",

384 "smallvec",

385 "smallvec",

385 "tendril",

386 "tendril",

386 "toml",

387 "toml",

helix-core/Cargo.toml+1

@@ -22,6 +22,7 @@ unicode-segmentation = "1.8"

22unicode-width = "0.1"

23unicode-general-category = "0.4"

24# slab = "0.4.2"

25+slotmap = "1.0"

25tree-sitter = "0.20"

26tree-sitter = "0.20"

26once_cell = "1.9"

27once_cell = "1.9"

27arc-swap = "1"

28arc-swap = "1"

helix-core/src/indent.rs+1−1

@@ -454,7 +454,7 @@ where

454

455 let language_config = loader.language_config_for_scope("source.rust").unwrap();

456 let highlight_config = language_config.highlight_config(&[]).unwrap();

457− let syntax = Syntax::new(&doc, highlight_config.clone());

457+ let syntax = Syntax::new(&doc, highlight_config.clone(), std::sync::Arc::new(loader));

458 let text = doc.slice(..);

459 let tab_width = 4;

460

helix-core/src/syntax.rs+510−616

@@ -9,6 +9,7 @@ use crate::{

9pub use helix_syntax::get_language;

11use arc_swap::ArcSwap;

12+use slotmap::{DefaultKey as LayerId, HopSlotMap};

13use std::{

14use std::{

14 borrow::Cow,

15 borrow::Cow,

@@ -388,9 +389,9 @@ thread_local! {

388

389

389#[derive(Debug)]

390#[derive(Debug)]

390pub struct Syntax {

391pub struct Syntax {

391− config: Arc<HighlightConfiguration>,

392+ layers: HopSlotMap<LayerId, LanguageLayer>,

392−

393+ root: LayerId,

393− root_layer: LanguageLayer,

394+ loader: Arc<Loader>,

394}

395}

395

396

396fn byte_range_to_str(range: std::ops::Range<usize>, source: RopeSlice) -> Cow<str> {

397fn byte_range_to_str(range: std::ops::Range<usize>, source: RopeSlice) -> Cow<str> {

@@ -400,38 +401,36 @@ fn byte_range_to_str(range: std::ops::Range<usize>, source: RopeSlice) -> Cow<st

400}

401}

401

402

402impl Syntax {

403impl Syntax {

403− // buffer, grammar, config, grammars, sync_timeout?

404− pub fn new(

405− /*language: Lang,*/ source: &Rope,

406− config: Arc<HighlightConfiguration>,

404+ pub fn new(source: &Rope, config: Arc<HighlightConfiguration>, loader: Arc<Loader>) -> Self {

407− ) -> Self {

408− let root_layer = LanguageLayer { tree: None };

405+ let root_layer = LanguageLayer {

406+ tree: None,

407+ config,

408+ depth: 0,

409+ ranges: vec![Range {

410+ start_byte: 0,

411+ end_byte: usize::MAX,

412+ start_point: Point::new(0, 0),

413+ end_point: Point::new(usize::MAX, usize::MAX),

414+ }],

415+ };

409

416

410 // track markers of injections

417 // track markers of injections

411 // track scope_descriptor: a Vec of scopes for item in tree

418 // track scope_descriptor: a Vec of scopes for item in tree

412

419

420+ let mut layers = HopSlotMap::default();

421+ let root = layers.insert(root_layer);

422+

413 let mut syntax = Self {

423 let mut syntax = Self {

414 // grammar,

424 // grammar,

415− config,

425+ root,

416− root_layer,

426+ layers,

427+ loader,

417 };

428 };

418

429

419− // update root layer

430+ syntax

420− PARSER.with(|ts_parser| {

431+ .update(source, source, &ChangeSet::new(&source))

421− // TODO: handle the returned `Result` properly.

432+ .unwrap();

422− let _ = syntax.root_layer.parse(

433+

423− &mut ts_parser.borrow_mut(),

424− &syntax.config,

425− source,

426− 0,

427− vec![Range {

428− start_byte: 0,

429− end_byte: usize::MAX,

430− start_point: Point::new(0, 0),

431− end_point: Point::new(usize::MAX, usize::MAX),

432− }],

433− );

434− });

435 syntax

434 syntax

436 }

435 }

437

436

@@ -441,30 +440,197 @@ impl Syntax {

441 source: &Rope,

440 source: &Rope,

442 changeset: &ChangeSet,

441 changeset: &ChangeSet,

443 ) -> Result<(), Error> {

442 ) -> Result<(), Error> {

443+ use std::collections::VecDeque;

444+ let mut queue = VecDeque::new();

445+ // let source = source.slice(..);

446+ let injection_callback = |language: &str| {

447+ self.loader

448+ .language_configuration_for_injection_string(language)

449+ .and_then(|language_config| {

450+ // TODO: get these theme.scopes from somewhere, probably make them settable on Loader

451+ let scopes = &[

452+ "attribute",

453+ "constant",

454+ "function.builtin",

455+ "function",

456+ "keyword",

457+ "operator",

458+ "property",

459+ "punctuation",

460+ "punctuation.bracket",

461+ "punctuation.delimiter",

462+ "string",

463+ "string.special",

464+ "tag",

465+ "type",

466+ "type.builtin",

467+ "variable",

468+ "variable.builtin",

469+ "variable.parameter",

470+ ];

471+ language_config.highlight_config(

472+ &scopes

473+ .iter()

474+ .map(|scope| scope.to_string())

475+ .collect::<Vec<_>>(),

476+ )

477+ })

478+ };

479+

480+ queue.push_back(self.root);

481+

482+ // HAXX: for now, clear all layers except root so they get re-parsed

483+ self.layers.retain(|id, _| id == self.root);

484+

485+ // Workaround for Syntax::new() with empty changeset

486+ if !changeset.is_empty() {

487+ // TODO: do this in a recursive way

488+ // Notify the tree about all the changes

489+ let edits = generate_edits(old_source.slice(..), changeset);

490+ let tree = self.layers[self.root].tree.as_mut().unwrap();

491+ for edit in edits.iter().rev() {

492+ // apply the edits in reverse. If we applied them in order then edit 1 would disrupt

493+ // the positioning of edit 2

494+ tree.edit(edit);

495+ }

496+ }

497+

444 PARSER.with(|ts_parser| {

498 PARSER.with(|ts_parser| {

445− self.root_layer.update(

446− &mut ts_parser.borrow_mut(),

499+ let ts_parser = &mut ts_parser.borrow_mut();

447− &self.config,

500+ let mut cursor = ts_parser.cursors.pop().unwrap_or_else(QueryCursor::new);

448− old_source,

501+ // TODO: might need to set cursor range

502+

503+ while let Some(layer_id) = queue.pop_front() {

504+ // Re-parse the tree.

505+ self.layers[layer_id].parse(ts_parser, source)?;

506+

507+ let source = source.slice(..);

508+ let layer = &self.layers[layer_id];

509+

510+ // Process injections.

511+ let matches = cursor.matches(

512+ &layer.config.injections_query,

513+ layer.tree().root_node(),

449− source,

514+ RopeProvider(source),

450− changeset,

515+ );

516+ let mut injections = Vec::new();

517+ for mat in matches {

518+ let (language_name, content_node, include_children) = injection_for_match(

519+ &layer.config,

520+ &layer.config.injections_query,

521+ &mat,

522+ source,

451− )

523+ );

452− })

524+

525+ // Explicitly remove this match so that none of its other captures will remain

526+ // in the stream of captures.

527+ mat.remove(); // TODO: is this still necessary?

528+

529+ // If a language is found with the given name, then add a new language layer

530+ // to the highlighted document.

531+ if let (Some(language_name), Some(content_node)) = (language_name, content_node)

532+ {

533+ if let Some(config) = (injection_callback)(&language_name) {

534+ let ranges =

535+ intersect_ranges(&layer.ranges, &[content_node], include_children);

536+

537+ if !ranges.is_empty() {

538+ log::info!("{} {:?}", language_name, ranges);

539+ injections.push((config, ranges));

540+ }

541+ }

542+ }

543+ }

453

544

454− // TODO: deal with injections and update them too

545+ // Process combined injections.

546+ if let Some(combined_injections_query) = &layer.config.combined_injections_query {

547+ let mut injections_by_pattern_index =

548+ vec![(None, Vec::new(), false); combined_injections_query.pattern_count()];

549+ let matches = cursor.matches(

550+ combined_injections_query,

551+ layer.tree().root_node(),

552+ RopeProvider(source),

553+ );

554+ for mat in matches {

555+ let entry = &mut injections_by_pattern_index[mat.pattern_index];

556+ let (language_name, content_node, include_children) = injection_for_match(

557+ &layer.config,

558+ combined_injections_query,

559+ &mat,

560+ source,

561+ );

562+ if language_name.is_some() {

563+ entry.0 = language_name;

564+ }

565+ if let Some(content_node) = content_node {

566+ entry.1.push(content_node);

567+ }

568+ entry.2 = include_children;

569+ }

570+ for (lang_name, content_nodes, includes_children) in injections_by_pattern_index

571+ {

572+ if let (Some(lang_name), false) = (lang_name, content_nodes.is_empty()) {

573+ if let Some(config) = (injection_callback)(&lang_name) {

574+ let ranges = intersect_ranges(

575+ &layer.ranges,

576+ &content_nodes,

577+ includes_children,

578+ );

579+ if !ranges.is_empty() {

580+ injections.push((config, ranges));

581+ }

582+ }

583+ }

584+ }

585+ }

586+

587+ let depth = layer.depth + 1;

588+ // TODO: can't inline this since matches borrows self.layers

589+ for (config, ranges) in injections {

590+ let layer_id = self.layers.insert(LanguageLayer {

591+ tree: None,

592+ config,

593+ depth,

594+ ranges,

595+ });

596+ queue.push_back(layer_id);

597+ }

598+ }

599+

600+ // Return the cursor back in the pool.

601+ ts_parser.cursors.push(cursor);

602+

603+ Ok(()) // so we can use the try operator

604+ })?;

605+

606+ Ok(())

455 }

607 }

456

608

457 // fn buffer_changed -> call layer.update(range, new_text) on root layer and then all marker layers

609 // fn buffer_changed -> call layer.update(range, new_text) on root layer and then all marker layers

458−

459 // call this on transaction.apply() -> buffer_changed(changes)

610 // call this on transaction.apply() -> buffer_changed(changes)

460− //

611+

461− // fn parse(language, old_tree, ranges)

462− //

463 pub fn tree(&self) -> &Tree {

612 pub fn tree(&self) -> &Tree {

464− self.root_layer.tree()

613+ self.layers[self.root].tree()

465 }

614 }

615+

616+ // root: Tree

617+ // injections: Vec<(Tree, Range marker)>

618+

619+ // handle updates that go over a part of the layer by truncating them to start/end appropriately

620+

621+ // injections tracked by marker:

622+ // if marker areas match it's fine and update

623+ // if not found add new layer

624+ // if length 0 then area got removed, clean up the layer

466 //

625 //

467− //

626+ // layer update:

627+ // if range.len = 0 then remove the layer

628+ // for change in changes { tree.edit(change) }

629+ // tree = parser.parse(.., tree, ..)

630+ // calculate affected range and update injections

631+ // injection update:

632+ // look for existing injections

633+ // if present, range = (first injection start, last injection end)

468

634

469 // Highlighting

635 // Highlighting

470

636

@@ -474,61 +640,76 @@ impl Syntax {

474 source: RopeSlice<'a>,

640 source: RopeSlice<'a>,

475 range: Option<std::ops::Range<usize>>,

641 range: Option<std::ops::Range<usize>>,

476 cancellation_flag: Option<&'a AtomicUsize>,

642 cancellation_flag: Option<&'a AtomicUsize>,

477− injection_callback: impl FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,

478 ) -> impl Iterator<Item = Result<HighlightEvent, Error>> + 'a {

643 ) -> impl Iterator<Item = Result<HighlightEvent, Error>> + 'a {

644+ let mut layers = self

645+ .layers

646+ .iter()

647+ .map(|(_, layer)| {

648+ // Reuse a cursor from the pool if available.

649+ let mut cursor = PARSER.with(|ts_parser| {

650+ let highlighter = &mut ts_parser.borrow_mut();

651+ highlighter.cursors.pop().unwrap_or_else(QueryCursor::new)

652+ });

653+

479− // The `captures` iterator borrows the `Tree` and the `QueryCursor`, which

654+ // The `captures` iterator borrows the `Tree` and the `QueryCursor`, which

480− // prevents them from being moved. But both of these values are really just

655+ // prevents them from being moved. But both of these values are really just

481− // pointers, so it's actually ok to move them.

656+ // pointers, so it's actually ok to move them.

482−

657+ let cursor_ref =

483− // reuse a cursor from the pool if possible

658+ unsafe { mem::transmute::<_, &'static mut QueryCursor>(&mut cursor) };

484− let mut cursor = PARSER.with(|ts_parser| {

659+

485− let highlighter = &mut ts_parser.borrow_mut();

660+ // if reusing cursors & no range this resets to whole range

486− highlighter.cursors.pop().unwrap_or_else(QueryCursor::new)

661+ // TODO: handle intersect (range & layer.range)

662+ // cursor_ref.set_byte_range(range.clone().unwrap_or(0..usize::MAX));

663+ cursor_ref.set_byte_range(0..usize::MAX);

664+

665+ let captures = cursor_ref

666+ .captures(

667+ &layer.config.query,

668+ layer.tree().root_node(),

669+ RopeProvider(source),

670+ )

671+ .peekable();

672+

673+ HighlightIterLayer {

674+ highlight_end_stack: Vec::new(),

675+ scope_stack: vec![LocalScope {

676+ inherits: false,

677+ range: 0..usize::MAX,

678+ local_defs: Vec::new(),

679+ }],

680+ cursor,

681+ _tree: None,

682+ captures,

683+ config: layer.config.as_ref(), // TODO: just reuse

684+ depth: layer.depth, // TODO: just reuse

685+ ranges: layer.ranges.clone(),

686+ }

687+ })

688+ .collect::<Vec<_>>();

689+

690+ log::info!("--");

691+

692+ // HAXX: arrange layers by byte range, with deeper layers positioned first

693+ layers.sort_by_key(|layer| {

694+ (

695+ layer.ranges.first().cloned(),

696+ std::cmp::Reverse(layer.depth),

697+ )

487 });

698 });

488− let tree_ref = self.tree();

489− let cursor_ref = unsafe { mem::transmute::<_, &'static mut QueryCursor>(&mut cursor) };

490− let query_ref = &self.config.query;

491− let config_ref = self.config.as_ref();

492−

493− // if reusing cursors & no range this resets to whole range

494− cursor_ref.set_byte_range(range.clone().unwrap_or(0..usize::MAX));

495−

496− let captures = cursor_ref

497− .captures(query_ref, tree_ref.root_node(), RopeProvider(source))

498− .peekable();

499−

500− // manually craft the root layer based on the existing tree

501− let layer = HighlightIterLayer {

502− highlight_end_stack: Vec::new(),

503− scope_stack: vec![LocalScope {

504− inherits: false,

505− range: 0..usize::MAX,

506− local_defs: Vec::new(),

507− }],

508− cursor,

509− depth: 0,

510− _tree: None,

511− captures,

512− config: config_ref,

513− ranges: vec![Range {

514− start_byte: 0,

515− end_byte: usize::MAX,

516− start_point: Point::new(0, 0),

517− end_point: Point::new(usize::MAX, usize::MAX),

518− }],

519− };

520

699

521 let mut result = HighlightIter {

700 let mut result = HighlightIter {

522 source,

701 source,

523 byte_offset: range.map_or(0, |r| r.start), // TODO: simplify

702 byte_offset: range.map_or(0, |r| r.start), // TODO: simplify

524− injection_callback,

525 cancellation_flag,

703 cancellation_flag,

526 iter_count: 0,

704 iter_count: 0,

527− layers: vec![layer],

705+ layers,

528 next_event: None,

706 next_event: None,

529 last_highlight_range: None,

707 last_highlight_range: None,

530 };

708 };

531 result.sort_layers();

709 result.sort_layers();

710+ for layer in &result.layers {

711+ log::info!("> {:?} {:?}", layer.depth, layer.ranges); // <- for some reason layers are reversed here

712+ }

532 result

713 result

533 }

714 }

534 // on_tokenize

715 // on_tokenize

@@ -556,234 +737,155 @@ impl Syntax {

556pub struct LanguageLayer {

737pub struct LanguageLayer {

557 // mode

738 // mode

558 // grammar

739 // grammar

559− // depth

740+ pub config: Arc<HighlightConfiguration>,

560 pub(crate) tree: Option<Tree>,

741 pub(crate) tree: Option<Tree>,

742+ pub ranges: Vec<Range>,

743+ pub depth: usize,

561}

744}

562

745

563impl LanguageLayer {

746impl LanguageLayer {

564− // pub fn new() -> Self {

565− // Self { tree: None }

566− // }

567−

568 pub fn tree(&self) -> &Tree {

747 pub fn tree(&self) -> &Tree {

569 // TODO: no unwrap

748 // TODO: no unwrap

570 self.tree.as_ref().unwrap()

749 self.tree.as_ref().unwrap()

571 }

750 }

572

751

573− fn parse(

574− &mut self,

575− ts_parser: &mut TsParser,

752+ fn parse(&mut self, ts_parser: &mut TsParser, source: &Rope) -> Result<(), Error> {

576− config: &HighlightConfiguration,

577− source: &Rope,

578− _depth: usize,

579− ranges: Vec<Range>,

580− ) -> Result<(), Error> {

581− if ts_parser.parser.set_included_ranges(&ranges).is_ok() {

753+ ts_parser.parser.set_included_ranges(&self.ranges).unwrap();

754+

582− ts_parser

755+ ts_parser

583− .parser

756+ .parser

584− .set_language(config.language)

757+ .set_language(self.config.language)

585− .map_err(|_| Error::InvalidLanguage)?;

758+ .map_err(|_| Error::InvalidLanguage)?;

586−

759+

587− // unsafe { syntax.parser.set_cancellation_flag(cancellation_flag) };

760+ // unsafe { syntax.parser.set_cancellation_flag(cancellation_flag) };

588− let tree = ts_parser

761+ let tree = ts_parser

589− .parser

762+ .parser

590− .parse_with(

763+ .parse_with(

591− &mut |byte, _| {

764+ &mut |byte, _| {

592− if byte <= source.len_bytes() {

765+ if byte <= source.len_bytes() {

593− let (chunk, start_byte, _, _) = source.chunk_at_byte(byte);

766+ let (chunk, start_byte, _, _) = source.chunk_at_byte(byte);

594− chunk[byte - start_byte..].as_bytes()

767+ chunk[byte - start_byte..].as_bytes()

595− } else {

768+ } else {

596− // out of range

769+ // out of range

597− &[]

770+ &[]

598− }

771+ }

599− },

772+ },

600− self.tree.as_ref(),

773+ self.tree.as_ref(),

601− )

774+ )

602− .ok_or(Error::Cancelled)?;

775+ .ok_or(Error::Cancelled)?;

603−

776+ // unsafe { ts_parser.parser.set_cancellation_flag(None) };

604− self.tree = Some(tree)

777+ self.tree = Some(tree);

605− }

606 Ok(())

778 Ok(())

607 }

779 }

780+}

608

781

609− pub(crate) fn generate_edits(

782+pub(crate) fn generate_edits(

610− old_text: RopeSlice,

783+ old_text: RopeSlice,

611− changeset: &ChangeSet,

784+ changeset: &ChangeSet,

612− ) -> Vec<tree_sitter::InputEdit> {

785+) -> Vec<tree_sitter::InputEdit> {

613− use Operation::*;

786+ use Operation::*;

614− let mut old_pos = 0;

787+ let mut old_pos = 0;

615

788

616− let mut edits = Vec::new();

789+ let mut edits = Vec::new();

617

790

618− let mut iter = changeset.changes.iter().peekable();

791+ let mut iter = changeset.changes.iter().peekable();

619

792

620− // TODO; this is a lot easier with Change instead of Operation.

793+ // TODO; this is a lot easier with Change instead of Operation.

621

794

622− fn point_at_pos(text: RopeSlice, pos: usize) -> (usize, Point) {

795+ fn point_at_pos(text: RopeSlice, pos: usize) -> (usize, Point) {

623− let byte = text.char_to_byte(pos); // <- attempted to index past end

796+ let byte = text.char_to_byte(pos); // <- attempted to index past end

624− let line = text.char_to_line(pos);

797+ let line = text.char_to_line(pos);

625− let line_start_byte = text.line_to_byte(line);

798+ let line_start_byte = text.line_to_byte(line);

626− let col = byte - line_start_byte;

799+ let col = byte - line_start_byte;

627

800

628− (byte, Point::new(line, col))

801+ (byte, Point::new(line, col))

629− }

802+ }

630

803

631− fn traverse(point: Point, text: &Tendril) -> Point {

804+ fn traverse(point: Point, text: &Tendril) -> Point {

632− let Point {

805+ let Point {

633− mut row,

806+ mut row,

634− mut column,

807+ mut column,

635− } = point;

808+ } = point;

636−

809+

637− // TODO: there should be a better way here.

810+ // TODO: there should be a better way here.

638− let mut chars = text.chars().peekable();

811+ let mut chars = text.chars().peekable();

639− while let Some(ch) = chars.next() {

812+ while let Some(ch) = chars.next() {

640− if char_is_line_ending(ch) && !(ch == '\r' && chars.peek() == Some(&'\n')) {

813+ if char_is_line_ending(ch) && !(ch == '\r' && chars.peek() == Some(&'\n')) {

641− row += 1;

814+ row += 1;

642− column = 0;

815+ column = 0;

643− } else {

816+ } else {

644− column += 1;

817+ column += 1;

645− }

646 }

818 }

647− Point { row, column }

648 }

819 }

820+ Point { row, column }

821+ }

649

822

650− while let Some(change) = iter.next() {

823+ while let Some(change) = iter.next() {

651− let len = match change {

824+ let len = match change {

652− Delete(i) | Retain(i) => *i,

825+ Delete(i) | Retain(i) => *i,

653− Insert(_) => 0,

826+ Insert(_) => 0,

654− };

827+ };

655− let mut old_end = old_pos + len;

828+ let mut old_end = old_pos + len;

829+

830+ match change {

831+ Retain(_) => {}

832+ Delete(_) => {

833+ let (start_byte, start_position) = point_at_pos(old_text, old_pos);

834+ let (old_end_byte, old_end_position) = point_at_pos(old_text, old_end);

835+

836+ // TODO: Position also needs to be byte based...

837+ // let byte = char_to_byte(old_pos)

838+ // let line = char_to_line(old_pos)

839+ // let line_start_byte = line_to_byte()

840+ // Position::new(line, line_start_byte - byte)

841+

842+ // deletion

843+ edits.push(tree_sitter::InputEdit {

844+ start_byte, // old_pos to byte

845+ old_end_byte, // old_end to byte

846+ new_end_byte: start_byte, // old_pos to byte

847+ start_position, // old pos to coords

848+ old_end_position, // old_end to coords

849+ new_end_position: start_position, // old pos to coords

850+ });

851+ }

852+ Insert(s) => {

853+ let (start_byte, start_position) = point_at_pos(old_text, old_pos);

656

854

657− match change {

855+ // a subsequent delete means a replace, consume it

658− Retain(_) => {}

856+ if let Some(Delete(len)) = iter.peek() {

659− Delete(_) => {

857+ old_end = old_pos + len;

660− let (start_byte, start_position) = point_at_pos(old_text, old_pos);

661 let (old_end_byte, old_end_position) = point_at_pos(old_text, old_end);

858 let (old_end_byte, old_end_position) = point_at_pos(old_text, old_end);

662

859

663− // TODO: Position also needs to be byte based...

860+ iter.next();

664− // let byte = char_to_byte(old_pos)

665− // let line = char_to_line(old_pos)

666− // let line_start_byte = line_to_byte()

667− // Position::new(line, line_start_byte - byte)

668

861

669− // deletion

862+ // replacement

670 edits.push(tree_sitter::InputEdit {

863 edits.push(tree_sitter::InputEdit {

671− start_byte, // old_pos to byte

864+ start_byte, // old_pos to byte

672− old_end_byte, // old_end to byte

865+ old_end_byte, // old_end to byte

673− new_end_byte: start_byte, // old_pos to byte

866+ new_end_byte: start_byte + s.len(), // old_pos to byte + s.len()

674− start_position, // old pos to coords

867+ start_position, // old pos to coords

675− old_end_position, // old_end to coords

868+ old_end_position, // old_end to coords

676− new_end_position: start_position, // old pos to coords

869+ new_end_position: traverse(start_position, s), // old pos + chars, newlines matter too (iter over)

870+ });

871+ } else {

872+ // insert

873+ edits.push(tree_sitter::InputEdit {

874+ start_byte, // old_pos to byte

875+ old_end_byte: start_byte, // same

876+ new_end_byte: start_byte + s.len(), // old_pos + s.len()

877+ start_position, // old pos to coords

878+ old_end_position: start_position, // same

879+ new_end_position: traverse(start_position, s), // old pos + chars, newlines matter too (iter over)

677 });

880 });

678− }

679− Insert(s) => {

680− let (start_byte, start_position) = point_at_pos(old_text, old_pos);

681−

682− // a subsequent delete means a replace, consume it

683− if let Some(Delete(len)) = iter.peek() {

684− old_end = old_pos + len;

685− let (old_end_byte, old_end_position) = point_at_pos(old_text, old_end);

686−

687− iter.next();

688−

689− // replacement

690− edits.push(tree_sitter::InputEdit {

691− start_byte, // old_pos to byte

692− old_end_byte, // old_end to byte

693− new_end_byte: start_byte + s.len(), // old_pos to byte + s.len()

694− start_position, // old pos to coords

695− old_end_position, // old_end to coords

696− new_end_position: traverse(start_position, s), // old pos + chars, newlines matter too (iter over)

697− });

698− } else {

699− // insert

700− edits.push(tree_sitter::InputEdit {

701− start_byte, // old_pos to byte

702− old_end_byte: start_byte, // same

703− new_end_byte: start_byte + s.len(), // old_pos + s.len()

704− start_position, // old pos to coords

705− old_end_position: start_position, // same

706− new_end_position: traverse(start_position, s), // old pos + chars, newlines matter too (iter over)

707− });

708− }

709 }

881 }

710 }

882 }

711− old_pos = old_end;

712− }

713− edits

714− }

715−

716− fn update(

717− &mut self,

718− ts_parser: &mut TsParser,

719− config: &HighlightConfiguration,

720− old_source: &Rope,

721− source: &Rope,

722− changeset: &ChangeSet,

723− ) -> Result<(), Error> {

724− if changeset.is_empty() {

725− return Ok(());

726 }

883 }

727−

884+ old_pos = old_end;

728− let edits = Self::generate_edits(old_source.slice(..), changeset);

729−

730− // Notify the tree about all the changes

731− for edit in edits.iter().rev() {

732− // apply the edits in reverse. If we applied them in order then edit 1 would disrupt

733− // the positioning of edit 2

734− self.tree.as_mut().unwrap().edit(edit);

735− }

736−

737− self.parse(

738− ts_parser,

739− config,

740− source,

741− 0,

742− // TODO: what to do about this range on update

743− vec![Range {

744− start_byte: 0,

745− end_byte: usize::MAX,

746− start_point: Point::new(0, 0),

747− end_point: Point::new(usize::MAX, usize::MAX),

748− }],

749− )

750 }

885 }

751−

886+ edits

752− // fn highlight_iter() -> same as Mode but for this layer. Mode composits these

753− // fn buffer_changed

754− // fn update(range)

755− // fn update_injections()

756}

887}

757

888

758−// -- refactored from tree-sitter-highlight to be able to retain state

759−// TODO: add seek() to iter

760−

761−// problem: any time a layer is updated it must update it's injections on the parent (potentially

762−// removing some from use)

763−// can't modify to vec and exist in it at the same time since that would violate borrows

764−// maybe we can do with an arena

765−// maybe just caching on the top layer and nevermind the injections for now?

766−//

767−// Grammar {

768−// layers: Vec<Box<Layer>> to prevent memory moves when vec is modified

769−// }

770−// injections tracked by marker:

771−// if marker areas match it's fine and update

772−// if not found add new layer

773−// if length 0 then area got removed, clean up the layer

774−//

775−// layer update:

776−// if range.len = 0 then remove the layer

777−// for change in changes { tree.edit(change) }

778−// tree = parser.parse(.., tree, ..)

779−// calculate affected range and update injections

780−// injection update:

781−// look for existing injections

782−// if present, range = (first injection start, last injection end)

783−//

784−// For now cheat and just throw out non-root layers if they exist. This should still improve

785−// parsing in majority of cases.

786−

787use std::sync::atomic::{AtomicUsize, Ordering};

889use std::sync::atomic::{AtomicUsize, Ordering};

788use std::{iter, mem, ops, str, usize};

890use std::{iter, mem, ops, str, usize};

789use tree_sitter::{

891use tree_sitter::{

@@ -820,8 +922,8 @@ pub enum HighlightEvent {

820pub struct HighlightConfiguration {

922pub struct HighlightConfiguration {

821 pub language: Grammar,

923 pub language: Grammar,

822 pub query: Query,

924 pub query: Query,

925+ injections_query: Query,

823 combined_injections_query: Option<Query>,

926 combined_injections_query: Option<Query>,

824− locals_pattern_index: usize,

825 highlights_pattern_index: usize,

927 highlights_pattern_index: usize,

826 highlight_indices: ArcSwap<Vec<Option<Highlight>>>,

928 highlight_indices: ArcSwap<Vec<Option<Highlight>>>,

827 non_local_variable_patterns: Vec<bool>,

929 non_local_variable_patterns: Vec<bool>,

@@ -848,13 +950,9 @@ struct LocalScope<'a> {

848}

950}

849

951

850#[derive(Debug)]

952#[derive(Debug)]

851−struct HighlightIter<'a, F>

953+struct HighlightIter<'a> {

852−where

853− F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,

854−{

855 source: RopeSlice<'a>,

954 source: RopeSlice<'a>,

856 byte_offset: usize,

955 byte_offset: usize,

857− injection_callback: F,

858 cancellation_flag: Option<&'a AtomicUsize>,

956 cancellation_flag: Option<&'a AtomicUsize>,

859 layers: Vec<HighlightIterLayer<'a>>,

957 layers: Vec<HighlightIterLayer<'a>>,

860 iter_count: usize,

958 iter_count: usize,

@@ -894,8 +992,8 @@ struct HighlightIterLayer<'a> {

894 config: &'a HighlightConfiguration,

992 config: &'a HighlightConfiguration,

895 highlight_end_stack: Vec<usize>,

993 highlight_end_stack: Vec<usize>,

896 scope_stack: Vec<LocalScope<'a>>,

994 scope_stack: Vec<LocalScope<'a>>,

897− ranges: Vec<Range>,

898 depth: usize,

995 depth: usize,

996+ ranges: Vec<Range>, // TEMP

899}

997}

900

998

901impl<'a> fmt::Debug for HighlightIterLayer<'a> {

999impl<'a> fmt::Debug for HighlightIterLayer<'a> {

@@ -927,38 +1025,32 @@ impl HighlightConfiguration {

927 ) -> Result<Self, QueryError> {

1025 ) -> Result<Self, QueryError> {

928 // Concatenate the query strings, keeping track of the start offset of each section.

1026 // Concatenate the query strings, keeping track of the start offset of each section.

929 let mut query_source = String::new();

1027 let mut query_source = String::new();

930− query_source.push_str(injection_query);

931− let locals_query_offset = query_source.len();

932 query_source.push_str(locals_query);

1028 query_source.push_str(locals_query);

933 let highlights_query_offset = query_source.len();

1029 let highlights_query_offset = query_source.len();

934 query_source.push_str(highlights_query);

1030 query_source.push_str(highlights_query);

935

1031

936 // Construct a single query by concatenating the three query strings, but record the

1032 // Construct a single query by concatenating the three query strings, but record the

937 // range of pattern indices that belong to each individual string.

1033 // range of pattern indices that belong to each individual string.

938− let mut query = Query::new(language, &query_source)?;

1034+ let query = Query::new(language, &query_source)?;

939− let mut locals_pattern_index = 0;

940 let mut highlights_pattern_index = 0;

1035 let mut highlights_pattern_index = 0;

941 for i in 0..(query.pattern_count()) {

1036 for i in 0..(query.pattern_count()) {

942 let pattern_offset = query.start_byte_for_pattern(i);

1037 let pattern_offset = query.start_byte_for_pattern(i);

943 if pattern_offset < highlights_query_offset {

1038 if pattern_offset < highlights_query_offset {

944− if pattern_offset < highlights_query_offset {

945− highlights_pattern_index += 1;

1039+ highlights_pattern_index += 1;

946− }

947− if pattern_offset < locals_query_offset {

948− locals_pattern_index += 1;

949− }

950 }

1040 }

951 }

1041 }

952

1042

1043+ let mut injections_query = Query::new(language, injection_query)?;

1044+

953 // Construct a separate query just for dealing with the 'combined injections'.

1045 // Construct a separate query just for dealing with the 'combined injections'.

954 // Disable the combined injection patterns in the main query.

1046 // Disable the combined injection patterns in the main query.

955 let mut combined_injections_query = Query::new(language, injection_query)?;

1047 let mut combined_injections_query = Query::new(language, injection_query)?;

956 let mut has_combined_queries = false;

1048 let mut has_combined_queries = false;

957− for pattern_index in 0..locals_pattern_index {

1049+ for pattern_index in 0..injections_query.pattern_count() {

958− let settings = query.property_settings(pattern_index);

1050+ let settings = injections_query.property_settings(pattern_index);

959 if settings.iter().any(|s| &*s.key == "injection.combined") {

1051 if settings.iter().any(|s| &*s.key == "injection.combined") {

960 has_combined_queries = true;

1052 has_combined_queries = true;

961− query.disable_pattern(pattern_index);

1053+ injections_query.disable_pattern(pattern_index);

962 } else {

1054 } else {

963 combined_injections_query.disable_pattern(pattern_index);

1055 combined_injections_query.disable_pattern(pattern_index);

964 }

1056 }

@@ -990,8 +1082,6 @@ impl HighlightConfiguration {

990 for (i, name) in query.capture_names().iter().enumerate() {

1082 for (i, name) in query.capture_names().iter().enumerate() {

991 let i = Some(i as u32);

1083 let i = Some(i as u32);

992 match name.as_str() {

1084 match name.as_str() {

993− "injection.content" => injection_content_capture_index = i,

994− "injection.language" => injection_language_capture_index = i,

995 "local.definition" => local_def_capture_index = i,

1085 "local.definition" => local_def_capture_index = i,

996 "local.definition-value" => local_def_value_capture_index = i,

1086 "local.definition-value" => local_def_value_capture_index = i,

997 "local.reference" => local_ref_capture_index = i,

1087 "local.reference" => local_ref_capture_index = i,

@@ -1000,12 +1090,21 @@ impl HighlightConfiguration {

1000 }

1090 }

1001 }

1091 }

1002

1092

1093+ for (i, name) in injections_query.capture_names().iter().enumerate() {

1094+ let i = Some(i as u32);

1095+ match name.as_str() {

1096+ "injection.content" => injection_content_capture_index = i,

1097+ "injection.language" => injection_language_capture_index = i,

1098+ _ => {}

1099+ }

1100+ }

1101+

1003 let highlight_indices = ArcSwap::from_pointee(vec![None; query.capture_names().len()]);

1102 let highlight_indices = ArcSwap::from_pointee(vec![None; query.capture_names().len()]);

1004 Ok(Self {

1103 Ok(Self {

1005 language,

1104 language,

1006 query,

1105 query,

1106+ injections_query,

1007 combined_injections_query,

1107 combined_injections_query,

1008− locals_pattern_index,

1009 highlights_pattern_index,

1108 highlights_pattern_index,

1010 highlight_indices,

1109 highlight_indices,

1011 non_local_variable_patterns,

1110 non_local_variable_patterns,

@@ -1070,238 +1169,6 @@ impl HighlightConfiguration {

1070}

1169}

1071

1170

1072impl<'a> HighlightIterLayer<'a> {

1171impl<'a> HighlightIterLayer<'a> {

1073− /// Create a new 'layer' of highlighting for this document.

1074− ///

1075− /// In the even that the new layer contains "combined injections" (injections where multiple

1076− /// disjoint ranges are parsed as one syntax tree), these will be eagerly processed and

1077− /// added to the returned vector.

1078− fn new<F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a>(

1079− source: RopeSlice<'a>,

1080− cancellation_flag: Option<&'a AtomicUsize>,

1081− injection_callback: &mut F,

1082− mut config: &'a HighlightConfiguration,

1083− mut depth: usize,

1084− mut ranges: Vec<Range>,

1085− ) -> Result<Vec<Self>, Error> {

1086− let mut result = Vec::with_capacity(1);

1087− let mut queue = Vec::new();

1088− loop {

1089− // --> Tree parsing part

1090−

1091− PARSER.with(|ts_parser| {

1092− let highlighter = &mut ts_parser.borrow_mut();

1093−

1094− if highlighter.parser.set_included_ranges(&ranges).is_ok() {

1095− highlighter

1096− .parser

1097− .set_language(config.language)

1098− .map_err(|_| Error::InvalidLanguage)?;

1099−

1100− unsafe { highlighter.parser.set_cancellation_flag(cancellation_flag) };

1101− let tree = highlighter

1102− .parser

1103− .parse_with(

1104− &mut |byte, _| {

1105− if byte <= source.len_bytes() {

1106− let (chunk, start_byte, _, _) = source.chunk_at_byte(byte);

1107− chunk[byte - start_byte..].as_bytes()

1108− } else {

1109− // out of range

1110− &[]

1111− }

1112− },

1113− None,

1114− )

1115− .ok_or(Error::Cancelled)?;

1116− unsafe { highlighter.parser.set_cancellation_flag(None) };

1117− let mut cursor = highlighter.cursors.pop().unwrap_or_else(QueryCursor::new);

1118−

1119− // Process combined injections.

1120− if let Some(combined_injections_query) = &config.combined_injections_query {

1121− let mut injections_by_pattern_index = vec![

1122− (None, Vec::new(), false);

1123− combined_injections_query

1124− .pattern_count()

1125− ];

1126− let matches = cursor.matches(

1127− combined_injections_query,

1128− tree.root_node(),

1129− RopeProvider(source),

1130− );

1131− for mat in matches {

1132− let entry = &mut injections_by_pattern_index[mat.pattern_index];

1133− let (language_name, content_node, include_children) =

1134− injection_for_match(

1135− config,

1136− combined_injections_query,

1137− &mat,

1138− source,

1139− );

1140− if language_name.is_some() {

1141− entry.0 = language_name;

1142− }

1143− if let Some(content_node) = content_node {

1144− entry.1.push(content_node);

1145− }

1146− entry.2 = include_children;

1147− }

1148− for (lang_name, content_nodes, includes_children) in

1149− injections_by_pattern_index

1150− {

1151− if let (Some(lang_name), false) = (lang_name, content_nodes.is_empty())

1152− {

1153− if let Some(next_config) = (injection_callback)(&lang_name) {

1154− let ranges = Self::intersect_ranges(

1155− &ranges,

1156− &content_nodes,

1157− includes_children,

1158− );

1159− if !ranges.is_empty() {

1160− queue.push((next_config, depth + 1, ranges));

1161− }

1162− }

1163− }

1164− }

1165− }

1166−

1167− // --> Highlighting query part

1168−

1169− // The `captures` iterator borrows the `Tree` and the `QueryCursor`, which

1170− // prevents them from being moved. But both of these values are really just

1171− // pointers, so it's actually ok to move them.

1172− let tree_ref = unsafe { mem::transmute::<_, &'static Tree>(&tree) };

1173− let cursor_ref =

1174− unsafe { mem::transmute::<_, &'static mut QueryCursor>(&mut cursor) };

1175− let captures = cursor_ref

1176− .captures(&config.query, tree_ref.root_node(), RopeProvider(source))

1177− .peekable();

1178−

1179− result.push(HighlightIterLayer {

1180− highlight_end_stack: Vec::new(),

1181− scope_stack: vec![LocalScope {

1182− inherits: false,

1183− range: 0..usize::MAX,

1184− local_defs: Vec::new(),

1185− }],

1186− cursor,

1187− depth,

1188− _tree: Some(tree),

1189− captures,

1190− config,

1191− ranges,

1192− });

1193− }

1194−

1195− Ok(()) // so we can use the try operator

1196− })?;

1197−

1198− if queue.is_empty() {

1199− break;

1200− }

1201−

1202− let (next_config, next_depth, next_ranges) = queue.remove(0);

1203− config = next_config;

1204− depth = next_depth;

1205− ranges = next_ranges;

1206− }

1207−

1208− Ok(result)

1209− }

1210−

1211− // Compute the ranges that should be included when parsing an injection.

1212− // This takes into account three things:

1213− // * `parent_ranges` - The ranges must all fall within the *current* layer's ranges.

1214− // * `nodes` - Every injection takes place within a set of nodes. The injection ranges

1215− // are the ranges of those nodes.

1216− // * `includes_children` - For some injections, the content nodes' children should be

1217− // excluded from the nested document, so that only the content nodes' *own* content

1218− // is reparsed. For other injections, the content nodes' entire ranges should be

1219− // reparsed, including the ranges of their children.

1220− fn intersect_ranges(

1221− parent_ranges: &[Range],

1222− nodes: &[Node],

1223− includes_children: bool,

1224− ) -> Vec<Range> {

1225− let mut cursor = nodes[0].walk();

1226− let mut result = Vec::new();

1227− let mut parent_range_iter = parent_ranges.iter();

1228− let mut parent_range = parent_range_iter

1229− .next()

1230− .expect("Layers should only be constructed with non-empty ranges vectors");

1231− for node in nodes.iter() {

1232− let mut preceding_range = Range {

1233− start_byte: 0,

1234− start_point: Point::new(0, 0),

1235− end_byte: node.start_byte(),

1236− end_point: node.start_position(),

1237− };

1238− let following_range = Range {

1239− start_byte: node.end_byte(),

1240− start_point: node.end_position(),

1241− end_byte: usize::MAX,

1242− end_point: Point::new(usize::MAX, usize::MAX),

1243− };

1244−

1245− for excluded_range in node

1246− .children(&mut cursor)

1247− .filter_map(|child| {

1248− if includes_children {

1249− None

1250− } else {

1251− Some(child.range())

1252− }

1253− })

1254− .chain([following_range].iter().cloned())

1255− {

1256− let mut range = Range {

1257− start_byte: preceding_range.end_byte,

1258− start_point: preceding_range.end_point,

1259− end_byte: excluded_range.start_byte,

1260− end_point: excluded_range.start_point,

1261− };

1262− preceding_range = excluded_range;

1263−

1264− if range.end_byte < parent_range.start_byte {

1265− continue;

1266− }

1267−

1268− while parent_range.start_byte <= range.end_byte {

1269− if parent_range.end_byte > range.start_byte {

1270− if range.start_byte < parent_range.start_byte {

1271− range.start_byte = parent_range.start_byte;

1272− range.start_point = parent_range.start_point;

1273− }

1274−

1275− if parent_range.end_byte < range.end_byte {

1276− if range.start_byte < parent_range.end_byte {

1277− result.push(Range {

1278− start_byte: range.start_byte,

1279− start_point: range.start_point,

1280− end_byte: parent_range.end_byte,

1281− end_point: parent_range.end_point,

1282− });

1283− }

1284− range.start_byte = parent_range.end_byte;

1285− range.start_point = parent_range.end_point;

1286− } else {

1287− if range.start_byte < range.end_byte {

1288− result.push(range);

1289− }

1290− break;

1291− }

1292− }

1293−

1294− if let Some(next_range) = parent_range_iter.next() {

1295− parent_range = next_range;

1296− } else {

1297− return result;

1298− }

1299− }

1300− }

1301− }

1302− result

1303− }

1304−

1305 // First, sort scope boundaries by their byte offset in the document. At a

1172 // First, sort scope boundaries by their byte offset in the document. At a

1306 // given position, emit scope endings before scope beginnings. Finally, emit

1173 // given position, emit scope endings before scope beginnings. Finally, emit

1307 // scope boundaries from deeper layers first.

1174 // scope boundaries from deeper layers first.

@@ -1327,10 +1194,101 @@ impl<'a> HighlightIterLayer<'a> {

1327 }

1194 }

1328}

1195}

1329

1196

1197+// Compute the ranges that should be included when parsing an injection.

1198+// This takes into account three things:

1199+// * `parent_ranges` - The ranges must all fall within the *current* layer's ranges.

1200+// * `nodes` - Every injection takes place within a set of nodes. The injection ranges

1201+// are the ranges of those nodes.

1202+// * `includes_children` - For some injections, the content nodes' children should be

1203+// excluded from the nested document, so that only the content nodes' *own* content

1204+// is reparsed. For other injections, the content nodes' entire ranges should be

1205+// reparsed, including the ranges of their children.

1206+fn intersect_ranges(

1207+ parent_ranges: &[Range],

1208+ nodes: &[Node],

1209+ includes_children: bool,

1210+) -> Vec<Range> {

1211+ let mut cursor = nodes[0].walk();

1212+ let mut result = Vec::new();

1213+ let mut parent_range_iter = parent_ranges.iter();

1214+ let mut parent_range = parent_range_iter

1215+ .next()

1216+ .expect("Layers should only be constructed with non-empty ranges vectors");

1217+ for node in nodes.iter() {

1218+ let mut preceding_range = Range {

1219+ start_byte: 0,

1220+ start_point: Point::new(0, 0),

1221+ end_byte: node.start_byte(),

1222+ end_point: node.start_position(),

1223+ };

1224+ let following_range = Range {

1225+ start_byte: node.end_byte(),

1226+ start_point: node.end_position(),

1227+ end_byte: usize::MAX,

1228+ end_point: Point::new(usize::MAX, usize::MAX),

1229+ };

1230+

1231+ for excluded_range in node

1232+ .children(&mut cursor)

1233+ .filter_map(|child| {

1234+ if includes_children {

1235+ None

1236+ } else {

1237+ Some(child.range())

1238+ }

1239+ })

1240+ .chain([following_range].iter().cloned())

1241+ {

1242+ let mut range = Range {

1243+ start_byte: preceding_range.end_byte,

1244+ start_point: preceding_range.end_point,

1245+ end_byte: excluded_range.start_byte,

1246+ end_point: excluded_range.start_point,

1247+ };

1248+ preceding_range = excluded_range;

1249+

1250+ if range.end_byte < parent_range.start_byte {

1251+ continue;

1252+ }

1253+

1254+ while parent_range.start_byte <= range.end_byte {

1255+ if parent_range.end_byte > range.start_byte {

1256+ if range.start_byte < parent_range.start_byte {

1257+ range.start_byte = parent_range.start_byte;

1258+ range.start_point = parent_range.start_point;

1259+ }

1260+

1261+ if parent_range.end_byte < range.end_byte {

1262+ if range.start_byte < parent_range.end_byte {

1263+ result.push(Range {

1264+ start_byte: range.start_byte,

1265+ start_point: range.start_point,

1266+ end_byte: parent_range.end_byte,

1267+ end_point: parent_range.end_point,

1268+ });

1269+ }

1270+ range.start_byte = parent_range.end_byte;

1271+ range.start_point = parent_range.end_point;

1272+ } else {

1273+ if range.start_byte < range.end_byte {

1274+ result.push(range);

1275+ }

1276+ break;

1277+ }

1278+ }

1279+

1280+ if let Some(next_range) = parent_range_iter.next() {

1281+ parent_range = next_range;

1282+ } else {

1283+ return result;

1284+ }

1285+ }

1286+ }

1287+ }

1288+ result

1289+}

1290+

1330−impl<'a, F> HighlightIter<'a, F>

1291+impl<'a> HighlightIter<'a> {

1331−where

1332− F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,

1333−{

1334 fn emit_event(

1292 fn emit_event(

1335 &mut self,

1293 &mut self,

1336 offset: usize,

1294 offset: usize,

@@ -1361,6 +1319,12 @@ where

1361 i += 1;

1319 i += 1;

1362 continue;

1320 continue;

1363 }

1321 }

1322+ } else {

1323+ let layer = self.layers.remove(i + 1);

1324+ PARSER.with(|ts_parser| {

1325+ let highlighter = &mut ts_parser.borrow_mut();

1326+ highlighter.cursors.push(layer.cursor);

1327+ });

1364 }

1328 }

1365 break;

1329 break;

1366 }

1330 }

@@ -1377,30 +1341,9 @@ where

1377 }

1341 }

1378 }

1342 }

1379 }

1343 }

1380−

1381− fn insert_layer(&mut self, mut layer: HighlightIterLayer<'a>) {

1382− if let Some(sort_key) = layer.sort_key() {

1383− let mut i = 1;

1384− while i < self.layers.len() {

1385− if let Some(sort_key_i) = self.layers[i].sort_key() {

1386− if sort_key_i > sort_key {

1387− self.layers.insert(i, layer);

1388− return;

1389− }

1390− i += 1;

1391− } else {

1392− self.layers.remove(i);

1393− }

1394− }

1395− self.layers.push(layer);

1396− }

1397− }

1398}

1344}

1399

1345

1400−impl<'a, F> Iterator for HighlightIter<'a, F>

1346+impl<'a> Iterator for HighlightIter<'a> {

1401−where

1402− F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,

1403−{

1404 type Item = Result<HighlightEvent, Error>;

1347 type Item = Result<HighlightEvent, Error>;

1405

1348

1406 fn next(&mut self) -> Option<Self::Item> {

1349 fn next(&mut self) -> Option<Self::Item> {

@@ -1460,55 +1403,12 @@ where

1460 layer.highlight_end_stack.pop();

1403 layer.highlight_end_stack.pop();

1461 return self.emit_event(end_byte, Some(HighlightEvent::HighlightEnd));

1404 return self.emit_event(end_byte, Some(HighlightEvent::HighlightEnd));

1462 } else {

1405 } else {

1463− // return self.emit_event(self.source.len(), None);

1406+ return self.emit_event(self.source.len_bytes(), None);

1464− return None;

1465 };

1407 };

1466

1408

1467 let (mut match_, capture_index) = layer.captures.next().unwrap();

1409 let (mut match_, capture_index) = layer.captures.next().unwrap();

1468 let mut capture = match_.captures[capture_index];

1410 let mut capture = match_.captures[capture_index];

1469

1411

1470− // If this capture represents an injection, then process the injection.

1471− if match_.pattern_index < layer.config.locals_pattern_index {

1472− let (language_name, content_node, include_children) =

1473− injection_for_match(layer.config, &layer.config.query, &match_, self.source);

1474−

1475− // Explicitly remove this match so that none of its other captures will remain

1476− // in the stream of captures.

1477− match_.remove();

1478−

1479− // If a language is found with the given name, then add a new language layer

1480− // to the highlighted document.

1481− if let (Some(language_name), Some(content_node)) = (language_name, content_node) {

1482− if let Some(config) = (self.injection_callback)(&language_name) {

1483− let ranges = HighlightIterLayer::intersect_ranges(

1484− &self.layers[0].ranges,

1485− &[content_node],

1486− include_children,

1487− );

1488− if !ranges.is_empty() {

1489− match HighlightIterLayer::new(

1490− self.source,

1491− self.cancellation_flag,

1492− &mut self.injection_callback,

1493− config,

1494− self.layers[0].depth + 1,

1495− ranges,

1496− ) {

1497− Ok(layers) => {

1498− for layer in layers {

1499− self.insert_layer(layer);

1500− }

1501− }

1502− Err(e) => return Some(Err(e)),

1503− }

1504− }

1505− }

1506− }

1507−

1508− self.sort_layers();

1509− continue 'main;

1510− }

1511−

1512 // Remove from the local scope stack any local scopes that have already ended.

1412 // Remove from the local scope stack any local scopes that have already ended.

1513 while range.start > layer.scope_stack.last().unwrap().range.end {

1413 while range.start > layer.scope_stack.last().unwrap().range.end {

1514 layer.scope_stack.pop();

1414 layer.scope_stack.pop();

@@ -1703,14 +1603,6 @@ fn injection_for_match<'a>(

1703 (language_name, content_node, include_children)

1603 (language_name, content_node, include_children)

1704}

1604}

1705

1605

1706−// fn shrink_and_clear<T>(vec: &mut Vec<T>, capacity: usize) {

1707−// if vec.len() > capacity {

1708−// vec.truncate(capacity);

1709−// vec.shrink_to_fit();

1710−// }

1711−// vec.clear();

1712−// }

1713−

1714pub struct Merge<I> {

1606pub struct Merge<I> {

1715 iter: I,

1607 iter: I,

1716 spans: Box<dyn Iterator<Item = (usize, std::ops::Range<usize>)>>,

1608 spans: Box<dyn Iterator<Item = (usize, std::ops::Range<usize>)>>,

@@ -1877,6 +1769,8 @@ mod test {

1877 .map(String::from)

1769 .map(String::from)

1878 .collect();

1770 .collect();

1879

1771

1772+ let loader = Loader::new(Configuration { language: vec![] });

1773+

1880 let language = get_language(&crate::RUNTIME_DIR, "Rust").unwrap();

1774 let language = get_language(&crate::RUNTIME_DIR, "Rust").unwrap();

1881 let config = HighlightConfiguration::new(

1775 let config = HighlightConfiguration::new(

1882 language,

1776 language,

@@ -1899,7 +1793,7 @@ mod test {

1899 fn main() {}

1793 fn main() {}

1900 ",

1794 ",

1901 );

1795 );

1902− let syntax = Syntax::new(&source, Arc::new(config));

1796+ let syntax = Syntax::new(&source, Arc::new(config), Arc::new(loader));

1903 let tree = syntax.tree();

1797 let tree = syntax.tree();

1904 let root = tree.root_node();

1798 let root = tree.root_node();

1905 assert_eq!(root.kind(), "source_file");

1799 assert_eq!(root.kind(), "source_file");

@@ -1926,7 +1820,7 @@ mod test {

1926 &doc,

1820 &doc,

1927 vec![(6, 11, Some("test".into())), (12, 17, None)].into_iter(),

1821 vec![(6, 11, Some("test".into())), (12, 17, None)].into_iter(),

1928 );

1822 );

1929− let edits = LanguageLayer::generate_edits(doc.slice(..), transaction.changes());

1823+ let edits = generate_edits(doc.slice(..), transaction.changes());

1930 // transaction.apply(&mut state);

1824 // transaction.apply(&mut state);

1931

1825

1932 assert_eq!(

1826 assert_eq!(

@@ -1955,7 +1849,7 @@ mod test {

1955 let mut doc = Rope::from("fn test() {}");

1849 let mut doc = Rope::from("fn test() {}");

1956 let transaction =

1850 let transaction =

1957 Transaction::change(&doc, vec![(8, 8, Some("a: u32".into()))].into_iter());

1851 Transaction::change(&doc, vec![(8, 8, Some("a: u32".into()))].into_iter());

1958− let edits = LanguageLayer::generate_edits(doc.slice(..), transaction.changes());

1852+ let edits = generate_edits(doc.slice(..), transaction.changes());

1959 transaction.apply(&mut doc);

1853 transaction.apply(&mut doc);

1960

1854

1961 assert_eq!(doc, "fn test(a: u32) {}");

1855 assert_eq!(doc, "fn test(a: u32) {}");

helix-term/src/ui/editor.rs+3−24

@@ -68,13 +68,12 @@ impl EditorView {

68 surface: &mut Surface,

69 theme: &Theme,

70 is_focused: bool,

71− loader: &syntax::Loader,

72 config: &helix_view::editor::Config,

71 config: &helix_view::editor::Config,

73 ) {

72 ) {

74 let inner = view.inner_area();

73 let inner = view.inner_area();

75 let area = view.area;

74 let area = view.area;

77− let highlights = Self::doc_syntax_highlights(doc, view.offset, inner.height, theme, loader);

76+ let highlights = Self::doc_syntax_highlights(doc, view.offset, inner.height, theme);

78 let highlights = syntax::merge(highlights, Self::doc_diagnostics_highlights(doc, theme));

77 let highlights = syntax::merge(highlights, Self::doc_diagnostics_highlights(doc, theme));

79 let highlights: Box<dyn Iterator<Item = HighlightEvent>> = if is_focused {

78 let highlights: Box<dyn Iterator<Item = HighlightEvent>> = if is_focused {

80 Box::new(syntax::merge(

79 Box::new(syntax::merge(

@@ -121,8 +120,7 @@ impl EditorView {

121 doc: &'doc Document,

120 doc: &'doc Document,

122 offset: Position,

121 offset: Position,

123 height: u16,

122 height: u16,

124− theme: &Theme,

123+ _theme: &Theme,

125− loader: &syntax::Loader,

126 ) -> Box<dyn Iterator<Item = HighlightEvent> + 'doc> {

124 ) -> Box<dyn Iterator<Item = HighlightEvent> + 'doc> {

127 let text = doc.text().slice(..);

125 let text = doc.text().slice(..);

128 let last_line = std::cmp::min(

126 let last_line = std::cmp::min(

@@ -142,25 +140,8 @@ impl EditorView {

142 // TODO: range doesn't actually restrict source, just highlight range

140 // TODO: range doesn't actually restrict source, just highlight range

143 let highlights = match doc.syntax() {

141 let highlights = match doc.syntax() {

144 Some(syntax) => {

142 Some(syntax) => {

145− let scopes = theme.scopes();

146 syntax

143 syntax

147− .highlight_iter(text.slice(..), Some(range), None, |language| {

144+ .highlight_iter(text.slice(..), Some(range), None)

148− loader.language_configuration_for_injection_string(language)

149− .and_then(|language_config| {

150− let config = language_config.highlight_config(scopes)?;

151− let config_ref = config.as_ref();

152− // SAFETY: the referenced `HighlightConfiguration` behind

153− // the `Arc` is guaranteed to remain valid throughout the

154− // duration of the highlight.

155− let config_ref = unsafe {

156− std::mem::transmute::<

157− _,

158− &'static syntax::HighlightConfiguration,

159− >(config_ref)

160− };

161− Some(config_ref)

162− })

163− })

164 .map(|event| event.unwrap())

145 .map(|event| event.unwrap())

165 .collect() // TODO: we collect here to avoid holding the lock, fix later

146 .collect() // TODO: we collect here to avoid holding the lock, fix later

166 }

147 }

@@ -1070,7 +1051,6 @@ impl Component for EditorView {

1070

1051

1071 for (view, is_focused) in cx.editor.tree.views() {

1052 for (view, is_focused) in cx.editor.tree.views() {

1072 let doc = cx.editor.document(view.doc).unwrap();

1053 let doc = cx.editor.document(view.doc).unwrap();

1073− let loader = &cx.editor.syn_loader;

1074 self.render_view(

1054 self.render_view(

1075 doc,

1055 doc,

1076 view,

1056 view,

@@ -1078,7 +1058,6 @@ impl Component for EditorView {

1078 surface,

1058 surface,

1079 &cx.editor.theme,

1059 &cx.editor.theme,

1080 is_focused,

1060 is_focused,

1081− loader,

1082 &cx.editor.config,

1061 &cx.editor.config,

1083 );

1062 );

1084 }

1063 }

helix-term/src/ui/markdown.rs+9−6

@@ -38,7 +38,7 @@ impl Markdown {

38fn parse<'a>(

39 contents: &'a str,

40 theme: Option<&Theme>,

41− loader: &syntax::Loader,

41+ loader: Arc<syntax::Loader>,

42) -> tui::text::Text<'a> {

43 // // also 2021-03-04T16:33:58.553 helix_lsp::transport [INFO] <- {"contents":{"kind":"markdown","value":"\n```rust\ncore::num\n```\n\n```rust\npub const fn saturating_sub(self, rhs:Self) ->Self\n```\n\n---\n\n```rust\n```"},"range":{"end":{"character":61,"line":101},"start":{"character":47,"line":101}}}

44 // let text = "\n```rust\ncore::iter::traits::iterator::Iterator\n```\n\n```rust\nfn collect<B: FromIterator<Self::Item>>(self) -> B\nwhere\n Self: Sized,\n```\n\n---\n\nTransforms an iterator into a collection.\n\n`collect()` can take anything iterable, and turn it into a relevant\ncollection. This is one of the more powerful methods in the standard\nlibrary, used in a variety of contexts.\n\nThe most basic pattern in which `collect()` is used is to turn one\ncollection into another. You take a collection, call [`iter`](https://doc.rust-lang.org/nightly/core/iter/traits/iterator/trait.Iterator.html) on it,\ndo a bunch of transformations, and then `collect()` at the end.\n\n`collect()` can also create instances of types that are not typical\ncollections. For example, a [`String`](https://doc.rust-lang.org/nightly/core/iter/std/string/struct.String.html) can be built from [`char`](type@char)s,\nand an iterator of [`Result<T, E>`](https://doc.rust-lang.org/nightly/core/result/enum.Result.html) items can be collected\ninto `Result<Collection<T>, E>`. See the examples below for more.\n\nBecause `collect()` is so general, it can cause problems with type\ninference. As such, `collect()` is one of the few times you'll see\nthe syntax affectionately known as the 'turbofish': `::<>`. This\nhelps the inference algorithm understand specifically which collection\nyou're trying to collect into.\n\n# Examples\n\nBasic usage:\n\n```rust\nlet a = [1, 2, 3];\n\nlet doubled: Vec<i32> = a.iter()\n .map(|&x| x * 2)\n .collect();\n\nassert_eq!(vec![2, 4, 6], doubled);\n```\n\nNote that we needed the `: Vec<i32>` on the left-hand side. This is because\nwe could collect into, for example, a [`VecDeque<T>`](https://doc.rust-lang.org/nightly/core/iter/std/collections/struct.VecDeque.html) instead:\n\n```rust\nuse std::collections::VecDeque;\n\nlet a = [1, 2, 3];\n\nlet doubled: VecDeque<i32> = a.iter().map(|&x| x * 2).collect();\n\nassert_eq!(2, doubled[0]);\nassert_eq!(4, doubled[1]);\nassert_eq!(6, doubled[2]);\n```\n\nUsing the 'turbofish' instead of annotating `doubled`:\n\n```rust\nlet a = [1, 2, 3];\n\nlet doubled = a.iter().map(|x| x * 2).collect::<Vec<i32>>();\n\nassert_eq!(vec![2, 4, 6], doubled);\n```\n\nBecause `collect()` only cares about what you're collecting into, you can\nstill use a partial type hint, `_`, with the turbofish:\n\n```rust\nlet a = [1, 2, 3];\n\nlet doubled = a.iter().map(|x| x * 2).collect::<Vec<_>>();\n\nassert_eq!(vec![2, 4, 6], doubled);\n```\n\nUsing `collect()` to make a [`String`](https://doc.rust-lang.org/nightly/core/iter/std/string/struct.String.html):\n\n```rust\nlet chars = ['g', 'd', 'k', 'k', 'n'];\n\nlet hello: String = chars.iter()\n .map(|&x| x as u8)\n .map(|x| (x + 1) as char)\n .collect();\n\nassert_eq!(\"hello\", hello);\n```\n\nIf you have a list of [`Result<T, E>`](https://doc.rust-lang.org/nightly/core/result/enum.Result.html)s, you can use `collect()` to\nsee if any of them failed:\n\n```rust\nlet results = [Ok(1), Err(\"nope\"), Ok(3), Err(\"bad\")];\n\nlet result: Result<Vec<_>, &str> = results.iter().cloned().collect();\n\n// gives us the first error\nassert_eq!(Err(\"nope\"), result);\n\nlet results = [Ok(1), Ok(3)];\n\nlet result: Result<Vec<_>, &str> = results.iter().cloned().collect();\n\n// gives us the list of answers\nassert_eq!(Ok(vec![1, 3]), result);\n```";

@@ -98,14 +98,13 @@ fn parse<'a>(

98 let syntax = loader

99 .language_configuration_for_injection_string(language)

100 .and_then(|config| config.highlight_config(theme.scopes()))

101− .map(|config| Syntax::new(&rope, config));

101+ .map(|config| Syntax::new(&rope, config, loader.clone()));

102

103 if let Some(syntax) = syntax {

104 // if we have a syntax available, highlight_iter and generate spans

105 let mut highlights = Vec::new();

106

107− for event in syntax.highlight_iter(rope.slice(..), None, None, |_| None)

107+ for event in syntax.highlight_iter(rope.slice(..), None, None) {

108− {

109 match event.unwrap() {

108 match event.unwrap() {

110 HighlightEvent::HighlightStart(span) => {

109 HighlightEvent::HighlightStart(span) => {

111 highlights.push(span);

110 highlights.push(span);

@@ -211,7 +210,11 @@ impl Component for Markdown {

211 fn render(&mut self, area: Rect, surface: &mut Surface, cx: &mut Context) {

210 fn render(&mut self, area: Rect, surface: &mut Surface, cx: &mut Context) {

212 use tui::widgets::{Paragraph, Widget, Wrap};

211 use tui::widgets::{Paragraph, Widget, Wrap};

213

212

213+ let text = parse(

214+ &self.contents,

214− let text = parse(&self.contents, Some(&cx.editor.theme), &self.config_loader);

215+ Some(&cx.editor.theme),

216+ self.config_loader.clone(),

217+ );

215

218

216 let par = Paragraph::new(text)

219 let par = Paragraph::new(text)

217 .wrap(Wrap { trim: false })

220 .wrap(Wrap { trim: false })

@@ -229,7 +232,7 @@ impl Component for Markdown {

229 if padding >= viewport.1 || padding >= viewport.0 {

232 if padding >= viewport.1 || padding >= viewport.0 {

230 return None;

233 return None;

231 }

234 }

232− let contents = parse(&self.contents, None, &self.config_loader);

235+ let contents = parse(&self.contents, None, self.config_loader.clone());

233 // TODO: account for tab width

236 // TODO: account for tab width

234 let max_text_width = (viewport.0 - padding).min(120);

237 let max_text_width = (viewport.0 - padding).min(120);

235 let mut text_width = 0;

238 let mut text_width = 0;

helix-term/src/ui/picker.rs+2−7

@@ -221,13 +221,8 @@ impl<T: 'static> Component for FilePicker<T> {

221

222 let offset = Position::new(first_line, 0);

223

224+ let highlights =

224− let highlights = EditorView::doc_syntax_highlights(

225+ EditorView::doc_syntax_highlights(doc, offset, area.height, &cx.editor.theme);

225− doc,

226− offset,

227− area.height,

228− &cx.editor.theme,

229− &cx.editor.syn_loader,

230− );

231 EditorView::render_text_highlights(

226 EditorView::render_text_highlights(

232 doc,

227 doc,

233 offset,

228 offset,

helix-view/src/document.rs+7−6

@@ -359,7 +359,7 @@ impl Document {

359 path: &Path,

360 encoding: Option<&'static encoding::Encoding>,

361 theme: Option<&Theme>,

362− config_loader: Option<&syntax::Loader>,

362+ config_loader: Option<Arc<syntax::Loader>>,

363 ) -> Result<Self, Error> {

364 // Open the file if it exists, otherwise assume it is a new file (and thus empty).

365 let (rope, encoding) = if path.exists() {

@@ -498,12 +498,12 @@ impl Document {

498 }

499

500 /// Detect the programming language based on the file type.

501− pub fn detect_language(&mut self, theme: Option<&Theme>, config_loader: &syntax::Loader) {

501+ pub fn detect_language(&mut self, theme: Option<&Theme>, config_loader: Arc<syntax::Loader>) {

502 if let Some(path) = &self.path {

503 let language_config = config_loader

504 .language_config_for_file_name(path)

505 .or_else(|| config_loader.language_config_for_shebang(self.text()));

506− self.set_language(theme, language_config);

506+ self.set_language(theme, language_config, Some(config_loader));

507 }

508 }

509

@@ -579,11 +579,12 @@ impl Document {

579 &mut self,

580 theme: Option<&Theme>,

581 language_config: Option<Arc<helix_core::syntax::LanguageConfiguration>>,

582+ loader: Option<Arc<helix_core::syntax::Loader>>,

582 ) {

583 ) {

583− if let Some(language_config) = language_config {

584+ if let (Some(language_config), Some(loader)) = (language_config, loader) {

584 let scopes = theme.map(|theme| theme.scopes()).unwrap_or(&[]);

585 let scopes = theme.map(|theme| theme.scopes()).unwrap_or(&[]);

585 if let Some(highlight_config) = language_config.highlight_config(scopes) {

586 if let Some(highlight_config) = language_config.highlight_config(scopes) {

586− let syntax = Syntax::new(&self.text, highlight_config);

587+ let syntax = Syntax::new(&self.text, highlight_config, loader);

587 self.syntax = Some(syntax);

588 self.syntax = Some(syntax);

588 // TODO: config.configure(scopes) is now delayed, is that ok?

589 // TODO: config.configure(scopes) is now delayed, is that ok?

589 }

590 }

@@ -605,7 +606,7 @@ impl Document {

605 ) {

606 ) {

606 let language_config = config_loader.language_config_for_scope(scope);

607 let language_config = config_loader.language_config_for_scope(scope);

607

608

608− self.set_language(theme, language_config);

609+ self.set_language(theme, language_config, Some(config_loader));

609 }

610 }

610

611

611 /// Set the LSP.

612 /// Set the LSP.

helix-view/src/editor.rs+7−2

@@ -283,7 +283,7 @@ impl Editor {

283 /// Refreshes the language server for a given document

284 pub fn refresh_language_server(&mut self, doc_id: DocumentId) -> Option<()> {

285 let doc = self.documents.get_mut(&doc_id)?;

286− doc.detect_language(Some(&self.theme), &self.syn_loader);

286+ doc.detect_language(Some(&self.theme), self.syn_loader.clone());

287 Self::launch_language_server(&mut self.language_servers, doc)

288 }

289

@@ -462,7 +462,12 @@ impl Editor {

462 let id = if let Some(id) = id {

463 id

464 } else {

465− let mut doc = Document::open(&path, None, Some(&self.theme), Some(&self.syn_loader))?;

465+ let mut doc = Document::open(

466+ &path,

467+ None,

468+ Some(&self.theme),

469+ Some(self.syn_loader.clone()),

470+ )?;

466

471

467 let _ = Self::launch_language_server(&mut self.language_servers, &mut doc);

472 let _ = Self::launch_language_server(&mut self.language_servers, &mut doc);

468

473