swc-project/swc
1
1
Fork 0
mirror of https://github.com/swc-project/swc.git synced 2024-10-04 04:07:18 +03:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

fix(es/parser): Ensure that comments are collected (#2407)

Browse Source
This commit is contained in:
David Sherret 2021-10-11 22:07:52 -04:00 committed by GitHub
parent a36f8e42bd
commit 4d4771109a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 266 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Cargo.lock generated
View File

@ -294,11 +294,11 @@ dependencies = [
[[package]]
name = "console_error_panic_hook"
version = "0.1.6"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8d976903543e0c48546a91908f21588a680a8c8f984df9a5d69feccb2b2a211"
checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc"
dependencies = [
"cfg-if 0.1.10",
"cfg-if 1.0.0",
"wasm-bindgen",
]
@ -2689,7 +2689,7 @@ dependencies = [
[[package]]
name = "swc_ecma_parser"
version = "0.73.9"
version = "0.73.10"
dependencies = [
"either",
"enum_kind",

2
ecmascript/parser/Cargo.toml
View File

@ -7,7 +7,7 @@ include = ["Cargo.toml", "src/**/*.rs", "examples/**/*.rs"]
license = "Apache-2.0/MIT"
name = "swc_ecma_parser"
repository = "https://github.com/swc-project/swc.git"
version = "0.73.9"
version = "0.73.10"
[package.metadata.docs.rs]
all-features = true

97
ecmascript/parser/src/lexer/comments_buffer.rs Normal file
View File

@ -0,0 +1,97 @@
use std::{iter::Rev, rc::Rc, vec::IntoIter};
use swc_common::{comments::Comment, BytePos};
#[derive(Clone)]
pub struct BufferedComment {
pub kind: BufferedCommentKind,
pub pos: BytePos,
pub comment: Comment,
}
#[derive(Clone)]
pub enum BufferedCommentKind {
Leading,
Trailing,
}
#[derive(Clone)]
pub struct CommentsBuffer {
comments: OneDirectionalList<BufferedComment>,
pending_leading: OneDirectionalList<Comment>,
}
impl CommentsBuffer {
pub fn new() -> Self {
Self {
comments: OneDirectionalList::new(),
pending_leading: OneDirectionalList::new(),
}
}
pub fn push(&mut self, comment: BufferedComment) {
self.comments.push(comment);
}
pub fn push_pending_leading(&mut self, comment: Comment) {
self.pending_leading.push(comment);
}
pub fn take_comments(&mut self) -> Rev<IntoIter<BufferedComment>> {
self.comments.take_all()
}
pub fn take_pending_leading(&mut self) -> Rev<IntoIter<Comment>> {
self.pending_leading.take_all()
}
}
/// A one direction linked list that can be cheaply
/// cloned with the clone maintaining its position in the list.
#[derive(Clone)]
struct OneDirectionalList<T: Clone> {
last_node: Option<Rc<OneDirectionalListNode<T>>>,
}
impl<T: Clone> OneDirectionalList<T> {
pub fn new() -> Self {
Self { last_node: None }
}
pub fn len(&self) -> usize {
self.last_node.as_ref().map(|i| i.length).unwrap_or(0)
}
pub fn take_all(&mut self) -> Rev<IntoIter<T>> {
// these are stored in reverse, so we need to reverse them back
let mut items = Vec::with_capacity(self.len());
let mut current_node = self.last_node.take();
while let Some(node) = current_node {
let mut node = match Rc::try_unwrap(node) {
Ok(n) => n,
Err(n) => n.as_ref().clone(),
};
items.push(node.item);
current_node = node.previous.take();
}
items.into_iter().rev()
}
pub fn push(&mut self, item: T) {
let previous = self.last_node.take();
let length = previous.as_ref().map(|p| p.length + 1).unwrap_or(1);
let new_item = OneDirectionalListNode {
item,
previous,
length,
};
self.last_node = Some(Rc::new(new_item));
}
}
#[derive(Clone)]
struct OneDirectionalListNode<T: Clone> {
item: T,
previous: Option<Rc<OneDirectionalListNode<T>>>,
length: usize,
}

29
ecmascript/parser/src/lexer/mod.rs
View File

@ -1,10 +1,10 @@
//! ECMAScript lexer.
use self::{comments_buffer::CommentsBuffer, state::State, util::*};
pub use self::{
input::Input,
state::{TokenContext, TokenContexts},
};
use self::{state::State, util::*};
use crate::{
error::{Error, SyntaxError},
token::*,
@ -14,12 +14,10 @@ use either::Either::{Left, Right};
use smallvec::{smallvec, SmallVec};
use std::{cell::RefCell, char, iter::FusedIterator, rc::Rc};
use swc_atoms::{js_word, JsWord};
use swc_common::{
comments::{Comment, Comments},
BytePos, Span,
};
use swc_common::{comments::Comments, BytePos, Span};
use swc_ecma_ast::op;
mod comments_buffer;
pub mod input;
mod jsx;
mod number;
@ -101,15 +99,10 @@ impl FusedIterator for CharIter {}
pub struct Lexer<'a, I: Input> {
comments: Option<&'a dyn Comments>,
/// [Some] if comment comment parsing is enabled. Otherwise [None]
leading_comments_buffer: Option<Rc<RefCell<Vec<Comment>>>>,
comments_buffer: Option<CommentsBuffer>,
pub(crate) ctx: Context,
input: I,
/// Stores last position of the last comment.
///
/// [Rc] and [RefCell] is used because this value should always increment,
/// even if backtracking fails.
last_comment_pos: Rc<RefCell<BytePos>>,
state: State,
pub(crate) syntax: Syntax,
@ -132,14 +125,9 @@ impl<'a, I: Input> Lexer<'a, I> {
) -> Self {
Lexer {
comments,
leading_comments_buffer: if comments.is_some() {
Some(Default::default())
} else {
None
},
comments_buffer: comments.is_some().then(CommentsBuffer::new),
ctx: Default::default(),
input,
last_comment_pos: Rc::new(RefCell::new(BytePos(0))),
state: State::new(syntax),
syntax,
target,
@ -635,12 +623,7 @@ impl<'a, I: Input> Lexer<'a, I> {
self.bump();
// XML style comment. `<!--`
if !self.ctx.dont_store_comments
&& c == '<'
&& self.is(b'!')
&& self.peek() == Some('-')
&& self.peek_ahead() == Some('-')
{
if c == '<' && self.is(b'!') && self.peek() == Some('-') && self.peek_ahead() == Some('-') {
self.skip_line_comment(3);
self.skip_space()?;
self.emit_module_mode_error(start, SyntaxError::LegacyCommentInModule);

77
ecmascript/parser/src/lexer/state.rs
View File

@ -1,7 +1,10 @@
use super::{Context, Input, Lexer};
use super::{
comments_buffer::{BufferedComment, BufferedCommentKind},
Context, Input, Lexer,
};
use crate::{error::Error, input::Tokens, lexer::util::CharExt, token::*, JscTarget, Syntax};
use enum_kind::Kind;
use std::{mem, mem::take};
use std::mem::take;
use swc_common::BytePos;
use tracing::trace;
@ -182,30 +185,39 @@ impl<'a, I: Input> Iterator for Lexer<'a, I> {
Some(c) => c,
// End of input.
None => {
if self
.leading_comments_buffer
.as_ref()
.map(|v| !v.borrow().is_empty())
.unwrap_or(false)
{
if let Some(comments) = self.comments.as_mut() {
let comments_buffer = self.comments_buffer.as_mut().unwrap();
let last = self.state.prev_hi;
for c in self
.leading_comments_buffer
.as_ref()
.map(|v| v.borrow_mut())
.unwrap()
.drain(..)
{
let comments = self.comments.as_mut().unwrap();
// move the pending to the leading or trailing
for c in comments_buffer.take_pending_leading() {
// if the file had no tokens and no shebang, then treat any
// comments in the leading comments buffer as leading.
// Otherwise treat them as trailing.
if last == BytePos(0) {
comments.add_leading(last, c);
comments_buffer.push(BufferedComment {
kind: BufferedCommentKind::Leading,
pos: last,
comment: c,
});
} else {
comments.add_trailing(last, c);
comments_buffer.push(BufferedComment {
kind: BufferedCommentKind::Trailing,
pos: last,
comment: c,
});
}
}
// now fill the user's passed in comments
for comment in comments_buffer.take_comments() {
match comment.kind {
BufferedCommentKind::Leading => {
comments.add_leading(comment.pos, comment.comment);
}
BufferedCommentKind::Trailing => {
comments.add_trailing(comment.pos, comment.comment);
}
}
}
}
@ -283,25 +295,14 @@ impl<'a, I: Input> Iterator for Lexer<'a, I> {
let span = self.span(start);
if let Some(ref token) = token {
if self.leading_comments_buffer.is_some()
&& !self
.leading_comments_buffer
.as_ref()
.unwrap()
.borrow()
.is_empty()
{
self.comments.as_ref().unwrap().add_leading_comments(
start,
mem::replace(
&mut *self
.leading_comments_buffer
.as_ref()
.map(|v| v.borrow_mut())
.unwrap(),
vec![],
),
);
if let Some(comments) = self.comments_buffer.as_mut() {
for comment in comments.take_pending_leading() {
comments.push(BufferedComment {
kind: BufferedCommentKind::Leading,
pos: start,
comment,
});
}
}
self.state.update(start, &token);
self.state.prev_hi = self.last_pos();

75
ecmascript/parser/src/lexer/util.rs
View File

@ -5,9 +5,10 @@
//!
//!
//! [babylon/util/identifier.js]:https://github.com/babel/babel/blob/master/packages/babylon/src/util/identifier.js
use super::{input::Input, Char, LexResult, Lexer};
use super::{comments_buffer::BufferedComment, input::Input, Char, LexResult, Lexer};
use crate::{
error::{Error, SyntaxError},
lexer::comments_buffer::BufferedCommentKind,
Tokens,
};
use std::char;
@ -230,26 +231,22 @@ impl<'a, I: Input> Lexer<'a, I> {
}
}
if !self.ctx.dont_store_comments {
if let Some(ref comments) = self.comments {
let s = self.input.slice(slice_start, end);
let cmt = Comment {
kind: CommentKind::Line,
span: Span::new(start, end, SyntaxContext::empty()),
text: s.into(),
};
if let Some(comments) = self.comments_buffer.as_mut() {
let s = self.input.slice(slice_start, end);
let cmt = Comment {
kind: CommentKind::Line,
span: Span::new(start, end, SyntaxContext::empty()),
text: s.into(),
};
if start >= *self.last_comment_pos.borrow() {
*self.last_comment_pos.borrow_mut() = end;
if is_for_next {
if let Some(buf) = &self.leading_comments_buffer {
buf.borrow_mut().push(cmt);
}
} else {
comments.add_trailing(self.state.prev_hi, cmt);
}
}
if is_for_next {
comments.push_pending_leading(cmt);
} else {
comments.push(BufferedComment {
kind: BufferedCommentKind::Trailing,
pos: self.state.prev_hi,
comment: cmt,
});
}
}
@ -284,28 +281,24 @@ impl<'a, I: Input> Lexer<'a, I> {
let end = self.cur_pos();
if !self.ctx.dont_store_comments {
if let Some(ref comments) = self.comments {
let src = self.input.slice(slice_start, end);
let s = &src[..src.len() - 2];
let cmt = Comment {
kind: CommentKind::Block,
span: Span::new(start, end, SyntaxContext::empty()),
text: s.into(),
};
if let Some(comments) = self.comments_buffer.as_mut() {
let src = self.input.slice(slice_start, end);
let s = &src[..src.len() - 2];
let cmt = Comment {
kind: CommentKind::Block,
span: Span::new(start, end, SyntaxContext::empty()),
text: s.into(),
};
let _ = self.input.peek();
if start >= *self.last_comment_pos.borrow() {
*self.last_comment_pos.borrow_mut() = end;
if is_for_next {
if let Some(buf) = &self.leading_comments_buffer {
buf.borrow_mut().push(cmt);
}
} else {
comments.add_trailing(self.state.prev_hi, cmt);
}
}
let _ = self.input.peek();
if is_for_next {
comments.push_pending_leading(cmt);
} else {
comments.push(BufferedComment {
kind: BufferedCommentKind::Trailing,
pos: self.state.prev_hi,
comment: cmt,
});
}
}

3
ecmascript/parser/src/lib.rs
View File

@ -428,9 +428,6 @@ pub struct Context {
/// If true, `:` should not be treated as a type annotation.
dont_parse_colon_as_type_ann: bool,
/// See: https://github.com/swc-project/swc/issues/2264
dont_store_comments: bool,
}
#[cfg(test)]

1
ecmascript/parser/src/parser/expr.rs
View File

@ -82,7 +82,6 @@ impl<'a, I: Tokens> Parser<I> {
{
let ctx = Context {
is_direct_child_of_cond: false,
dont_store_comments: true,
..self.ctx()
};
let res = self.with_ctx(ctx).try_parse_ts(|p| {

131
ecmascript/parser/src/parser/tests.rs
View File

@ -183,56 +183,95 @@ fn issue_1878() {
#[test]
fn issue_2264_1() {
// file with only comments should have the comments
// in the leading map instead of the trailing
{
let c = SingleThreadedComments::default();
let s = "
const t = <Switch>
// 1
/* 2 */
</Switch>
";
let _ = super::test_parser_comment(
&c,
s,
Syntax::Typescript(TsConfig {
tsx: true,
..Default::default()
}),
|p| p.parse_typescript_module(),
);
let c = SingleThreadedComments::default();
let s = "
const t = <Switch>
// 1
/* 2 */
</Switch>
";
let _ = super::test_parser_comment(
&c,
s,
Syntax::Typescript(TsConfig {
tsx: true,
..Default::default()
}),
|p| p.parse_typescript_module(),
);
let (leading, trailing) = c.take_all();
assert!(leading.borrow().is_empty());
assert!(trailing.borrow().is_empty());
}
let (leading, trailing) = c.take_all();
assert!(leading.borrow().is_empty());
assert!(trailing.borrow().is_empty());
}
#[test]
fn issue_2264_2() {
// file with only comments should have the comments
// in the leading map instead of the trailing
{
let c = SingleThreadedComments::default();
let s = "
const t = <Switch>
// 1
/* 2 */
</Switch>
";
let _ = super::test_parser_comment(
&c,
s,
Syntax::Es(EsConfig {
jsx: true,
..Default::default()
}),
|p| p.parse_module(),
);
let c = SingleThreadedComments::default();
let s = "
const t = <Switch>
// 1
/* 2 */
</Switch>
";
let _ = super::test_parser_comment(
&c,
s,
Syntax::Es(EsConfig {
jsx: true,
..Default::default()
}),
|p| p.parse_module(),
);
let (leading, trailing) = c.take_all();
assert!(leading.borrow().is_empty());
assert!(trailing.borrow().is_empty());
}
let (leading, trailing) = c.take_all();
assert!(leading.borrow().is_empty());
assert!(trailing.borrow().is_empty());
}
#[test]
fn issue_2264_3() {
let c = SingleThreadedComments::default();
let s = "const foo = <h1>/* no */{/* 1 */ bar /* 2 */}/* no */</h1>;";
let _ = super::test_parser_comment(
&c,
s,
Syntax::Typescript(TsConfig {
tsx: true,
..Default::default()
}),
|p| p.parse_typescript_module(),
);
let (leading, trailing) = c.take_all();
assert!(leading.borrow().is_empty());
assert_eq!(trailing.borrow().len(), 2);
assert_eq!(trailing.borrow().get(&BytePos(25)).unwrap().len(), 1);
assert_eq!(trailing.borrow().get(&BytePos(36)).unwrap().len(), 1);
}
#[test]
fn issue_2339_1() {
let c = SingleThreadedComments::default();
let s = "
const t = <T>() => {
// 1
/* 2 */
test;
};
";
let _ = super::test_parser_comment(
&c,
s,
Syntax::Typescript(TsConfig {
tsx: true,
..Default::default()
}),
|p| p.parse_typescript_module(),
);
let (leading, trailing) = c.take_all();
assert_eq!(leading.borrow().len(), 1);
assert_eq!(leading.borrow().get(&BytePos(79)).unwrap().len(), 2);
assert!(trailing.borrow().is_empty());
}