mirror of
https://github.com/swc-project/swc.git
synced 2024-11-28 02:29:04 +03:00
Fix parser (#727)
- Allow await in an yield expression (fixes #720) - Prevent duplicate tokens while capturing (fixes #726)
This commit is contained in:
강동윤
GitHub
parent
b17b249fa6
commit
ebc7070d1f
@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "swc_ecma_parser"
|
||||
version = "0.21.6"
|
||||
version = "0.21.7"
|
||||
authors = ["강동윤 <kdy1997.dev@gmail.com>"]
|
||||
license = "Apache-2.0/MIT"
|
||||
repository = "https://github.com/swc-project/swc.git"
|
||||
|
||||
@ -116,6 +116,9 @@ impl<I: Input> Tokens for Lexer<'_, I> {
|
||||
self.target
|
||||
}
|
||||
|
||||
/// no-op, as `Lexer` does not use `Rc<RefCelll<T>>`.
|
||||
fn revert(&mut self) {}
|
||||
|
||||
fn set_expr_allowed(&mut self, allow: bool) {
|
||||
self.set_expr_allowed(allow)
|
||||
}
|
||||
|
||||
@ -14,6 +14,10 @@ pub trait Tokens: Clone + Iterator<Item = TokenAndSpan> {
|
||||
fn syntax(&self) -> Syntax;
|
||||
fn target(&self) -> JscTarget;
|
||||
|
||||
/// Revert to lastest clone. THis method exists to removed captured token
|
||||
/// while backtracking.
|
||||
fn revert(&mut self);
|
||||
|
||||
fn set_expr_allowed(&mut self, allow: bool);
|
||||
fn token_context(&self) -> &lexer::TokenContexts;
|
||||
fn token_context_mut(&mut self) -> &mut lexer::TokenContexts;
|
||||
@ -65,6 +69,9 @@ impl Tokens for TokensInput {
|
||||
self.target
|
||||
}
|
||||
|
||||
/// no-op, as `TokensInput` does not use `Rc<RefCelll<T>>`.
|
||||
fn revert(&mut self) {}
|
||||
|
||||
fn set_expr_allowed(&mut self, _: bool) {}
|
||||
|
||||
fn token_context(&self) -> &TokenContexts {
|
||||
@ -81,16 +88,28 @@ impl Tokens for TokensInput {
|
||||
}
|
||||
|
||||
/// Note: Lexer need access to parser's context to lex correctly.
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Debug)]
|
||||
pub struct Capturing<I: Tokens> {
|
||||
inner: I,
|
||||
last_clone_idx: usize,
|
||||
captured: Rc<RefCell<Vec<TokenAndSpan>>>,
|
||||
}
|
||||
|
||||
impl<I: Tokens> Clone for Capturing<I> {
|
||||
fn clone(&self) -> Self {
|
||||
Capturing {
|
||||
last_clone_idx: self.captured.borrow().len(),
|
||||
inner: self.inner.clone(),
|
||||
captured: self.captured.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<I: Tokens> Capturing<I> {
|
||||
pub fn new(input: I) -> Self {
|
||||
Capturing {
|
||||
inner: input,
|
||||
last_clone_idx: 0,
|
||||
captured: Default::default(),
|
||||
}
|
||||
}
|
||||
@ -127,6 +146,12 @@ impl<I: Tokens> Tokens for Capturing<I> {
|
||||
self.inner.target()
|
||||
}
|
||||
|
||||
fn revert(&mut self) {
|
||||
self.inner.revert();
|
||||
let len = self.last_clone_idx;
|
||||
self.captured.borrow_mut().truncate(len);
|
||||
}
|
||||
|
||||
fn set_expr_allowed(&mut self, allow: bool) {
|
||||
self.inner.set_expr_allowed(allow)
|
||||
}
|
||||
@ -171,6 +196,10 @@ impl<I: Tokens> Buffer<I> {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn revert(&mut self) {
|
||||
self.iter.revert()
|
||||
}
|
||||
|
||||
pub fn store(&mut self, token: Token) {
|
||||
debug_assert!(self.next.is_none());
|
||||
debug_assert!(self.cur.is_none());
|
||||
|
||||
@ -473,6 +473,7 @@ impl<'a, I: Tokens> Parser<'a, I> {
|
||||
match res {
|
||||
Ok(Some(res)) if res => {
|
||||
*self = cloned;
|
||||
self.input.revert();
|
||||
self.emit_err = true;
|
||||
Ok(res)
|
||||
}
|
||||
@ -498,6 +499,7 @@ impl<'a, I: Tokens> Parser<'a, I> {
|
||||
match res {
|
||||
Ok(Some(res)) => {
|
||||
*self = cloned;
|
||||
self.input.revert();
|
||||
self.emit_err = true;
|
||||
Some(res)
|
||||
}
|
||||
@ -1014,7 +1016,9 @@ impl<'a, I: Tokens> Parser<'a, I> {
|
||||
|
||||
let mut cloned = self.clone();
|
||||
cloned.emit_err = false;
|
||||
op(&mut cloned)
|
||||
let res = op(&mut cloned);
|
||||
cloned.input.revert();
|
||||
res
|
||||
}
|
||||
|
||||
/// `tsIsUnambiguouslyStartOfFunctionType`
|
||||
@ -2320,7 +2324,10 @@ fn make_decl_declare(mut decl: Decl) -> Decl {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::{test_parser, Syntax};
|
||||
use crate::{
|
||||
lexer::Lexer, test_parser, token::TokenAndSpan, Capturing, JscTarget, Parser, Syntax,
|
||||
TsConfig,
|
||||
};
|
||||
use swc_common::DUMMY_SP;
|
||||
use swc_ecma_ast::*;
|
||||
use testing::assert_eq_ignore_span;
|
||||
@ -2393,4 +2400,34 @@ mod tests {
|
||||
|
||||
assert_eq_ignore_span!(actual, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn issue_726() {
|
||||
crate::with_test_sess(
|
||||
"type Test = (
|
||||
string | number);",
|
||||
|sess, input| {
|
||||
let lexer = Lexer::new(
|
||||
sess,
|
||||
Syntax::Typescript(TsConfig {
|
||||
..Default::default()
|
||||
}),
|
||||
JscTarget::Es2019,
|
||||
input,
|
||||
None,
|
||||
);
|
||||
let lexer = Capturing::new(lexer);
|
||||
|
||||
let mut parser = Parser::new_from(sess, lexer);
|
||||
parser.parse_typescript_module().map_err(|mut e| {
|
||||
e.emit();
|
||||
})?;
|
||||
let tokens: Vec<TokenAndSpan> = parser.input().take();
|
||||
let tokens = tokens.into_iter().map(|t| t.token).collect::<Vec<_>>();
|
||||
assert_eq!(tokens.len(), 9, "Tokens: {:#?}", tokens);
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
@ -375,7 +375,7 @@ impl Debug for Word {
|
||||
#[kind(function(before_expr = "bool", starts_expr = "bool"))]
|
||||
pub enum Keyword {
|
||||
/// Spec says this might be identifier.
|
||||
#[kind(before_expr)]
|
||||
#[kind(before_expr, starts_expr)]
|
||||
Await,
|
||||
|
||||
Break,
|
||||
|
||||
@ -0,0 +1,3 @@
|
||||
async function* main() {
|
||||
yield await 0;
|
||||
}
|
||||
@ -0,0 +1,81 @@
|
||||
{
|
||||
"type": "Module",
|
||||
"span": {
|
||||
"start": 0,
|
||||
"end": 45,
|
||||
"ctxt": 0
|
||||
},
|
||||
"body": [
|
||||
{
|
||||
"type": "FunctionDeclaration",
|
||||
"identifier": {
|
||||
"type": "Identifier",
|
||||
"span": {
|
||||
"start": 16,
|
||||
"end": 20,
|
||||
"ctxt": 0
|
||||
},
|
||||
"value": "main",
|
||||
"typeAnnotation": null,
|
||||
"optional": false
|
||||
},
|
||||
"declare": false,
|
||||
"params": [],
|
||||
"decorators": [],
|
||||
"span": {
|
||||
"start": 0,
|
||||
"end": 45,
|
||||
"ctxt": 0
|
||||
},
|
||||
"body": {
|
||||
"type": "BlockStatement",
|
||||
"span": {
|
||||
"start": 23,
|
||||
"end": 45,
|
||||
"ctxt": 0
|
||||
},
|
||||
"stmts": [
|
||||
{
|
||||
"type": "ExpressionStatement",
|
||||
"span": {
|
||||
"start": 29,
|
||||
"end": 43,
|
||||
"ctxt": 0
|
||||
},
|
||||
"expression": {
|
||||
"type": "YieldExpression",
|
||||
"span": {
|
||||
"start": 29,
|
||||
"end": 42,
|
||||
"ctxt": 0
|
||||
},
|
||||
"argument": {
|
||||
"type": "AwaitExpression",
|
||||
"span": {
|
||||
"start": 35,
|
||||
"end": 42,
|
||||
"ctxt": 0
|
||||
},
|
||||
"argument": {
|
||||
"type": "NumericLiteral",
|
||||
"span": {
|
||||
"start": 41,
|
||||
"end": 42,
|
||||
"ctxt": 0
|
||||
},
|
||||
"value": 0.0
|
||||
}
|
||||
},
|
||||
"delegate": false
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"generator": true,
|
||||
"async": true,
|
||||
"typeParameters": null,
|
||||
"returnType": null
|
||||
}
|
||||
],
|
||||
"interpreter": null
|
||||
}
|
||||
@ -0,0 +1,3 @@
|
||||
async function* main() {
|
||||
yield await 0;
|
||||
}
|
||||
@ -0,0 +1,81 @@
|
||||
{
|
||||
"type": "Module",
|
||||
"span": {
|
||||
"start": 0,
|
||||
"end": 45,
|
||||
"ctxt": 0
|
||||
},
|
||||
"body": [
|
||||
{
|
||||
"type": "FunctionDeclaration",
|
||||
"identifier": {
|
||||
"type": "Identifier",
|
||||
"span": {
|
||||
"start": 16,
|
||||
"end": 20,
|
||||
"ctxt": 0
|
||||
},
|
||||
"value": "main",
|
||||
"typeAnnotation": null,
|
||||
"optional": false
|
||||
},
|
||||
"declare": false,
|
||||
"params": [],
|
||||
"decorators": [],
|
||||
"span": {
|
||||
"start": 0,
|
||||
"end": 45,
|
||||
"ctxt": 0
|
||||
},
|
||||
"body": {
|
||||
"type": "BlockStatement",
|
||||
"span": {
|
||||
"start": 23,
|
||||
"end": 45,
|
||||
"ctxt": 0
|
||||
},
|
||||
"stmts": [
|
||||
{
|
||||
"type": "ExpressionStatement",
|
||||
"span": {
|
||||
"start": 29,
|
||||
"end": 43,
|
||||
"ctxt": 0
|
||||
},
|
||||
"expression": {
|
||||
"type": "YieldExpression",
|
||||
"span": {
|
||||
"start": 29,
|
||||
"end": 42,
|
||||
"ctxt": 0
|
||||
},
|
||||
"argument": {
|
||||
"type": "AwaitExpression",
|
||||
"span": {
|
||||
"start": 35,
|
||||
"end": 42,
|
||||
"ctxt": 0
|
||||
},
|
||||
"argument": {
|
||||
"type": "NumericLiteral",
|
||||
"span": {
|
||||
"start": 41,
|
||||
"end": 42,
|
||||
"ctxt": 0
|
||||
},
|
||||
"value": 0.0
|
||||
}
|
||||
},
|
||||
"delegate": false
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"generator": true,
|
||||
"async": true,
|
||||
"typeParameters": null,
|
||||
"returnType": null
|
||||
}
|
||||
],
|
||||
"interpreter": null
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user