Changed the tokenizer so it is bug-for-bug compatible with the CPython tokenizer in versions 3.10 and newer in the case where a backslash (continuation character) is located by itself on a line. This addresses #7799.

This commit is contained in:
Eric Traut 2024-04-29 16:00:54 -07:00
parent 1aed0e3261
commit fcf5b372be
4 changed files with 35 additions and 6 deletions

View File

@ -494,13 +494,25 @@ export class Tokenizer {
} else {
this._cs.advance(2);
}
this._addLineRange();
return true;
} else if (this._cs.nextChar === Char.LineFeed) {
this._cs.advance(2);
this._addLineRange();
if (this._tokens.length > 0 && this._tokens[this._tokens.length - 1].type === TokenType.NewLine) {
this._readIndentationAfterNewLine();
}
return true;
}
if (this._cs.nextChar === Char.LineFeed) {
this._cs.advance(2);
this._addLineRange();
if (this._tokens.length > 0 && this._tokens[this._tokens.length - 1].type === TokenType.NewLine) {
this._readIndentationAfterNewLine();
}
return true;
}
return this._handleInvalid();
}

View File

@ -25,14 +25,20 @@ test('Empty', () => {
assert.equal(parserOutput.parseTree.statements.length, 0);
});
test('Sample1', () => {
test('Parser1', () => {
const diagSink = new DiagnosticSink();
const parserOutput = TestUtils.parseSampleFile('sample1.py', diagSink).parserOutput;
const parserOutput = TestUtils.parseSampleFile('parser1.py', diagSink).parserOutput;
assert.equal(diagSink.fetchAndClear().length, 0);
assert.equal(parserOutput.parseTree.statements.length, 4);
});
test('Parser2', () => {
const diagSink = new DiagnosticSink();
TestUtils.parseSampleFile('parser2.py', diagSink);
assert.strictEqual(diagSink.getErrors().length, 0);
});
test('FStringEmptyTuple', () => {
assert.doesNotThrow(() => {
const diagSink = new DiagnosticSink();

View File

@ -0,0 +1,11 @@
class A:
\
pass
class B:
\
pass
class C:
\
pass