From 2cf7f61fc6825c2868082ab9e7ab2112bf8a3bce Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Fri, 26 Jun 2020 19:38:22 -0700 Subject: [PATCH] Added code to better handle the obsolete "<>" operator from Python 2 - including a better error message and better parse recovery. --- server/src/analyzer/parseTreeUtils.ts | 1 + server/src/localization/localize.ts | 1 + server/src/localization/package.nls.en-us.json | 1 + server/src/parser/parser.ts | 4 ++++ server/src/parser/tokenizer.ts | 4 ++++ server/src/parser/tokenizerTypes.ts | 2 ++ server/src/tests/checker.test.ts | 6 ++++++ server/src/tests/samples/operators5.py | 5 +++++ server/src/tests/tokenizer.test.ts | 5 +++-- 9 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 server/src/tests/samples/operators5.py diff --git a/server/src/analyzer/parseTreeUtils.ts b/server/src/analyzer/parseTreeUtils.ts index 2ec1a7ce2..f87b2a825 100644 --- a/server/src/analyzer/parseTreeUtils.ts +++ b/server/src/analyzer/parseTreeUtils.ts @@ -404,6 +404,7 @@ export function printOperator(operator: OperatorType): string { [OperatorType.GreaterThanOrEqual]: '>=', [OperatorType.LeftShift]: '<<', [OperatorType.LeftShiftEqual]: '<<=', + [OperatorType.LessOrGreaterThan]: '<>', [OperatorType.LessThan]: '<', [OperatorType.LessThanOrEqual]: '<=', [OperatorType.MatrixMultiply]: '@', diff --git a/server/src/localization/localize.ts b/server/src/localization/localize.ts index 1abef6d3e..dad36688c 100644 --- a/server/src/localization/localize.ts +++ b/server/src/localization/localize.ts @@ -398,6 +398,7 @@ export namespace Localizer { new ParameterizedString<{ name: string }>(getRawString('Diagnostic.obscuredParameterDeclaration')); export const obscuredVariableDeclaration = () => new ParameterizedString<{ name: string }>(getRawString('Diagnostic.obscuredVariableDeclaration')); + export const operatorLessOrGreaterDeprecated = () => getRawString('Diagnostic.operatorLessOrGreaterDeprecated'); export const optionalExtraArgs = () => getRawString('Diagnostic.optionalExtraArgs'); export const paramAfterKwargsParam = () => getRawString('Diagnostic.paramAfterKwargsParam'); export const paramAlreadyAssigned = () => diff --git a/server/src/localization/package.nls.en-us.json b/server/src/localization/package.nls.en-us.json index ec7b7d387..7f9756ac6 100644 --- a/server/src/localization/package.nls.en-us.json +++ b/server/src/localization/package.nls.en-us.json @@ -182,6 +182,7 @@ "obscuredFunctionDeclaration": "Function declaration \"{name}\" is obscured by a declaration of the same name", "obscuredParameterDeclaration": "Parameter declaration \"{name}\" is obscured by a declaration of the same name", "obscuredVariableDeclaration": "Declaration \"{name}\" is obscured by a declaration of the same name", + "operatorLessOrGreaterDeprecated": "Operator \"<>\" is not supported in Python 3; use \"!=\" instead", "optionalExtraArgs": "Expected one type argument after \"Optional\"", "paramAfterKwargsParam": "Parameter cannot follow \"**\" parameter", "paramAlreadyAssigned": "Parameter \"{name}\" is already assigned", diff --git a/server/src/parser/parser.ts b/server/src/parser/parser.ts index 2e9a54f6d..52398c171 100644 --- a/server/src/parser/parser.ts +++ b/server/src/parser/parser.ts @@ -1805,6 +1805,10 @@ export class Parser { if (Tokenizer.isOperatorComparison(this._peekOperatorType())) { comparisonOperator = this._peekOperatorType(); + if (comparisonOperator === OperatorType.LessOrGreaterThan) { + this._addError(Localizer.Diagnostic.operatorLessOrGreaterDeprecated(), peekToken); + comparisonOperator = OperatorType.NotEquals; + } this._getNextToken(); } else if (this._consumeTokenIfKeyword(KeywordType.In)) { comparisonOperator = OperatorType.In; diff --git a/server/src/parser/tokenizer.ts b/server/src/parser/tokenizer.ts index f8615f1e5..c5ecd6d2b 100644 --- a/server/src/parser/tokenizer.ts +++ b/server/src/parser/tokenizer.ts @@ -94,6 +94,7 @@ const _operatorInfo: { [key: number]: OperatorFlags } = { [OperatorType.GreaterThanOrEqual]: OperatorFlags.Binary | OperatorFlags.Comparison, [OperatorType.LeftShift]: OperatorFlags.Binary, [OperatorType.LeftShiftEqual]: OperatorFlags.Assignment, + [OperatorType.LessOrGreaterThan]: OperatorFlags.Binary | OperatorFlags.Comparison | OperatorFlags.Deprecated, [OperatorType.LessThan]: OperatorFlags.Binary | OperatorFlags.Comparison, [OperatorType.LessThanOrEqual]: OperatorFlags.Binary | OperatorFlags.Comparison, [OperatorType.MatrixMultiply]: OperatorFlags.Binary, @@ -869,6 +870,9 @@ export class Tokenizer { if (nextChar === Char.Less) { length = this._cs.lookAhead(2) === Char.Equal ? 3 : 2; operatorType = length === 3 ? OperatorType.LeftShiftEqual : OperatorType.LeftShift; + } else if (nextChar === Char.Greater) { + length = 2; + operatorType = OperatorType.LessOrGreaterThan; } else { length = nextChar === Char.Equal ? 2 : 1; operatorType = length === 2 ? OperatorType.LessThanOrEqual : OperatorType.LessThan; diff --git a/server/src/parser/tokenizerTypes.ts b/server/src/parser/tokenizerTypes.ts index d68052f78..7513535b9 100644 --- a/server/src/parser/tokenizerTypes.ts +++ b/server/src/parser/tokenizerTypes.ts @@ -68,6 +68,7 @@ export const enum OperatorType { GreaterThanOrEqual, LeftShift, LeftShiftEqual, + LessOrGreaterThan, LessThan, LessThanOrEqual, MatrixMultiply, @@ -101,6 +102,7 @@ export const enum OperatorFlags { Binary = 1 << 1, Assignment = 1 << 2, Comparison = 1 << 3, + Deprecated = 1 << 4, } export const enum KeywordType { diff --git a/server/src/tests/checker.test.ts b/server/src/tests/checker.test.ts index c8a4cb107..37cefcf8e 100644 --- a/server/src/tests/checker.test.ts +++ b/server/src/tests/checker.test.ts @@ -536,6 +536,12 @@ test('Operators4', () => { validateResults(analysisResults, 0); }); +test('Operators5', () => { + const analysisResults = TestUtils.typeAnalyzeSampleFiles(['operators5.py']); + + validateResults(analysisResults, 1); +}); + test('Optional1', () => { const configOptions = new ConfigOptions('.'); diff --git a/server/src/tests/samples/operators5.py b/server/src/tests/samples/operators5.py new file mode 100644 index 000000000..2e8952064 --- /dev/null +++ b/server/src/tests/samples/operators5.py @@ -0,0 +1,5 @@ +# This sample tests the parsing of the deprecated <> operator. + +# This should generate a single error, not a cascade of errors. +if 3 <> 5: + print("OK") diff --git a/server/src/tests/tokenizer.test.ts b/server/src/tests/tokenizer.test.ts index 550650656..b18a65371 100644 --- a/server/src/tests/tokenizer.test.ts +++ b/server/src/tests/tokenizer.test.ts @@ -1259,9 +1259,9 @@ test('Operators', () => { '* ** / // /= //=' + '*= += -= %= **= ' + '& &= | |= ^ ^= ' + - ':='; + ':= <>'; const results = new Tokenizer().tokenize(text); - const lengths = [1, 2, 3, 2, 2, 1, 2, 3, 2, 2, 1, 1, 1, 1, 1, 2, 1, 2, 2, 3, 2, 2, 2, 2, 3, 1, 2, 1, 2, 1, 2, 2]; + const lengths = [1, 2, 3, 2, 2, 1, 2, 3, 2, 2, 1, 1, 1, 1, 1, 2, 1, 2, 2, 3, 2, 2, 2, 2, 3, 1, 2, 1, 2, 1, 2, 2, 2]; const operatorTypes = [ OperatorType.LessThan, OperatorType.LeftShift, @@ -1295,6 +1295,7 @@ test('Operators', () => { OperatorType.BitwiseXor, OperatorType.BitwiseXorEqual, OperatorType.Walrus, + OperatorType.LessOrGreaterThan, ]; assert.equal(results.tokens.count - _implicitTokenCount, lengths.length); assert.equal(results.tokens.count - _implicitTokenCount, operatorTypes.length);