microsoft/pyright
1
1
Fork 0
mirror of https://github.com/microsoft/pyright.git synced 2024-09-11 16:06:39 +03:00
Code Issues Projects Releases Wiki Activity

Started to put in place support for f-strings.

Browse Source
This commit is contained in:
Eric Traut 2019-06-09 22:48:08 -07:00
parent 00e063e213
commit 644163a2ed
9 changed files with 139 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
server/src/analyzer/analyzerNodeInfo.ts
View File

@ -11,7 +11,7 @@
*/
import { NameBindings } from '../parser/nameBindings';
import { ParseNode, StringNode } from '../parser/parseNodes';
import { ParseNode, StringListNode } from '../parser/parseNodes';
import { ImportResult } from './importResult';
import { TypeSourceId } from './inferredType';
import { Scope } from './scope';
@ -147,12 +147,12 @@ export class AnalyzerNodeInfo {
return analyzerNode._typeSourceId;
}
static setIgnoreTypeAnnotation(node: StringNode) {
static setIgnoreTypeAnnotation(node: StringListNode) {
const analyzerNode = node as AnalyzerNodeInfo;
analyzerNode._ignoreTypeAnnotation = true;
}
static getIgnoreTypeAnnotation(node: StringNode) {
static getIgnoreTypeAnnotation(node: StringListNode) {
const analyzerNode = node as AnalyzerNodeInfo;
return !!analyzerNode._ignoreTypeAnnotation;
}

5
server/src/analyzer/completionProvider.ts
View File

@ -18,8 +18,7 @@ import { convertPositionToOffset } from '../common/positionUtils';
import { ErrorExpressionCategory, ErrorExpressionNode, ExpressionNode,
ImportFromAsNode, ImportFromNode, MemberAccessExpressionNode,
ModuleNameNode, ModuleNode, NameNode, ParseNode,
StringNode,
SuiteNode } from '../parser/parseNodes';
StringListNode, SuiteNode } from '../parser/parseNodes';
import { ParseResults } from '../parser/parser';
import { ImportMap } from './analyzerFileInfo';
import { AnalyzerNodeInfo } from './analyzerNodeInfo';
@ -130,7 +129,7 @@ export class CompletionProvider {
let curNode = errorNode || node;
while (true) {
// Don't offer completions inside of a string node.
if (curNode instanceof StringNode) {
if (curNode instanceof StringListNode) {
return undefined;
}

22
server/src/analyzer/expressionEvaluator.ts
View File

@ -24,7 +24,7 @@ import { ArgumentCategory, AssignmentNode, AwaitExpressionNode,
IndexItemsNode, LambdaNode, ListComprehensionForNode, ListComprehensionIfNode,
ListComprehensionNode, ListNode, MemberAccessExpressionNode, NameNode, NumberNode,
ParameterCategory, ParseNode, SetNode, SliceExpressionNode, StatementListNode,
StringNode, TernaryExpressionNode, TupleExpressionNode,
StringListNode, TernaryExpressionNode, TupleExpressionNode,
TypeAnnotationExpressionNode, UnaryExpressionNode, UnpackExpressionNode,
YieldExpressionNode, YieldFromExpressionNode } from '../parser/parseNodes';
import { KeywordToken, KeywordType, OperatorType, StringTokenFlags,
@ -560,7 +560,7 @@ export class ExpressionEvaluator {
} else if (node instanceof ConstantNode) {
this._reportUsageErrorForReadOnly(node, usage);
typeResult = this._getTypeFromConstantExpression(node);
} else if (node instanceof StringNode) {
} else if (node instanceof StringListNode) {
this._reportUsageErrorForReadOnly(node, usage);
if (node.typeAnnotation && !AnalyzerNodeInfo.getIgnoreTypeAnnotation(node)) {
let typeResult: TypeResult = { node, type: UnknownType.create() };
@ -574,7 +574,7 @@ export class ExpressionEvaluator {
return typeResult;
}
let isBytes = (node.tokens[0].flags & StringTokenFlags.Bytes) !== 0;
const isBytes = (node.strings[0].token.flags & StringTokenFlags.Bytes) !== 0;
typeResult = { node, type: this._cloneBuiltinTypeWithLiteral(
isBytes ? 'bytes' : 'str', node.getValue()) };
} else if (node instanceof NumberNode) {
@ -1752,7 +1752,7 @@ export class ExpressionEvaluator {
}
let firstArg = argList[0];
if (firstArg.valueExpression instanceof StringNode) {
if (firstArg.valueExpression instanceof StringListNode) {
typeVarName = firstArg.valueExpression.getValue();
} else {
this._addError('Expected name of type var as first parameter',
@ -1860,7 +1860,7 @@ export class ExpressionEvaluator {
if (nameArg.argumentCategory !== ArgumentCategory.Simple) {
this._addError('Expected enum class name as first parameter',
argList[0].valueExpression || errorNode);
} else if (nameArg.valueExpression instanceof StringNode) {
} else if (nameArg.valueExpression instanceof StringListNode) {
className = nameArg.valueExpression.getValue();
}
}
@ -1891,7 +1891,7 @@ export class ExpressionEvaluator {
} else {
const entriesArg = argList[1];
if (entriesArg.argumentCategory !== ArgumentCategory.Simple ||
!(entriesArg.valueExpression instanceof StringNode)) {
!(entriesArg.valueExpression instanceof StringListNode)) {
this._addError('Expected enum item string as second parameter', errorNode);
} else {
@ -1940,7 +1940,7 @@ export class ExpressionEvaluator {
if (nameArg.argumentCategory !== ArgumentCategory.Simple) {
this._addError('Expected named tuple class name as first parameter',
argList[0].valueExpression || errorNode);
} else if (nameArg.valueExpression instanceof StringNode) {
} else if (nameArg.valueExpression instanceof StringListNode) {
className = nameArg.valueExpression.getValue();
}
}
@ -1994,7 +1994,7 @@ export class ExpressionEvaluator {
if (entriesArg.argumentCategory !== ArgumentCategory.Simple) {
addGenericGetAttribute = true;
} else {
if (!includesTypes && entriesArg.valueExpression instanceof StringNode) {
if (!includesTypes && entriesArg.valueExpression instanceof StringListNode) {
let entries = entriesArg.valueExpression.getValue().split(' ');
entries.forEach(entryName => {
entryName = entryName.trim();
@ -2052,7 +2052,7 @@ export class ExpressionEvaluator {
entryType = UnknownType.create();
}
if (entryNameNode instanceof StringNode) {
if (entryNameNode instanceof StringListNode) {
entryName = entryNameNode.getValue();
if (!entryName) {
this._addError(
@ -2939,13 +2939,13 @@ export class ExpressionEvaluator {
for (let item of node.items.items) {
let type: Type | undefined;
if (item instanceof StringNode) {
if (item instanceof StringListNode) {
// Note that the contents of the string should not be treated
// as a type annotation, as they normally are for quoted type
// arguments.
AnalyzerNodeInfo.setIgnoreTypeAnnotation(item);
const isBytes = (item.tokens[0].flags & StringTokenFlags.Bytes) !== 0;
const isBytes = (item.strings[0].token.flags & StringTokenFlags.Bytes) !== 0;
if (isBytes) {
type = this._cloneBuiltinTypeWithLiteral('bytes', item.getValue());
} else {

4
server/src/analyzer/expressionUtils.ts
View File

@ -10,7 +10,7 @@
import { ExecutionEnvironment } from '../common/configOptions';
import { BinaryExpressionNode, ConstantNode, ExpressionNode, IndexExpressionNode,
MemberAccessExpressionNode, NameNode, NumberNode, StringNode,
MemberAccessExpressionNode, NameNode, NumberNode, StringListNode,
TupleExpressionNode } from '../parser/parseNodes';
import { KeywordType, OperatorType } from '../parser/tokenizerTypes';
@ -40,7 +40,7 @@ export class ExpressionUtils {
return this._evaluateNumericBinaryOperation(node.operator,
execEnv.pythonVersion / 256, node.rightExpression.token.value);
} else if (this._isSysPlatformInfoExpression(node.leftExpression) &&
node.rightExpression instanceof StringNode) {
node.rightExpression instanceof StringListNode) {
// Handle the special case of "sys.platform != 'X'"
let comparisonPlatform = node.rightExpression.getValue();
if (execEnv.pythonPlatform !== undefined) {

18
server/src/analyzer/parseTreeWalker.ts
View File

@ -13,12 +13,12 @@ import { ArgumentNode, AssertNode, AssignmentNode, AugmentedAssignmentExpression
AwaitExpressionNode, BinaryExpressionNode, BreakNode, CallExpressionNode, ClassNode,
ConstantNode, ContinueNode, DecoratorNode, DelNode, DictionaryExpandEntryNode,
DictionaryKeyEntryNode, DictionaryNode, EllipsisNode, ErrorExpressionNode,
ExceptNode, ForNode, FunctionNode, GlobalNode, IfNode, ImportAsNode,
ExceptNode, FormatStringNode, ForNode, FunctionNode, GlobalNode, IfNode, ImportAsNode,
ImportFromAsNode, ImportFromNode, ImportNode, IndexExpressionNode, IndexItemsNode,
LambdaNode, ListComprehensionForNode, ListComprehensionIfNode, ListComprehensionNode,
ListNode, MemberAccessExpressionNode, ModuleNameNode, ModuleNode, NameNode, NonlocalNode,
NumberNode, ParameterNode, ParseNode, ParseNodeType, PassNode, RaiseNode,
ReturnNode, SetNode, SliceExpressionNode, StatementListNode, StringNode,
ReturnNode, SetNode, SliceExpressionNode, StatementListNode, StringListNode, StringNode,
SuiteNode, TernaryExpressionNode, TryNode, TupleExpressionNode,
TypeAnnotationExpressionNode, UnaryExpressionNode, UnpackExpressionNode, WhileNode,
WithItemNode, WithNode, YieldExpressionNode, YieldFromExpressionNode } from '../parser/parseNodes';
@ -132,6 +132,9 @@ export class ParseTreeWalker {
case ParseNodeType.For:
return this.visitFor(node as ForNode);
case ParseNodeType.FormatString:
return this.visitFormatString(node as FormatStringNode);
case ParseNodeType.Function:
return this.visitFunction(node as FunctionNode);
@ -195,6 +198,9 @@ export class ParseTreeWalker {
case ParseNodeType.String:
return this.visitString(node as StringNode);
case ParseNodeType.StringList:
return this.visitStringList(node as StringListNode);
case ParseNodeType.Suite:
return this.visitSuite(node as SuiteNode);
@ -348,6 +354,10 @@ export class ParseTreeWalker {
return true;
}
visitFormatString(node: FormatStringNode) {
return true;
}
visitFunction(node: FunctionNode) {
return true;
}
@ -432,6 +442,10 @@ export class ParseTreeWalker {
return true;
}
visitStringList(node: StringListNode) {
return true;
}
visitSuite(node: SuiteNode) {
return true;
}

34
server/src/analyzer/semanticAnalyzer.ts
View File

@ -24,10 +24,10 @@ import { PythonVersion } from '../common/pythonVersion';
import { TextRange } from '../common/textRange';
import { AwaitExpressionNode, ClassNode, ErrorExpressionNode,
ExpressionNode, FunctionNode, GlobalNode, IfNode, LambdaNode, ModuleNameNode,
ModuleNode, NonlocalNode, RaiseNode, StatementListNode, StatementNode, StringNode,
SuiteNode, TryNode, TypeAnnotationExpressionNode, WhileNode, YieldExpressionNode,
YieldFromExpressionNode } from '../parser/parseNodes';
import { StringTokenFlags } from '../parser/tokenizerTypes';
ModuleNode, NonlocalNode, RaiseNode, StatementListNode, StatementNode,
StringListNode, SuiteNode, TryNode, TypeAnnotationExpressionNode, WhileNode,
YieldExpressionNode, YieldFromExpressionNode } from '../parser/parseNodes';
import { StringToken, StringTokenFlags } from '../parser/tokenizerTypes';
import { ScopeUtils } from '../scopeUtils';
import { AnalyzerFileInfo } from './analyzerFileInfo';
import { AnalyzerNodeInfo } from './analyzerNodeInfo';
@ -381,8 +381,9 @@ export abstract class SemanticAnalyzer extends ParseTreeWalker {
return true;
}
visitString(node: StringNode): boolean {
for (let stringToken of node.tokens) {
visitStringList(node: StringListNode): boolean {
for (let string of node.strings) {
const stringToken = string.token;
if (stringToken.flags & StringTokenFlags.Unterminated) {
this._addError('String literal is unterminated', stringToken);
}
@ -391,6 +392,20 @@ export abstract class SemanticAnalyzer extends ParseTreeWalker {
this._addError('Non-ASCII character not allowed in bytes string literal', stringToken);
}
if (stringToken.flags & StringTokenFlags.Format) {
if (this._fileInfo.executionEnvironment.pythonVersion < PythonVersion.V36) {
this._addError('Format string literals (f-strings) require Python 3.6 or newer', stringToken);
}
if (stringToken.flags & StringTokenFlags.Bytes) {
this._addError('Format string literals (f-strings) cannot be binary', stringToken);
}
if (stringToken.flags & StringTokenFlags.Unicode) {
this._addError('Format string literals (f-strings) cannot be unicode', stringToken);
}
}
if (stringToken.flags & StringTokenFlags.UnrecognizedEscape) {
if (stringToken.invalidEscapeOffsets) {
stringToken.invalidEscapeOffsets.forEach(offset => {
@ -491,12 +506,13 @@ export abstract class SemanticAnalyzer extends ParseTreeWalker {
// If the first statement in the suite isn't a StringNode,
// assume there is no docString.
const statementList = statemetns[0] as StatementListNode;
if (statementList.statements.length === 0 || !(statementList.statements[0] instanceof StringNode)) {
if (statementList.statements.length === 0 ||
!(statementList.statements[0] instanceof StringListNode)) {
return undefined;
}
const docStringNode = statementList.statements[0] as StringNode;
const docStringToken = docStringNode.tokens[0];
const docStringNode = statementList.statements[0] as StringListNode;
const docStringToken = docStringNode.strings[0].token;
// Ignore f-strings.
if ((docStringToken.flags & StringTokenFlags.Format) !== 0) {

5
server/src/analyzer/typeAnalyzer.ts
View File

@ -22,7 +22,7 @@ import { AssertNode, AssignmentNode, AugmentedAssignmentExpressionNode,
FunctionNode, IfNode, ImportAsNode, ImportFromNode, IndexExpressionNode,
LambdaNode, ListComprehensionForNode, ListComprehensionNode, ListNode,
MemberAccessExpressionNode, ModuleNode, NameNode, ParameterCategory, ParseNode,
RaiseNode, ReturnNode, SliceExpressionNode, StringNode, SuiteNode,
RaiseNode, ReturnNode, SliceExpressionNode, StringListNode, SuiteNode,
TernaryExpressionNode, TryNode, TupleExpressionNode, TypeAnnotationExpressionNode,
UnaryExpressionNode, UnpackExpressionNode, WhileNode, WithNode, YieldExpressionNode,
YieldFromExpressionNode } from '../parser/parseNodes';
@ -1072,7 +1072,7 @@ export class TypeAnalyzer extends ParseTreeWalker {
return true;
}
visitString(node: StringNode): boolean {
visitStringList(node: StringListNode): boolean {
if (node.typeAnnotation) {
// Should we ignore this type annotation?
if (AnalyzerNodeInfo.getIgnoreTypeAnnotation(node)) {
@ -1081,6 +1081,7 @@ export class TypeAnalyzer extends ParseTreeWalker {
this._getTypeOfExpression(node.typeAnnotation, true, true);
}
return true;
}

53
server/src/parser/parseNodes.ts
View File

@ -45,6 +45,7 @@ export enum ParseNodeType {
IndexItems,
Except,
For,
FormatString,
Function,
Global,
Lambda,
@ -65,6 +66,7 @@ export enum ParseNodeType {
Set,
Slice,
StatementList,
StringList,
String,
Suite,
Ternary,
@ -762,24 +764,63 @@ export class NumberNode extends ExpressionNode {
export class StringNode extends ExpressionNode {
readonly nodeType = ParseNodeType.String;
tokens: StringToken[];
token: StringToken;
constructor(token: StringToken) {
super(token);
this.token = token;
}
getChildren(): RecursiveParseNodeArray {
return undefined;
}
getValue(): string {
return this.token.value;
}
}
export class FormatStringNode extends ExpressionNode {
readonly nodeType = ParseNodeType.String;
token: StringToken;
constructor(token: StringToken) {
super(token);
this.token = token;
}
getChildren(): RecursiveParseNodeArray {
return undefined;
}
getValue(): string {
return this.token.value;
}
}
export class StringListNode extends ExpressionNode {
readonly nodeType = ParseNodeType.StringList;
strings: (StringNode | FormatStringNode)[];
// If strings are found within the context of
// a type annotation, they are further parsed
// into an expression.
typeAnnotation?: ExpressionNode;
constructor(tokens: StringToken[]) {
super(tokens[0]);
this.tokens = tokens;
constructor(strings: (StringNode | FormatStringNode)[]) {
super(strings[0]);
this.strings = strings;
if (strings.length > 1) {
this.extend(strings[strings.length - 1]);
}
}
getChildren(): RecursiveParseNodeArray {
return this.typeAnnotation ? [this.typeAnnotation] : undefined;
return this.strings;
}
getValue(): string {
return this.tokens.map(t => t.value).join('');
return this.strings.map(t => t.getValue()).join('');
}
}

49
server/src/parser/parser.ts
View File

@ -28,16 +28,16 @@ import { ArgumentCategory, ArgumentNode, AssertNode,
ConstantNode, ContinueNode, DecoratorNode, DelNode,
DictionaryEntryNode, DictionaryExpandEntryNode, DictionaryKeyEntryNode,
DictionaryNode, EllipsisNode, ErrorExpressionCategory, ErrorExpressionNode,
ExceptNode, ExpressionNode, ForNode, FunctionNode, GlobalNode, IfNode,
ImportAsNode, ImportFromAsNode, ImportFromNode, ImportNode,
ExceptNode, ExpressionNode, FormatStringNode, ForNode, FunctionNode, GlobalNode,
IfNode, ImportAsNode, ImportFromAsNode, ImportFromNode, ImportNode,
IndexExpressionNode, IndexItemsNode, LambdaNode, ListComprehensionForNode,
ListComprehensionIfNode, ListComprehensionIterNode, ListComprehensionNode, ListNode,
MemberAccessExpressionNode, ModuleNameNode, ModuleNode, NameNode, NonlocalNode, NumberNode,
ParameterCategory, ParameterNode, ParseNode, PassNode, RaiseNode, ReturnNode,
SetNode, SliceExpressionNode, StatementListNode, StatementNode,
StringNode, SuiteNode, TernaryExpressionNode, TryNode, TupleExpressionNode,
TypeAnnotationExpressionNode, UnaryExpressionNode, UnpackExpressionNode,
WhileNode, WithItemNode, WithNode, YieldExpressionNode,
StringListNode, StringNode, SuiteNode, TernaryExpressionNode, TryNode,
TupleExpressionNode, TypeAnnotationExpressionNode, UnaryExpressionNode,
UnpackExpressionNode, WhileNode, WithItemNode, WithNode, YieldExpressionNode,
YieldFromExpressionNode } from './parseNodes';
import { Tokenizer, TokenizerOutput } from './tokenizer';
import { DedentToken, IdentifierToken, KeywordToken, KeywordType,
@ -1848,7 +1848,7 @@ export class Parser {
}
if (nextToken.type === TokenType.String) {
return this._parseString();
return this._parseStringList();
}
if (nextToken.type === TokenType.OpenParenthesis) {
@ -2265,7 +2265,7 @@ export class Parser {
const stringToken = new StringToken(tokenOffset,
typeString.length, StringTokenFlags.None, typeString, 0,
undefined, undefined);
const stringNode = new StringNode([stringToken]);
const stringNode = new StringListNode([new StringNode(stringToken)]);
let parser = new Parser();
let parseResults = parser.parseTextExpression(this._fileContents!,
@ -2284,35 +2284,46 @@ export class Parser {
return stringNode;
}
private _parseString(): StringNode {
let stringTokenList: StringToken[] = [];
private _parseFormatString(token: StringToken): FormatStringNode {
// TODO - need to implement
return new FormatStringNode(token);
}
private _parseStringList(): StringListNode {
const stringList: (StringNode | FormatStringNode)[] = [];
while (this._peekTokenType() === TokenType.String) {
stringTokenList.push(this._getNextToken() as StringToken);
const stringToken = this._getNextToken() as StringToken;
if (stringToken.flags & StringTokenFlags.Format) {
stringList.push(this._parseFormatString(stringToken));
} else {
stringList.push(new StringNode(stringToken));
}
}
const stringNode = new StringNode(stringTokenList);
const stringNode = new StringListNode(stringList);
// If we're parsing a type annotation, parse the contents of the string.
if (this._isParsingTypeAnnotation) {
// Don't allow multiple strings because we have no way of reporting
// parse errors that span strings.
if (stringNode.tokens.length > 1) {
if (stringNode.strings.length > 1) {
this._addError('Type hints cannot span multiple string literals', stringNode);
} else if (stringNode.tokens[0].flags & StringTokenFlags.Triplicate) {
} else if (stringNode.strings[0].token.flags & StringTokenFlags.Triplicate) {
this._addError('Type hints cannot use triple quotes', stringNode);
} else if (stringNode.tokens[0].flags & StringTokenFlags.Format) {
this._addError('Type hints cannot use format string literals', stringNode);
} else if (stringNode.strings[0].token.flags & StringTokenFlags.Format) {
this._addError('Type hints cannot use format string literals (f-strings)', stringNode);
} else {
const stringValue = stringNode.tokens[0].value;
const tokenOffset = stringNode.tokens[0].start;
const stringToken = stringNode.strings[0].token;
const stringValue = stringToken.value;
const tokenOffset = stringToken.start;
// Add one character to the prefix to also include the quote.
const prefixLength = stringNode.tokens[0].prefixLength + 1;
const prefixLength = stringToken.prefixLength + 1;
// Don't allow escape characters because we have no way of mapping
// error ranges back to the escaped text.
if (stringNode.tokens[0].value.length !== stringNode.tokens[0].length - prefixLength - 1) {
if (stringToken.value.length !== stringToken.length - prefixLength - 1) {
this._addError('Type hints cannot contain escape characters', stringNode);
} else {
let parser = new Parser();