From 9e231b129242f2f24a3adbdadddfd5aed107b90e Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Wed, 2 Sep 2020 19:01:49 -0700 Subject: [PATCH] Extensibility updates in threading and fourslash, add completion context (#995) --- docs/configuration.md | 2 +- .../src/analyzer/backgroundAnalysisProgram.ts | 9 + .../pyright-internal/src/analyzer/program.ts | 27 +- .../pyright-internal/src/analyzer/service.ts | 45 ++- .../src/analyzer/sourceFile.ts | 4 +- .../src/backgroundAnalysisBase.ts | 350 ++++++++++-------- .../src/languageServerBase.ts | 101 +++-- .../src/languageService/completionProvider.ts | 116 +++++- ...ntext.UnknownMemberOnInstance.fourslash.ts | 17 + ....UnknownStaticFunctionOnClass.fourslash.ts | 57 +++ ...s.moduleContext.libCodeNoStub.fourslash.ts | 30 ++ .../src/tests/fourslash/fourslash.ts | 5 + .../src/tests/harness/fourslash/runner.ts | 21 +- .../src/tests/harness/fourslash/testState.ts | 122 +++--- packages/vscode-pyright/package.json | 2 +- 15 files changed, 618 insertions(+), 290 deletions(-) create mode 100644 packages/pyright-internal/src/tests/fourslash/completions.moduleContext.UnknownMemberOnInstance.fourslash.ts create mode 100644 packages/pyright-internal/src/tests/fourslash/completions.moduleContext.UnknownStaticFunctionOnClass.fourslash.ts create mode 100644 packages/pyright-internal/src/tests/fourslash/completions.moduleContext.libCodeNoStub.fourslash.ts diff --git a/docs/configuration.md b/docs/configuration.md index cf427e902..fc2eb51e0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -4,7 +4,7 @@ Pyright offers flexible configuration options specified in a JSON-formatted text Relative paths specified within the config file are relative to the config file’s location. Paths with shell variables (including `~`) are not supported. -## Master Pyright Config Options +## Main Pyright Config Options **include** [array of paths, optional]: Paths of directories or files that should be included. If no paths are specified, pyright defaults to the directory that contains the config file. Paths may contain wildcard characters ** (a directory or multiple levels of directories), * (a sequence of zero or more characters), or ? (a single character). If no include paths are specified, the root path for the workspace is assumed. diff --git a/packages/pyright-internal/src/analyzer/backgroundAnalysisProgram.ts b/packages/pyright-internal/src/analyzer/backgroundAnalysisProgram.ts index 996c31030..6c53daf5f 100644 --- a/packages/pyright-internal/src/analyzer/backgroundAnalysisProgram.ts +++ b/packages/pyright-internal/src/analyzer/backgroundAnalysisProgram.ts @@ -264,3 +264,12 @@ export class BackgroundAnalysisProgram { } } } + +export type BackgroundAnalysisProgramFactory = ( + console: ConsoleInterface, + configOptions: ConfigOptions, + importResolver: ImportResolver, + extension?: LanguageServiceExtension, + backgroundAnalysis?: BackgroundAnalysisBase, + maxAnalysisTime?: MaxAnalysisTime +) => BackgroundAnalysisProgram; diff --git a/packages/pyright-internal/src/analyzer/program.ts b/packages/pyright-internal/src/analyzer/program.ts index c404b74a1..164ab416e 100644 --- a/packages/pyright-internal/src/analyzer/program.ts +++ b/packages/pyright-internal/src/analyzer/program.ts @@ -8,13 +8,7 @@ * and all of their recursive imports. */ -import { - CancellationToken, - CompletionItem, - CompletionList, - DocumentSymbol, - SymbolInformation, -} from 'vscode-languageserver'; +import { CancellationToken, CompletionItem, DocumentSymbol, SymbolInformation } from 'vscode-languageserver'; import { CallHierarchyIncomingCall, CallHierarchyItem, @@ -51,6 +45,7 @@ import { ModuleSymbolMap, } from '../languageService/autoImporter'; import { CallHierarchyProvider } from '../languageService/callHierarchyProvider'; +import { CompletionResults } from '../languageService/completionProvider'; import { IndexResults } from '../languageService/documentSymbolProvider'; import { HoverResults } from '../languageService/hoverProvider'; import { ReferencesResult } from '../languageService/referencesProvider'; @@ -149,6 +144,10 @@ export class Program { this._createNewEvaluator(); } + get evaluator(): TypeEvaluator | undefined { + return this._evaluator; + } + setConfigOptions(configOptions: ConfigOptions) { this._configOptions = configOptions; @@ -1251,13 +1250,13 @@ export class Program { workspacePath: string, libraryMap: Map | undefined, token: CancellationToken - ): Promise { + ): Promise { const sourceFileInfo = this._sourceFileMap.get(filePath); if (!sourceFileInfo) { return undefined; } - let completionList = this._runEvaluatorWithCancellationToken(token, () => { + const completionResult = this._runEvaluatorWithCancellationToken(token, () => { this._bindFile(sourceFileInfo); const execEnv = this._configOptions.findExecEnvironment(filePath); @@ -1275,8 +1274,8 @@ export class Program { ); }); - if (!completionList || !this._extension?.completionListExtension) { - return completionList; + if (!completionResult?.completionList || !this._extension?.completionListExtension) { + return completionResult; } const pr = sourceFileInfo.sourceFile.getParseResults(); @@ -1284,8 +1283,8 @@ export class Program { if (pr?.parseTree && content) { const offset = convertPositionToOffset(position, pr.tokenizerOutput.lines); if (offset !== undefined) { - completionList = await this._extension.completionListExtension.updateCompletionList( - completionList, + completionResult.completionList = await this._extension.completionListExtension.updateCompletionList( + completionResult.completionList, pr.parseTree, content, offset, @@ -1295,7 +1294,7 @@ export class Program { } } - return completionList; + return completionResult; } resolveCompletionItem( diff --git a/packages/pyright-internal/src/analyzer/service.ts b/packages/pyright-internal/src/analyzer/service.ts index a24fc174e..cb93071a7 100644 --- a/packages/pyright-internal/src/analyzer/service.ts +++ b/packages/pyright-internal/src/analyzer/service.ts @@ -12,7 +12,6 @@ import { AbstractCancellationTokenSource, CancellationToken, CompletionItem, - CompletionList, DocumentSymbol, SymbolInformation, } from 'vscode-languageserver'; @@ -46,11 +45,12 @@ import { } from '../common/pathUtils'; import { DocumentRange, Position, Range } from '../common/textRange'; import { timingStats } from '../common/timing'; +import { CompletionResults } from '../languageService/completionProvider'; import { IndexResults } from '../languageService/documentSymbolProvider'; import { HoverResults } from '../languageService/hoverProvider'; import { SignatureHelpResults } from '../languageService/signatureHelpProvider'; import { AnalysisCompleteCallback } from './analysis'; -import { BackgroundAnalysisProgram } from './backgroundAnalysisProgram'; +import { BackgroundAnalysisProgram, BackgroundAnalysisProgramFactory } from './backgroundAnalysisProgram'; import { ImportedModuleDescriptor, ImportResolver, ImportResolverFactory } from './importResolver'; import { MaxAnalysisTime } from './program'; import { findPythonSearchPaths, getPythonPathFromPythonInterpreter } from './pythonPathUtils'; @@ -83,6 +83,7 @@ export class AnalyzerService { private _backgroundAnalysisProgram: BackgroundAnalysisProgram; private _backgroundAnalysisCancellationSource: AbstractCancellationTokenSource | undefined; private _maxAnalysisTimeInForeground?: MaxAnalysisTime; + private _backgroundAnalysisProgramFactory?: BackgroundAnalysisProgramFactory; private _disposed = false; constructor( @@ -93,7 +94,8 @@ export class AnalyzerService { configOptions?: ConfigOptions, extension?: LanguageServiceExtension, backgroundAnalysis?: BackgroundAnalysisBase, - maxAnalysisTime?: MaxAnalysisTime + maxAnalysisTime?: MaxAnalysisTime, + backgroundAnalysisProgramFactory?: BackgroundAnalysisProgramFactory ) { this._instanceName = instanceName; this._console = console || new StandardConsole(); @@ -101,17 +103,29 @@ export class AnalyzerService { this._extension = extension; this._importResolverFactory = importResolverFactory || AnalyzerService.createImportResolver; this._maxAnalysisTimeInForeground = maxAnalysisTime; + this._backgroundAnalysisProgramFactory = backgroundAnalysisProgramFactory; configOptions = configOptions ?? new ConfigOptions(process.cwd()); const importResolver = this._importResolverFactory(fs, configOptions); - this._backgroundAnalysisProgram = new BackgroundAnalysisProgram( - this._console, - configOptions, - importResolver, - this._extension, - backgroundAnalysis, - this._maxAnalysisTimeInForeground - ); + + this._backgroundAnalysisProgram = + backgroundAnalysisProgramFactory !== undefined + ? backgroundAnalysisProgramFactory( + this._console, + configOptions, + importResolver, + this._extension, + backgroundAnalysis, + this._maxAnalysisTimeInForeground + ) + : new BackgroundAnalysisProgram( + this._console, + configOptions, + importResolver, + this._extension, + backgroundAnalysis, + this._maxAnalysisTimeInForeground + ); } clone(instanceName: string, backgroundAnalysis?: BackgroundAnalysisBase): AnalyzerService { @@ -123,7 +137,8 @@ export class AnalyzerService { this._backgroundAnalysisProgram.configOptions, this._extension, backgroundAnalysis, - this._maxAnalysisTimeInForeground + this._maxAnalysisTimeInForeground, + this._backgroundAnalysisProgramFactory ); } @@ -137,6 +152,10 @@ export class AnalyzerService { this._clearLibraryReanalysisTimer(); } + get backgroundAnalysisProgram(): BackgroundAnalysisProgram { + return this._backgroundAnalysisProgram; + } + static createImportResolver(fs: FileSystem, options: ConfigOptions): ImportResolver { return new ImportResolver(fs, options); } @@ -263,7 +282,7 @@ export class AnalyzerService { position: Position, workspacePath: string, token: CancellationToken - ): Promise { + ): Promise { return this._program.getCompletionsForPosition( filePath, position, diff --git a/packages/pyright-internal/src/analyzer/sourceFile.ts b/packages/pyright-internal/src/analyzer/sourceFile.ts index 103903b7a..25eb2ceaf 100644 --- a/packages/pyright-internal/src/analyzer/sourceFile.ts +++ b/packages/pyright-internal/src/analyzer/sourceFile.ts @@ -10,7 +10,6 @@ import { CancellationToken, CompletionItem, - CompletionList, DocumentHighlight, DocumentSymbol, SymbolInformation, @@ -33,6 +32,7 @@ import { DocumentRange, getEmptyRange, Position, TextRange } from '../common/tex import { TextRangeCollection } from '../common/textRangeCollection'; import { timingStats } from '../common/timing'; import { ModuleSymbolMap } from '../languageService/autoImporter'; +import { CompletionResults } from '../languageService/completionProvider'; import { CompletionItemData, CompletionProvider } from '../languageService/completionProvider'; import { DefinitionProvider } from '../languageService/definitionProvider'; import { DocumentHighlightProvider } from '../languageService/documentHighlightProvider'; @@ -777,7 +777,7 @@ export class SourceFile { libraryMap: Map | undefined, moduleSymbolsCallback: () => ModuleSymbolMap, token: CancellationToken - ): CompletionList | undefined { + ): CompletionResults | undefined { // If we have no completed analysis job, there's nothing to do. if (!this._parseResults) { return undefined; diff --git a/packages/pyright-internal/src/backgroundAnalysisBase.ts b/packages/pyright-internal/src/backgroundAnalysisBase.ts index 85bb7d550..6425a6189 100644 --- a/packages/pyright-internal/src/backgroundAnalysisBase.ts +++ b/packages/pyright-internal/src/backgroundAnalysisBase.ts @@ -6,7 +6,7 @@ * run analyzer from background thread */ -import { CancellationToken } from 'vscode-languageserver'; +import { CancellationToken } from 'vscode-languageserver/node'; import { MessageChannel, MessagePort, parentPort, threadId, Worker, workerData } from 'worker_threads'; import { AnalysisCompleteCallback, AnalysisResults, analyzeProgram, nullCallback } from './analyzer/analysis'; @@ -79,34 +79,42 @@ export class BackgroundAnalysisBase { } setConfigOptions(configOptions: ConfigOptions) { - this._enqueueRequest({ requestType: 'setConfigOptions', data: configOptions }); + this.enqueueRequest({ requestType: 'setConfigOptions', data: configOptions }); } setTrackedFiles(filePaths: string[]) { - this._enqueueRequest({ requestType: 'setTrackedFiles', data: filePaths }); + this.enqueueRequest({ requestType: 'setTrackedFiles', data: filePaths }); } setAllowedThirdPartyImports(importNames: string[]) { - this._enqueueRequest({ requestType: 'setAllowedThirdPartyImports', data: importNames }); + this.enqueueRequest({ requestType: 'setAllowedThirdPartyImports', data: importNames }); } setFileOpened(filePath: string, version: number | null, contents: string) { - this._enqueueRequest({ requestType: 'setFileOpened', data: { filePath, version, contents } }); + this.enqueueRequest({ requestType: 'setFileOpened', data: { filePath, version, contents } }); } setFileClosed(filePath: string) { - this._enqueueRequest({ requestType: 'setFileClosed', data: filePath }); + this.enqueueRequest({ requestType: 'setFileClosed', data: filePath }); } markAllFilesDirty(evenIfContentsAreSame: boolean) { - this._enqueueRequest({ requestType: 'markAllFilesDirty', data: evenIfContentsAreSame }); + this.enqueueRequest({ requestType: 'markAllFilesDirty', data: evenIfContentsAreSame }); } markFilesDirty(filePaths: string[], evenIfContentsAreSame: boolean) { - this._enqueueRequest({ requestType: 'markFilesDirty', data: { filePaths, evenIfContentsAreSame } }); + this.enqueueRequest({ requestType: 'markFilesDirty', data: { filePaths, evenIfContentsAreSame } }); } startAnalysis(indices: Indices | undefined, token: CancellationToken) { + this._startOrResumeAnalysis('analyze', indices, token); + } + + private _startOrResumeAnalysis( + requestType: 'analyze' | 'resumeAnalysis', + indices: Indices | undefined, + token: CancellationToken + ) { const { port1, port2 } = new MessageChannel(); // Handle response from background thread to main thread. @@ -117,6 +125,17 @@ export class BackgroundAnalysisBase { break; } + case 'analysisPaused': { + disposeCancellationToken(token); + port2.close(); + port1.close(); + + // Analysis request has completed, but there is more to + // analyze, so queue another message to resume later. + this._startOrResumeAnalysis('resumeAnalysis', indices, token); + break; + } + case 'indexResult': { const { path, indexResults } = msg.data; indices?.setWorkspaceIndex(path, indexResults); @@ -136,7 +155,7 @@ export class BackgroundAnalysisBase { }); const cancellationId = getCancellationTokenId(token); - this._enqueueRequest({ requestType: 'analyze', data: cancellationId, port: port2 }); + this.enqueueRequest({ requestType, data: cancellationId, port: port2 }); } startIndexing(configOptions: ConfigOptions, indices: Indices) { @@ -158,7 +177,7 @@ export class BackgroundAnalysisBase { const waiter = getBackgroundWaiter(port1); const cancellationId = getCancellationTokenId(token); - this._enqueueRequest({ + this.enqueueRequest({ requestType: 'getDiagnosticsForRange', data: { filePath, range, cancellationId }, port: port2, @@ -184,7 +203,7 @@ export class BackgroundAnalysisBase { const waiter = getBackgroundWaiter(port1); const cancellationId = getCancellationTokenId(token); - this._enqueueRequest({ + this.enqueueRequest({ requestType: 'writeTypeStub', data: { targetImportPath, targetIsSingleFile, stubPath, cancellationId }, port: port2, @@ -197,14 +216,14 @@ export class BackgroundAnalysisBase { } invalidateAndForceReanalysis() { - this._enqueueRequest({ requestType: 'invalidateAndForceReanalysis', data: null }); + this.enqueueRequest({ requestType: 'invalidateAndForceReanalysis', data: null }); } restart() { - this._enqueueRequest({ requestType: 'restart', data: null }); + this.enqueueRequest({ requestType: 'restart', data: null }); } - private _enqueueRequest(request: AnalysisRequest) { + protected enqueueRequest(request: AnalysisRequest) { if (this._worker) { this._worker.postMessage(request, request.port ? [request.port] : undefined); } @@ -218,7 +237,11 @@ export class BackgroundAnalysisBase { export class BackgroundAnalysisRunnerBase extends BackgroundThreadBase { private _configOptions: ConfigOptions; private _importResolver: ImportResolver; - protected program: Program; + private _program: Program; + + get program(): Program { + return this._program; + } protected constructor(private _extension?: LanguageServiceExtension) { super(workerData as InitializationData); @@ -229,7 +252,7 @@ export class BackgroundAnalysisRunnerBase extends BackgroundThreadBase { this._configOptions = new ConfigOptions(data.rootDirectory); this._importResolver = this.createImportResolver(this.fs, this._configOptions); - this.program = new Program( + this._program = new Program( this._importResolver, this._configOptions, this.getConsole(), @@ -242,139 +265,7 @@ export class BackgroundAnalysisRunnerBase extends BackgroundThreadBase { this.log(LogLevel.Info, `Background analysis(${threadId}) started`); // Get requests from main thread. - parentPort?.on('message', (msg: AnalysisRequest) => { - switch (msg.requestType) { - case 'analyze': { - const port = msg.port!; - const token = getCancellationTokenFromId(msg.data); - - // Report files to analyze first. - const filesLeftToAnalyze = this.program.getFilesToAnalyzeCount(); - - this._onAnalysisCompletion(port, { - diagnostics: [], - filesInProgram: this.program.getFileCount(), - filesRequiringAnalysis: filesLeftToAnalyze, - checkingOnlyOpenFiles: this.program.isCheckingOnlyOpenFiles(), - fatalErrorOccurred: false, - configParseErrorOccurred: false, - elapsedTime: 0, - }); - - // Report results at the interval of the max analysis time. - const maxTime = { openFilesTimeInMs: 50, noOpenFilesTimeInMs: 200 }; - let moreToAnalyze = true; - - while (moreToAnalyze) { - moreToAnalyze = analyzeProgram( - this.program, - maxTime, - this._configOptions, - (result) => this._onAnalysisCompletion(port, result), - this.getConsole(), - token - ); - } - - this.processIndexing(port, token); - - this._analysisDone(port, msg.data); - break; - } - - case 'getDiagnosticsForRange': { - run(() => { - const { filePath, range, cancellationId } = msg.data; - const token = getCancellationTokenFromId(cancellationId); - throwIfCancellationRequested(token); - - return this.program.getDiagnosticsForRange(filePath, range); - }, msg.port!); - break; - } - - case 'writeTypeStub': { - run(() => { - const { targetImportPath, targetIsSingleFile, stubPath, cancellationId } = msg.data; - const token = getCancellationTokenFromId(cancellationId); - - analyzeProgram( - this.program, - undefined, - this._configOptions, - nullCallback, - this.getConsole(), - token - ); - this.program.writeTypeStub(targetImportPath, targetIsSingleFile, stubPath, token); - }, msg.port!); - break; - } - - case 'setConfigOptions': { - this._configOptions = createConfigOptionsFrom(msg.data); - this._importResolver = this.createImportResolver(this.fs, this._configOptions); - this.program.setConfigOptions(this._configOptions); - this.program.setImportResolver(this._importResolver); - break; - } - - case 'setTrackedFiles': { - const diagnostics = this.program.setTrackedFiles(msg.data); - this._reportDiagnostics(diagnostics, this.program.getFilesToAnalyzeCount(), 0); - break; - } - - case 'setAllowedThirdPartyImports': { - this.program.setAllowedThirdPartyImports(msg.data); - break; - } - - case 'setFileOpened': { - const { filePath, version, contents } = msg.data; - this.program.setFileOpened(filePath, version, contents); - break; - } - - case 'setFileClosed': { - const diagnostics = this.program.setFileClosed(msg.data); - this._reportDiagnostics(diagnostics, this.program.getFilesToAnalyzeCount(), 0); - break; - } - - case 'markAllFilesDirty': { - this.program.markAllFilesDirty(msg.data); - break; - } - - case 'markFilesDirty': { - const { filePaths, evenIfContentsAreSame } = msg.data; - this.program.markFilesDirty(filePaths, evenIfContentsAreSame); - break; - } - - case 'invalidateAndForceReanalysis': { - // Make sure the import resolver doesn't have invalid - // cached entries. - this._importResolver.invalidateCache(); - - // Mark all files with one or more errors dirty. - this.program.markAllFilesDirty(true); - break; - } - - case 'restart': { - // recycle import resolver - this._importResolver = this.createImportResolver(this.fs, this._configOptions); - this.program.setImportResolver(this._importResolver); - break; - } - - default: { - debug.fail(`${msg.requestType} is not expected`); - } - } - }); + parentPort?.on('message', (msg: AnalysisRequest) => this.onMessage(msg)); parentPort?.on('error', (msg) => debug.fail(`failed ${msg}`)); parentPort?.on('exit', (c) => { @@ -384,6 +275,155 @@ export class BackgroundAnalysisRunnerBase extends BackgroundThreadBase { }); } + protected onMessage(msg: AnalysisRequest) { + this.log(LogLevel.Info, `Background analysis message: ${msg.requestType}`); + switch (msg.requestType) { + case 'analyze': { + const port = msg.port!; + const token = getCancellationTokenFromId(msg.data); + + // Report files to analyze first. + const filesLeftToAnalyze = this.program.getFilesToAnalyzeCount(); + + this._onAnalysisCompletion(port, { + diagnostics: [], + filesInProgram: this.program.getFileCount(), + filesRequiringAnalysis: filesLeftToAnalyze, + checkingOnlyOpenFiles: this.program.isCheckingOnlyOpenFiles(), + fatalErrorOccurred: false, + configParseErrorOccurred: false, + elapsedTime: 0, + }); + + this._analyzeOneChunk(port, token, msg); + break; + } + + case 'resumeAnalysis': { + const port = msg.port!; + const token = getCancellationTokenFromId(msg.data); + + this._analyzeOneChunk(port, token, msg); + break; + } + + case 'getDiagnosticsForRange': { + run(() => { + const { filePath, range, cancellationId } = msg.data; + const token = getCancellationTokenFromId(cancellationId); + throwIfCancellationRequested(token); + + return this.program.getDiagnosticsForRange(filePath, range); + }, msg.port!); + break; + } + + case 'writeTypeStub': { + run(() => { + const { targetImportPath, targetIsSingleFile, stubPath, cancellationId } = msg.data; + const token = getCancellationTokenFromId(cancellationId); + + analyzeProgram( + this.program, + undefined, + this._configOptions, + nullCallback, + this.getConsole(), + token + ); + this.program.writeTypeStub(targetImportPath, targetIsSingleFile, stubPath, token); + }, msg.port!); + break; + } + + case 'setConfigOptions': { + this._configOptions = createConfigOptionsFrom(msg.data); + this._importResolver = this.createImportResolver(this.fs, this._configOptions); + this.program.setConfigOptions(this._configOptions); + this.program.setImportResolver(this._importResolver); + break; + } + + case 'setTrackedFiles': { + const diagnostics = this.program.setTrackedFiles(msg.data); + this._reportDiagnostics(diagnostics, this.program.getFilesToAnalyzeCount(), 0); + break; + } + + case 'setAllowedThirdPartyImports': { + this.program.setAllowedThirdPartyImports(msg.data); + break; + } + + case 'setFileOpened': { + const { filePath, version, contents } = msg.data; + this.program.setFileOpened(filePath, version, contents); + break; + } + + case 'setFileClosed': { + const diagnostics = this.program.setFileClosed(msg.data); + this._reportDiagnostics(diagnostics, this.program.getFilesToAnalyzeCount(), 0); + break; + } + + case 'markAllFilesDirty': { + this.program.markAllFilesDirty(msg.data); + break; + } + + case 'markFilesDirty': { + const { filePaths, evenIfContentsAreSame } = msg.data; + this.program.markFilesDirty(filePaths, evenIfContentsAreSame); + break; + } + + case 'invalidateAndForceReanalysis': { + // Make sure the import resolver doesn't have invalid + // cached entries. + this._importResolver.invalidateCache(); + + // Mark all files with one or more errors dirty. + this.program.markAllFilesDirty(true); + break; + } + + case 'restart': { + // recycle import resolver + this._importResolver = this.createImportResolver(this.fs, this._configOptions); + this.program.setImportResolver(this._importResolver); + break; + } + + default: { + debug.fail(`${msg.requestType} is not expected`); + } + } + } + + private _analyzeOneChunk(port: MessagePort, token: CancellationToken, msg: AnalysisRequest) { + // Report results at the interval of the max analysis time. + const maxTime = { openFilesTimeInMs: 50, noOpenFilesTimeInMs: 200 }; + const moreToAnalyze = analyzeProgram( + this.program, + maxTime, + this._configOptions, + (result) => this._onAnalysisCompletion(port, result), + this.getConsole(), + token + ); + + if (moreToAnalyze) { + // There's more to analyze after we exceeded max time, + // so report that we are paused. The foreground thread will + // then queue up a message to resume the analysis. + this._analysisPaused(port, msg.data); + } else { + this.processIndexing(port, token); + this._analysisDone(port, msg.data); + } + } + protected createImportResolver(fs: FileSystem, options: ConfigOptions): ImportResolver { return new ImportResolver(fs, options); } @@ -414,6 +454,10 @@ export class BackgroundAnalysisRunnerBase extends BackgroundThreadBase { port.postMessage({ requestType: 'analysisResult', data: result }); } + private _analysisPaused(port: MessagePort, cancellationId: string) { + port.postMessage({ requestType: 'analysisPaused', data: cancellationId }); + } + private _analysisDone(port: MessagePort, cancellationId: string) { port.postMessage({ requestType: 'analysisDone', data: cancellationId }); } @@ -461,9 +505,10 @@ export interface InitializationData { runner?: string; } -interface AnalysisRequest { +export interface AnalysisRequest { requestType: | 'analyze' + | 'resumeAnalysis' | 'setConfigOptions' | 'setTrackedFiles' | 'setAllowedThirdPartyImports' @@ -474,13 +519,14 @@ interface AnalysisRequest { | 'invalidateAndForceReanalysis' | 'restart' | 'getDiagnosticsForRange' - | 'writeTypeStub'; + | 'writeTypeStub' + | 'getSemanticTokens'; data: any; port?: MessagePort; } interface AnalysisResponse { - requestType: 'log' | 'analysisResult' | 'indexResult' | 'analysisDone'; + requestType: 'log' | 'analysisResult' | 'analysisPaused' | 'indexResult' | 'analysisDone'; data: any; } diff --git a/packages/pyright-internal/src/languageServerBase.ts b/packages/pyright-internal/src/languageServerBase.ts index 14a7bec3d..353b0128d 100644 --- a/packages/pyright-internal/src/languageServerBase.ts +++ b/packages/pyright-internal/src/languageServerBase.ts @@ -44,9 +44,11 @@ import { WatchKind, WorkDoneProgressReporter, WorkspaceEdit, + WorkspaceFolder, } from 'vscode-languageserver/node'; import { AnalysisResults } from './analyzer/analysis'; +import { BackgroundAnalysisProgram } from './analyzer/backgroundAnalysisProgram'; import { ImportResolver } from './analyzer/importResolver'; import { MaxAnalysisTime } from './analyzer/program'; import { AnalyzerService, configFileNames } from './analyzer/service'; @@ -76,7 +78,7 @@ import { ProgressReporter, ProgressReportTracker } from './common/progressReport import { convertWorkspaceEdits } from './common/textEditUtils'; import { Position } from './common/textRange'; import { AnalyzerServiceExecutor } from './languageService/analyzerServiceExecutor'; -import { CompletionItemData } from './languageService/completionProvider'; +import { CompletionItemData, CompletionResults } from './languageService/completionProvider'; import { convertHoverResults } from './languageService/hoverProvider'; import { Localizer } from './localization/localize'; import { WorkspaceMap } from './workspaceMap'; @@ -215,7 +217,7 @@ export abstract class LanguageServerBase implements LanguageServerInterface { this._workspaceMap = new WorkspaceMap(this); // Set up callbacks. - this._setupConnection(_serverOptions.supportedCommands ?? [], _serverOptions.supportedCodeActions ?? []); + this.setupConnection(_serverOptions.supportedCommands ?? [], _serverOptions.supportedCodeActions ?? []); this._progressReporter = new ProgressReportTracker( this._serverOptions.progressReporterFactory @@ -287,6 +289,24 @@ export abstract class LanguageServerBase implements LanguageServerInterface { return new ImportResolver(fs, options); } + protected createBackgroundAnalysisProgram( + console: ConsoleInterface, + configOptions: ConfigOptions, + importResolver: ImportResolver, + extension?: LanguageServiceExtension, + backgroundAnalysis?: BackgroundAnalysisBase, + maxAnalysisTime?: MaxAnalysisTime + ): BackgroundAnalysisProgram { + return new BackgroundAnalysisProgram( + console, + configOptions, + importResolver, + extension, + backgroundAnalysis, + maxAnalysisTime + ); + } + protected setExtension(extension: any): void { this._serverOptions.extension = extension; } @@ -308,7 +328,8 @@ export abstract class LanguageServerBase implements LanguageServerInterface { undefined, this._serverOptions.extension, this.createBackgroundAnalysis(), - this._serverOptions.maxAnalysisTimeInForeground + this._serverOptions.maxAnalysisTimeInForeground, + this.createBackgroundAnalysisProgram.bind(this) ); service.setCompletionCallback((results) => this.onAnalysisCompletedHandler(results)); @@ -380,7 +401,7 @@ export abstract class LanguageServerBase implements LanguageServerInterface { return fileWatcher; } - private _setupConnection(supportedCommands: string[], supportedCodeActions: string[]): void { + protected setupConnection(supportedCommands: string[], supportedCodeActions: string[]): void { // After the server has started the client sends an initialize request. The server receives // in the passed params the rootPath of the workspace plus the client capabilities. this._connection.onInitialize((params) => this.initialize(params, supportedCommands, supportedCodeActions)); @@ -598,6 +619,7 @@ export abstract class LanguageServerBase implements LanguageServerInterface { }); this._connection.onCompletion((params, token) => this.onCompletion(params, token)); + this._connection.onCompletionResolve(async (params, token) => { // Cancellation bugs in vscode and LSP: // https://github.com/microsoft/vscode-languageserver-node/issues/615 @@ -762,15 +784,7 @@ export abstract class LanguageServerBase implements LanguageServerInterface { event.added.forEach(async (workspace) => { const rootPath = convertUriToPath(workspace.uri); - const newWorkspace: WorkspaceServiceInstance = { - workspaceName: workspace.name, - rootPath, - rootUri: workspace.uri, - serviceInstance: this.createAnalyzerService(workspace.name), - disableLanguageServices: false, - disableOrganizeImports: false, - isInitialized: createDeferred(), - }; + const newWorkspace = this.createWorkspaceServiceInstance(workspace, rootPath); this._workspaceMap.set(rootPath, newWorkspace); await this.updateSettingsForWorkspace(newWorkspace); }); @@ -833,6 +847,16 @@ export abstract class LanguageServerBase implements LanguageServerInterface { }); } + protected getWorkspaceCompletionsForPosition( + workspace: WorkspaceServiceInstance, + filePath: string, + position: Position, + workspacePath: string, + token: CancellationToken + ): Promise { + return workspace.serviceInstance.getCompletionsForPosition(filePath, position, workspacePath, token); + } + updateSettingsForAllWorkspaces(): void { this._workspaceMap.forEach((workspace) => { this.updateSettingsForWorkspace(workspace).ignoreErrors(); @@ -864,26 +888,10 @@ export abstract class LanguageServerBase implements LanguageServerInterface { if (params.workspaceFolders) { params.workspaceFolders.forEach((folder) => { const path = convertUriToPath(folder.uri); - this._workspaceMap.set(path, { - workspaceName: folder.name, - rootPath: path, - rootUri: folder.uri, - serviceInstance: this.createAnalyzerService(folder.name), - disableLanguageServices: false, - disableOrganizeImports: false, - isInitialized: createDeferred(), - }); + this._workspaceMap.set(path, this.createWorkspaceServiceInstance(folder, path)); }); } else if (params.rootPath) { - this._workspaceMap.set(params.rootPath, { - workspaceName: '', - rootPath: params.rootPath, - rootUri: '', - serviceInstance: this.createAnalyzerService(params.rootPath), - disableLanguageServices: false, - disableOrganizeImports: false, - isInitialized: createDeferred(), - }); + this._workspaceMap.set(params.rootPath, this.createWorkspaceServiceInstance(undefined, params.rootPath)); } const result: InitializeResult = { @@ -922,6 +930,21 @@ export abstract class LanguageServerBase implements LanguageServerInterface { return result; } + protected createWorkspaceServiceInstance( + workspace: WorkspaceFolder | undefined, + rootPath: string + ): WorkspaceServiceInstance { + return { + workspaceName: workspace?.name ?? '', + rootPath, + rootUri: workspace?.uri ?? '', + serviceInstance: this.createAnalyzerService(workspace?.name ?? rootPath), + disableLanguageServices: false, + disableOrganizeImports: false, + isInitialized: createDeferred(), + }; + } + protected onAnalysisCompletedHandler(results: AnalysisResults): void { // Send the computed diagnostics to the client. results.diagnostics.forEach((fileDiag) => { @@ -956,8 +979,11 @@ export abstract class LanguageServerBase implements LanguageServerInterface { } } - async updateSettingsForWorkspace(workspace: WorkspaceServiceInstance): Promise { - const serverSettings = await this.getSettings(workspace); + async updateSettingsForWorkspace( + workspace: WorkspaceServiceInstance, + serverSettings?: ServerSettings + ): Promise { + serverSettings = serverSettings ?? (await this.getSettings(workspace)); // Set logging level first. (this.console as ConsoleWithLogLevel).level = serverSettings.logLevel ?? LogLevel.Info; @@ -1007,18 +1033,19 @@ export abstract class LanguageServerBase implements LanguageServerInterface { return; } - const completions = await workspace.serviceInstance.getCompletionsForPosition( + const completions = await this.getWorkspaceCompletionsForPosition( + workspace, filePath, position, workspace.rootPath, token ); - if (completions) { - completions.isIncomplete = completionIncomplete; + if (completions && completions.completionList) { + completions.completionList.isIncomplete = completionIncomplete; } - return completions; + return completions?.completionList; } protected convertLogLevel(logLevel?: string): LogLevel { diff --git a/packages/pyright-internal/src/languageService/completionProvider.ts b/packages/pyright-internal/src/languageService/completionProvider.ts index 84519640b..93c990abd 100644 --- a/packages/pyright-internal/src/languageService/completionProvider.ts +++ b/packages/pyright-internal/src/languageService/completionProvider.ts @@ -44,6 +44,8 @@ import { isNone, isObject, isTypeVar, + isUnbound, + isUnknown, ObjectType, Type, TypeBase, @@ -51,6 +53,7 @@ import { } from '../analyzer/types'; import { doForSubtypes, + getDeclaringModulesForType, getMembersForClass, getMembersForModule, isProperty, @@ -174,6 +177,18 @@ export interface CompletionItemData { symbolId?: number; } +// ModuleContext attempts to gather info for unknown types +export interface ModuleContext { + lastKnownModule?: string; + lastKnownMemberName?: string; + unknownMemberName?: string; +} + +export interface CompletionResults { + completionList: CompletionList | undefined; + moduleContext?: ModuleContext; +} + interface RecentCompletionInfo { label: string; autoImportText: string; @@ -209,7 +224,7 @@ export class CompletionProvider { private _cancellationToken: CancellationToken ) {} - getCompletionsForPosition(): CompletionList | undefined { + getCompletionsForPosition(): CompletionResults | undefined { const offset = convertPositionToOffset(this._position, this._parseResults.tokenizerOutput.lines); if (offset === undefined) { return undefined; @@ -412,7 +427,7 @@ export class CompletionProvider { priorWord: string, priorText: string, postText: string - ): CompletionList | undefined { + ): CompletionResults | undefined { // Is the error due to a missing member access name? If so, // we can evaluate the left side of the member access expression // to determine its type and offer suggestions based on it. @@ -451,15 +466,15 @@ export class CompletionProvider { return undefined; } - private _createSingleKeywordCompletionList(keyword: string): CompletionList { + private _createSingleKeywordCompletionList(keyword: string): CompletionResults { const completionItem = CompletionItem.create(keyword); completionItem.kind = CompletionItemKind.Keyword; completionItem.sortText = this._makeSortText(SortCategory.LikelyKeyword, keyword); - - return CompletionList.create([completionItem]); + const completionList = CompletionList.create([completionItem]); + return { completionList }; } - private _getMethodOverrideCompletions(partialName: NameNode): CompletionList | undefined { + private _getMethodOverrideCompletions(partialName: NameNode): CompletionResults | undefined { const enclosingClass = ParseTreeUtils.getEnclosingClass(partialName, true); if (!enclosingClass) { return undefined; @@ -498,7 +513,7 @@ export class CompletionProvider { } }); - return completionList; + return { completionList }; } private _printMethodSignature(node: FunctionNode): string { @@ -536,10 +551,14 @@ export class CompletionProvider { return methodSignature; } - private _getMemberAccessCompletions(leftExprNode: ExpressionNode, priorWord: string): CompletionList | undefined { + private _getMemberAccessCompletions( + leftExprNode: ExpressionNode, + priorWord: string + ): CompletionResults | undefined { const leftType = this._evaluator.getType(leftExprNode); const symbolTable = new Map(); const completionList = CompletionList.create(); + let lastKnownModule: ModuleContext | undefined; if (leftType) { doForSubtypes(leftType, (subtype) => { @@ -575,11 +594,65 @@ export class CompletionProvider { const objectThrough: ObjectType | undefined = leftType && isObject(leftType) ? leftType : undefined; this._addSymbolsForSymbolTable(symbolTable, (_) => true, priorWord, objectThrough, completionList); - // const moduleNamesForType = getDeclaringModulesForType(leftType); - // TODO - log modules to telemetry. + // If we dont know this type, look for a module we should stub + if (!leftType || isUnknown(leftType) || isUnbound(leftType)) { + lastKnownModule = this._getLastKnownModule(leftExprNode, leftType); + } } - return completionList; + return { completionList, moduleContext: lastKnownModule }; + } + + private _getLastKnownModule(leftExprNode: ExpressionNode, leftType: Type | undefined): ModuleContext | undefined { + let curNode: ExpressionNode | undefined = leftExprNode; + let curType: Type | undefined = leftType; + let unknownMemberName: string | undefined = + leftExprNode.nodeType === ParseNodeType.MemberAccess ? leftExprNode?.memberName.value : undefined; + + // Walk left of the expression scope till we find a known type. A.B.Unknown.<-- return B. + while (curNode) { + if (curNode.nodeType === ParseNodeType.Call || curNode.nodeType === ParseNodeType.MemberAccess) { + // Move left + curNode = curNode.leftExpression; + + // First time in the loop remember the name of the unknown type. + if (unknownMemberName === undefined) { + unknownMemberName = + curNode.nodeType === ParseNodeType.MemberAccess ? curNode?.memberName.value ?? '' : ''; + } + } else { + curNode = undefined; + } + + if (curNode) { + curType = this._evaluator.getType(curNode); + + // Breakout if we found a known type. + if (curType !== undefined && !isUnknown(curType) && !isUnbound(curType)) { + break; + } + } + } + + const context: ModuleContext = {}; + if (curType && !isUnknown(curType) && !isUnbound(curType) && curNode) { + const moduleNamesForType = getDeclaringModulesForType(curType); + + // For union types we only care about non 'typing' modules. + context.lastKnownModule = moduleNamesForType.find((n) => n !== 'typing'); + + if (curNode.nodeType === ParseNodeType.MemberAccess) { + context.lastKnownMemberName = curNode.memberName.value; + } else if (curNode.nodeType === ParseNodeType.Name && isClass(curType)) { + context.lastKnownMemberName = curType.details.name; + } else if (curNode.nodeType === ParseNodeType.Name && isObject(curType)) { + context.lastKnownMemberName = curType.classType.details.name; + } + + context.unknownMemberName = unknownMemberName; + } + + return context; } private _getStatementCompletions( @@ -587,7 +660,7 @@ export class CompletionProvider { priorWord: string, priorText: string, postText: string - ): CompletionList | undefined { + ): CompletionResults | undefined { // For now, use the same logic for expressions and statements. return this._getExpressionCompletions(parseNode, priorWord, priorText, postText); } @@ -597,7 +670,7 @@ export class CompletionProvider { priorWord: string, priorText: string, postText: string - ): CompletionList | undefined { + ): CompletionResults | undefined { // If the user typed a "." as part of a number, don't present // any completion options. if (parseNode.nodeType === ParseNodeType.Number) { @@ -643,7 +716,7 @@ export class CompletionProvider { } } - return completionList; + return { completionList }; } private _addCallArgumentCompletions( @@ -732,7 +805,7 @@ export class CompletionProvider { priorWord: string, priorText: string, postText: string - ): CompletionList | undefined { + ): CompletionResults | undefined { let parentNode: ParseNode | undefined = parseNode.parent; if (!parentNode || parentNode.nodeType !== ParseNodeType.StringList || parentNode.strings.length > 1) { return undefined; @@ -784,7 +857,7 @@ export class CompletionProvider { this._addCallArgumentCompletions(parseNode, priorWord, priorText, postText, completionList); } - return completionList; + return { completionList }; } // Given a string of text that precedes the current insertion point, @@ -926,7 +999,10 @@ export class CompletionProvider { } } - private _getImportFromCompletions(importFromNode: ImportFromNode, priorWord: string): CompletionList | undefined { + private _getImportFromCompletions( + importFromNode: ImportFromNode, + priorWord: string + ): CompletionResults | undefined { // Don't attempt to provide completions for "from X import *". if (importFromNode.isWildcardImport) { return undefined; @@ -965,7 +1041,7 @@ export class CompletionProvider { } }); - return completionList; + return { completionList }; } private _findMatchingKeywords(keywordList: string[], partialMatch: string): string[] { @@ -1443,7 +1519,7 @@ export class CompletionProvider { } } - private _getImportModuleCompletions(node: ModuleNameNode): CompletionList { + private _getImportModuleCompletions(node: ModuleNameNode): CompletionResults { const execEnvironment = this._configOptions.findExecEnvironment(this._filePath); const moduleDescriptor: ImportedModuleDescriptor = { leadingDots: node.leadingDots, @@ -1483,6 +1559,6 @@ export class CompletionProvider { completionItem.sortText = this._makeSortText(SortCategory.ImportModuleName, completionName); }); - return completionList; + return { completionList }; } } diff --git a/packages/pyright-internal/src/tests/fourslash/completions.moduleContext.UnknownMemberOnInstance.fourslash.ts b/packages/pyright-internal/src/tests/fourslash/completions.moduleContext.UnknownMemberOnInstance.fourslash.ts new file mode 100644 index 000000000..35b91219b --- /dev/null +++ b/packages/pyright-internal/src/tests/fourslash/completions.moduleContext.UnknownMemberOnInstance.fourslash.ts @@ -0,0 +1,17 @@ +/// + +// @filename: test.py +//// class Model: +//// pass +//// +//// def some_func1(a: Model): +//// x = a.unknownName.[|/*marker1*/|] +//// pass + +// @ts-ignore +await helper.verifyCompletion('included', { + marker1: { + completions: [], + moduleContext: { lastKnownModule: 'test', lastKnownMemberName: 'Model', unknownMemberName: 'unknownName' }, + }, +}); diff --git a/packages/pyright-internal/src/tests/fourslash/completions.moduleContext.UnknownStaticFunctionOnClass.fourslash.ts b/packages/pyright-internal/src/tests/fourslash/completions.moduleContext.UnknownStaticFunctionOnClass.fourslash.ts new file mode 100644 index 000000000..b4fa8e22b --- /dev/null +++ b/packages/pyright-internal/src/tests/fourslash/completions.moduleContext.UnknownStaticFunctionOnClass.fourslash.ts @@ -0,0 +1,57 @@ +/// + +// @filename: test.py +//// class Model: +//// @staticmethod +//// def foo( value ): +//// return value +//// +//// +//// x = Model.foo(unknownValue).[|/*marker1*/|] +//// pass +//// +//// y = Model.unknownMember.[|/*marker2*/|] +//// pass +//// +//// def some_func1(a: Model): +//// x = a.unknownMember.[|/*marker3*/|] +//// pass +//// +//// Model.unknownValue.[|/*marker4*/|] +//// +//// UnkownModel.unknownValue.[|/*marker5*/|] + +// @ts-ignore +await helper.verifyCompletion('included', { + // tests: _getLastKnownModule(): if (curNode.nodeType === ParseNodeType.MemberAccess && curNode.memberName) + marker1: { + completions: [], + moduleContext: { lastKnownModule: 'test', lastKnownMemberName: 'foo', unknownMemberName: 'foo' }, + }, + // tests: _getLastKnownModule(): else if (curNode.nodeType === ParseNodeType.Name && isClass(curType)) + marker2: { + completions: [], + moduleContext: { + lastKnownModule: 'test', + lastKnownMemberName: 'Model', + unknownMemberName: 'unknownMember', + }, + }, + // tests: _getLastKnownModule(): else if (curNode.nodeType === ParseNodeType.Name && isObject(curType)) + marker3: { + completions: [], + moduleContext: { + lastKnownModule: 'test', + lastKnownMemberName: 'Model', + unknownMemberName: 'unknownMember', + }, + }, + marker4: { + completions: [], + moduleContext: { lastKnownModule: 'test', lastKnownMemberName: 'Model', unknownMemberName: 'unknownValue' }, + }, + marker5: { + completions: [], + moduleContext: {}, + }, +}); diff --git a/packages/pyright-internal/src/tests/fourslash/completions.moduleContext.libCodeNoStub.fourslash.ts b/packages/pyright-internal/src/tests/fourslash/completions.moduleContext.libCodeNoStub.fourslash.ts new file mode 100644 index 000000000..71f9f08de --- /dev/null +++ b/packages/pyright-internal/src/tests/fourslash/completions.moduleContext.libCodeNoStub.fourslash.ts @@ -0,0 +1,30 @@ +/// + +// @filename: mspythonconfig.json +//// { +//// "useLibraryCodeForTypes": true +//// } + +// @filename: test.py +//// import testnumpy +//// obj = testnumpy.random.randint("foo").[|/*marker1*/|] + +// @filename: testnumpy/__init__.py +// @library: true +//// from . import random + +// @filename: testnumpy/random/__init__.py +// @library: true +//// __all__ = ['randint'] + +// @ts-ignore +await helper.verifyCompletion('included', { + marker1: { + completions: [], + moduleContext: { + lastKnownModule: 'testnumpy', + lastKnownMemberName: 'random', + unknownMemberName: 'randint', + }, + }, +}); diff --git a/packages/pyright-internal/src/tests/fourslash/fourslash.ts b/packages/pyright-internal/src/tests/fourslash/fourslash.ts index 95ea5dd3b..1276a50ab 100644 --- a/packages/pyright-internal/src/tests/fourslash/fourslash.ts +++ b/packages/pyright-internal/src/tests/fourslash/fourslash.ts @@ -154,6 +154,11 @@ declare namespace _ { map: { [marker: string]: { completions: FourSlashCompletionItem[]; + moduleContext?: { + lastKnownModule?: string; + lastKnownMemberName?: string; + unknownMemberName?: string; + }; }; } ): Promise; diff --git a/packages/pyright-internal/src/tests/harness/fourslash/runner.ts b/packages/pyright-internal/src/tests/harness/fourslash/runner.ts index 22db82c7d..63405e4bc 100644 --- a/packages/pyright-internal/src/tests/harness/fourslash/runner.ts +++ b/packages/pyright-internal/src/tests/harness/fourslash/runner.ts @@ -11,9 +11,17 @@ import * as ts from 'typescript'; import { combinePaths } from '../../../common/pathUtils'; import * as host from '../host'; import { parseTestData } from './fourSlashParser'; +import { FourSlashData } from './fourSlashTypes'; import { HostSpecificFeatures, TestState } from './testState'; import { Consts } from './testState.Consts'; +export type TestStateFactory = ( + basePath: string, + testData: FourSlashData, + mountPaths?: Map, + hostSpecificFeatures?: HostSpecificFeatures +) => TestState; + /** * run given fourslash test file * @@ -25,10 +33,11 @@ export function runFourSlashTest( fileName: string, cb?: jest.DoneCallback, mountPaths?: Map, - hostSpecificFeatures?: HostSpecificFeatures + hostSpecificFeatures?: HostSpecificFeatures, + testStateFactory?: TestStateFactory ) { const content = host.HOST.readFile(fileName)!; - runFourSlashTestContent(basePath, fileName, content, cb, mountPaths, hostSpecificFeatures); + runFourSlashTestContent(basePath, fileName, content, cb, mountPaths, hostSpecificFeatures, testStateFactory); } /** @@ -45,7 +54,8 @@ export function runFourSlashTestContent( content: string, cb?: jest.DoneCallback, mountPaths?: Map, - hostSpecificFeatures?: HostSpecificFeatures + hostSpecificFeatures?: HostSpecificFeatures, + testStateFactory?: TestStateFactory ) { // give file paths an absolute path for the virtual file system const absoluteBasePath = combinePaths('/', basePath); @@ -53,7 +63,10 @@ export function runFourSlashTestContent( // parse out the files and their metadata const testData = parseTestData(absoluteBasePath, content, absoluteFileName); - const state = new TestState(absoluteBasePath, testData, mountPaths, hostSpecificFeatures); + const state = + testStateFactory !== undefined + ? testStateFactory(absoluteBasePath, testData, mountPaths, hostSpecificFeatures) + : new TestState(absoluteBasePath, testData, mountPaths, hostSpecificFeatures); const output = ts.transpileModule(content, { reportDiagnostics: true, compilerOptions: { target: ts.ScriptTarget.ES2015 }, diff --git a/packages/pyright-internal/src/tests/harness/fourslash/testState.ts b/packages/pyright-internal/src/tests/harness/fourslash/testState.ts index 51f286af5..ae584805f 100644 --- a/packages/pyright-internal/src/tests/harness/fourslash/testState.ts +++ b/packages/pyright-internal/src/tests/harness/fourslash/testState.ts @@ -253,7 +253,7 @@ export class TestState { const marker = this.getMarkerByName(markerString); const ranges = this.getRanges().filter((r) => r.marker === marker); if (ranges.length !== 1) { - this._raiseError(`no matching range for ${markerString}`); + this.raiseError(`no matching range for ${markerString}`); } const range = ranges[0]; @@ -261,7 +261,7 @@ export class TestState { } convertPositionRange(range: Range) { - return this._convertOffsetsToRange(range.fileName, range.pos, range.end); + return this.convertOffsetsToRange(range.fileName, range.pos, range.end); } goToPosition(positionOrLineAndColumn: number | Position) { @@ -327,7 +327,7 @@ export class TestState { if (this.testData.rangesByText) { return this.testData.rangesByText; } - const result = this._createMultiMap(this.getRanges(), (r) => this._rangeText(r)); + const result = this.createMultiMap(this.getRanges(), (r) => this._rangeText(r)); this.testData.rangesByText = result; return result; @@ -462,7 +462,7 @@ export class TestState { // organize things per file const resultPerFile = this._getDiagnosticsPerFile(); - const rangePerFile = this._createMultiMap(this.getRanges(), (r) => r.fileName); + const rangePerFile = this.createMultiMap(this.getRanges(), (r) => r.fileName); if (!hasDiagnostics(resultPerFile) && rangePerFile.size === 0) { // no errors and no error is expected. we are done @@ -470,7 +470,7 @@ export class TestState { } for (const [file, ranges] of rangePerFile.entries()) { - const rangesPerCategory = this._createMultiMap(ranges, (r) => { + const rangesPerCategory = this.createMultiMap(ranges, (r) => { if (map) { const name = this.getMarkerName(r.marker!); return map[name].category; @@ -503,10 +503,10 @@ export class TestState { ? result.warnings : category === 'information' ? result.information - : this._raiseError(`unexpected category ${category}`); + : this.raiseError(`unexpected category ${category}`); if (expected.length !== actual.length) { - this._raiseError( + this.raiseError( `contains unexpected result - expected: ${stringify(expected)}, actual: ${stringify(actual)}` ); } @@ -522,7 +522,7 @@ export class TestState { }); if (matches.length === 0) { - this._raiseError(`doesn't contain expected range: ${stringify(range)}`); + this.raiseError(`doesn't contain expected range: ${stringify(range)}`); } // if map is provided, check message as well @@ -531,7 +531,7 @@ export class TestState { const message = map[name].message; if (matches.filter((d) => message === d.message).length !== 1) { - this._raiseError( + this.raiseError( `message doesn't match: ${message} of ${name} - ${stringify( range )}, actual: ${stringify(matches)}` @@ -543,7 +543,7 @@ export class TestState { } if (hasDiagnostics(resultPerFile)) { - this._raiseError(`these diagnostics were unexpected: ${stringify(resultPerFile)}`); + this.raiseError(`these diagnostics were unexpected: ${stringify(resultPerFile)}`); } function hasDiagnostics( @@ -586,7 +586,7 @@ export class TestState { const codeActions = await this._getCodeActions(range); if (verifyCodeActionCount) { if (codeActions.length !== map[name].codeActions.length) { - this._raiseError( + this.raiseError( `doesn't contain expected result: ${stringify(map[name])}, actual: ${stringify(codeActions)}` ); } @@ -616,7 +616,7 @@ export class TestState { }); if (matches.length !== 1) { - this._raiseError( + this.raiseError( `doesn't contain expected result: ${stringify(expected)}, actual: ${stringify(codeActions)}` ); } @@ -655,7 +655,7 @@ export class TestState { const expectedText: string = Object.values(files)[0]; if (actualText !== expectedText) { - this._raiseError( + this.raiseError( `doesn't contain expected result: ${stringify(expectedText)}, actual: ${stringify(actualText)}` ); } @@ -681,7 +681,7 @@ export class TestState { const codeActions = await this._getCodeActions(range); if (verifyCodeActionCount) { if (codeActions.length !== Object.keys(map).length) { - this._raiseError( + this.raiseError( `doesn't contain expected result: ${stringify(map[name])}, actual: ${stringify(codeActions)}` ); } @@ -689,7 +689,7 @@ export class TestState { const matches = codeActions.filter((c) => c.title === map[name].title); if (matches.length === 0) { - this._raiseError( + this.raiseError( `doesn't contain expected result: ${stringify(map[name])}, actual: ${stringify(codeActions)}` ); } @@ -713,7 +713,7 @@ export class TestState { (e) => rangesAreEqual(e.range, edit.range) && e.newText === edit.newText ).length !== 1 ) { - this._raiseError( + this.raiseError( `doesn't contain expected result: ${stringify(map[name])}, actual: ${stringify( edits )}` @@ -739,7 +739,7 @@ export class TestState { continue; } - const rangePos = this._convertOffsetsToRange(range.fileName, range.pos, range.end); + const rangePos = this.convertOffsetsToRange(range.fileName, range.pos, range.end); const actual = convertHoverResults( this.program.getHoverForPosition(range.fileName, rangePos.start, CancellationToken.None) @@ -807,6 +807,11 @@ export class TestState { map: { [marker: string]: { completions: _.FourSlashCompletionItem[]; + moduleContext?: { + lastKnownModule?: string; + lastKnownMemberName?: string; + unknownMemberName?: string; + }; }; } ): Promise { @@ -820,7 +825,7 @@ export class TestState { const filePath = marker.fileName; const expectedCompletions = map[markerName].completions; - const completionPosition = this._convertOffsetToPosition(filePath, marker.position); + const completionPosition = this.convertOffsetToPosition(filePath, marker.position); const result = await this.workspace.serviceInstance.getCompletionsForPosition( filePath, @@ -829,31 +834,33 @@ export class TestState { CancellationToken.None ); - if (result) { + if (result?.completionList) { if (verifyMode === 'exact') { - if (result.items.length !== expectedCompletions.length) { + if (result.completionList.items.length !== expectedCompletions.length) { assert.fail( `Expected ${expectedCompletions.length} items but received ${ - result.items.length - }. Actual completions:\n${stringify(result.items.map((r) => r.label))}` + result.completionList.items.length + }. Actual completions:\n${stringify(result.completionList.items.map((r) => r.label))}` ); } } for (let i = 0; i < expectedCompletions.length; i++) { const expected = expectedCompletions[i]; - const actualIndex = result.items.findIndex((a) => a.label === expected.label); + const actualIndex = result.completionList.items.findIndex((a) => a.label === expected.label); if (actualIndex >= 0) { if (verifyMode === 'excluded') { // we're not supposed to find the completions passed to the test assert.fail( `Completion item with label "${ expected.label - }" unexpected. Actual completions:\n${stringify(result.items.map((r) => r.label))}` + }" unexpected. Actual completions:\n${stringify( + result.completionList.items.map((r) => r.label) + )}` ); } - const actual: CompletionItem = result.items[actualIndex]; + const actual: CompletionItem = result.completionList.items[actualIndex]; assert.equal(actual.label, expected.label); if (expectedCompletions[i].documentation !== undefined) { if (actual.documentation === undefined) { @@ -870,28 +877,51 @@ export class TestState { } } - result.items.splice(actualIndex, 1); + result.completionList.items.splice(actualIndex, 1); } else { if (verifyMode === 'included' || verifyMode === 'exact') { // we're supposed to find all items passed to the test assert.fail( `Completion item with label "${ expected.label - }" expected. Actual completions:\n${stringify(result.items.map((r) => r.label))}` + }" expected. Actual completions:\n${stringify( + result.completionList.items.map((r) => r.label) + )}` ); } } } if (verifyMode === 'exact') { - if (result.items.length !== 0) { + if (result.completionList.items.length !== 0) { // we removed every item we found, there should not be any remaining - assert.fail(`Completion items unexpected: ${stringify(result.items.map((r) => r.label))}`); + assert.fail( + `Completion items unexpected: ${stringify(result.completionList.items.map((r) => r.label))}` + ); } } } else { assert.fail('Failed to get completions'); } + + if (map[markerName].moduleContext !== undefined && result?.moduleContext !== undefined) { + const expectedModule = map[markerName].moduleContext?.lastKnownModule; + const expectedType = map[markerName].moduleContext?.lastKnownMemberName; + const expectedName = map[markerName].moduleContext?.unknownMemberName; + if ( + result?.moduleContext?.lastKnownModule !== expectedModule || + result?.moduleContext?.lastKnownMemberName !== expectedType || + result?.moduleContext?.unknownMemberName !== expectedName + ) { + assert.fail( + `Expected completion results moduleContext with \n lastKnownModule: "${expectedModule}"\n lastKnownMemberName: "${expectedType}"\n unknownMemberName: "${expectedName}"\n Actual moduleContext:\n lastKnownModule: "${ + result.moduleContext?.lastKnownModule ?? '' + }"\n lastKnownMemberName: "${ + result.moduleContext?.lastKnownMemberName ?? '' + }\n unknownMemberName: "${result.moduleContext?.unknownMemberName ?? ''}" ` + ); + } + } } } @@ -917,7 +947,7 @@ export class TestState { } const expected = map[name]; - const position = this._convertOffsetToPosition(fileName, marker.position); + const position = this.convertOffsetToPosition(fileName, marker.position); const actual = this.program.getSignatureHelpForPosition(fileName, position, CancellationToken.None); @@ -973,7 +1003,7 @@ export class TestState { const expected = map[name].references; - const position = this._convertOffsetToPosition(fileName, marker.position); + const position = this.convertOffsetToPosition(fileName, marker.position); const actual = this.program.getReferencesForPosition(fileName, position, true, CancellationToken.None); assert.equal(actual?.length ?? 0, expected.length); @@ -1011,7 +1041,7 @@ export class TestState { const expected = map[name].references; - const position = this._convertOffsetToPosition(fileName, marker.position); + const position = this.convertOffsetToPosition(fileName, marker.position); const actual = this.program.getDocumentHighlight(fileName, position, CancellationToken.None); assert.equal(actual?.length ?? 0, expected.length); @@ -1044,7 +1074,7 @@ export class TestState { const expected = map[name].definitions; - const position = this._convertOffsetToPosition(fileName, marker.position); + const position = this.convertOffsetToPosition(fileName, marker.position); const actual = this.program.getDefinitionsForPosition(fileName, position, CancellationToken.None); assert.equal(actual?.length ?? 0, expected.length); @@ -1073,7 +1103,7 @@ export class TestState { const expected = map[name]; - const position = this._convertOffsetToPosition(fileName, marker.position); + const position = this.convertOffsetToPosition(fileName, marker.position); const actual = this.program.renameSymbolAtPosition( fileName, position, @@ -1144,13 +1174,13 @@ export class TestState { return convertPositionToOffset(position, lines)!; } - private _convertOffsetToPosition(fileName: string, offset: number): Position { + protected convertOffsetToPosition(fileName: string, offset: number): Position { const lines = this._getTextRangeCollection(fileName); return convertOffsetToPosition(offset, lines); } - private _convertOffsetsToRange(fileName: string, startOffset: number, endOffset: number): PositionRange { + protected convertOffsetsToRange(fileName: string, startOffset: number, endOffset: number): PositionRange { const lines = this._getTextRangeCollection(fileName); return { @@ -1175,7 +1205,7 @@ export class TestState { return tokenizer.tokenize(fileContents).lines; } - private _raiseError(message: string): never { + protected raiseError(message: string): never { throw new Error(this._messageAtLastKnownMarker(message)); } @@ -1211,7 +1241,7 @@ export class TestState { return text.replace(/\s/g, ''); } - private _createMultiMap(values?: T[], getKey?: (t: T) => string): MultiMap { + protected createMultiMap(values?: T[], getKey?: (t: T) => string): MultiMap { const map = new Map() as MultiMap; map.add = multiMapAdd; map.remove = multiMapRemove; @@ -1256,7 +1286,7 @@ export class TestState { private _getOnlyRange() { const ranges = this.getRanges(); if (ranges.length !== 1) { - this._raiseError('Exactly one range should be specified in the test file.'); + this.raiseError('Exactly one range should be specified in the test file.'); } return ranges[0]; @@ -1272,7 +1302,7 @@ export class TestState { private _verifyTextMatches(actualText: string, includeWhitespace: boolean, expectedText: string) { const removeWhitespace = (s: string): string => (includeWhitespace ? s : this._removeWhitespace(s)); if (removeWhitespace(actualText) !== removeWhitespace(expectedText)) { - this._raiseError( + this.raiseError( `Actual range text doesn't match expected text.\n${this._showTextDiff(expectedText, actualText)}` ); } @@ -1316,7 +1346,7 @@ export class TestState { // Get the text of the entire line the caret is currently at private _getCurrentLineContent() { return this._getLineContent( - this._convertOffsetToPosition(this.activeFile.fileName, this.currentCaretPosition).line + this.convertOffsetToPosition(this.activeFile.fileName, this.currentCaretPosition).line ); } @@ -1366,7 +1396,7 @@ export class TestState { } private _getLineColStringAtPosition(position: number, file: FourSlashFile = this.activeFile) { - const pos = this._convertOffsetToPosition(file.fileName, position); + const pos = this.convertOffsetToPosition(file.fileName, position); return `line ${pos.line + 1}, col ${pos.character}`; } @@ -1426,7 +1456,7 @@ export class TestState { }; return [filePath, value] as [string, typeof value]; } else { - this._raiseError(`Source file not found for ${this._files[index]}`); + this.raiseError(`Source file not found for ${this._files[index]}`); } }); @@ -1472,8 +1502,8 @@ export class TestState { private _getCodeActions(range: Range) { const file = range.fileName; const textRange = { - start: this._convertOffsetToPosition(file, range.pos), - end: this._convertOffsetToPosition(file, range.end), + start: this.convertOffsetToPosition(file, range.pos), + end: this.convertOffsetToPosition(file, range.end), }; return this._hostSpecificFeatures.getCodeActionsForPosition( @@ -1494,7 +1524,7 @@ export class TestState { const actual = this.fs.readFileSync(normalizedFilePath, 'utf8'); if (actual !== expected) { - this._raiseError( + this.raiseError( `doesn't contain expected result: ${stringify(expected)}, actual: ${stringify(actual)}` ); } diff --git a/packages/vscode-pyright/package.json b/packages/vscode-pyright/package.json index 279f60d65..06f42fefd 100644 --- a/packages/vscode-pyright/package.json +++ b/packages/vscode-pyright/package.json @@ -82,7 +82,7 @@ }, "python.analysis.stubPath": { "type": "string", - "default": "", + "default": "typings", "description": "Path to directory containing custom type stub files.", "scope": "resource" },