Push pylance changes to pyright (#5248)

* Push pylance changes to pyright

* Update packages/pyright-internal/src/backgroundThreadBase.ts

Co-authored-by: Erik De Bonte <erikd@microsoft.com>

---------

Co-authored-by: Erik De Bonte <erikd@microsoft.com>
This commit is contained in:
Heejae Chang 2023-06-07 15:03:40 -07:00 committed by GitHub
parent 51d2ff8beb
commit 73165c50c2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 5058 additions and 4234 deletions

22
.vscode/tasks.json vendored
View File

@ -18,17 +18,17 @@
"fileLocation": "absolute",
"pattern": [
{
"regexp": "\\[tsl\\] (ERROR|WARNING) in (.*)?\\((\\d+),(\\d+)\\)",
"severity": 1,
"file": 2,
"line": 3,
"column": 4
},
{
"regexp": "\\s*TS(\\d+):\\s*(.*)$",
"code": 1,
"message": 2
}
"regexp": "\\[tsl\\] (ERROR|WARNING) in (.*)?\\((\\d+),(\\d+)\\)",
"severity": 1,
"file": 2,
"line": 3,
"column": 4
},
{
"regexp": "\\s*TS(\\d+):\\s*(.*)$",
"code": 1,
"message": 2
}
],
"background": {
"activeOnStart": true,

View File

@ -39,4 +39,4 @@
"typescript": "~4.4.4",
"yargs": "^16.2.0"
}
}
}

File diff suppressed because it is too large Load Diff

View File

@ -42,4 +42,4 @@
"ts-jest": "^27.1.5",
"typescript": "~4.4.4"
}
}
}

View File

@ -1678,7 +1678,7 @@ export class Program {
const implicitPath = nextImplicitImport.sourceFile.getFilePath();
if (implicitSet.has(implicitPath)) {
// We've found a cycle. Break out of the loop.
debug.fail(`Found a cycle in implicit imports files for ${implicitPath}`);
debug.fail(`Found a cycle in implicit imports files`);
}
implicitSet.add(implicitPath);
@ -1891,20 +1891,14 @@ export class Program {
return false;
}
this._bindFile(fileToCheck);
if (this._preCheckCallback) {
const parseResults = fileToCheck.sourceFile.getParseResults();
if (parseResults) {
this._preCheckCallback(parseResults, this._evaluator!);
}
}
if (!this._disableChecker) {
// For ipython, make sure we check all its dependent files first since
// their results can affect this file's result.
let dependentFiles: ParseResults[] | undefined = undefined;
if (fileToCheck.sourceFile.getIPythonMode() === IPythonMode.CellDocs) {
// Parse file to get up to date dependency graph.
this._parseFile(fileToCheck);
dependentFiles = [];
const importedByFiles = collectImportedByFiles(fileToCheck);
for (const file of importedByFiles) {
@ -1930,6 +1924,15 @@ export class Program {
}
}
this._bindFile(fileToCheck);
if (this._preCheckCallback) {
const parseResults = fileToCheck.sourceFile.getParseResults();
if (parseResults) {
this._preCheckCallback(parseResults, this._evaluator!);
}
}
const execEnv = this._configOptions.findExecEnvironment(fileToCheck.sourceFile.getFilePath());
fileToCheck.sourceFile.check(
this.configOptions,

View File

@ -264,8 +264,8 @@ export abstract class BackgroundAnalysisRunnerBase extends BackgroundThreadBase
protected importResolver: ImportResolver;
protected logTracker: LogTracker;
protected constructor() {
super(workerData as InitializationData);
protected constructor(fileSystem?: FileSystem) {
super(workerData as InitializationData, fileSystem);
// Stash the base directory into a global variable.
const data = workerData as InitializationData;

View File

@ -10,23 +10,48 @@ import { MessagePort, parentPort, TransferListItem } from 'worker_threads';
import { OperationCanceledException, setCancellationFolderName } from './common/cancellationUtils';
import { ConfigOptions } from './common/configOptions';
import { LogLevel } from './common/console';
import { ConsoleInterface, LogLevel } from './common/console';
import * as debug from './common/debug';
import { FileSystem } from './common/fileSystem';
import { FileSpec } from './common/pathUtils';
import { createFromRealFileSystem } from './common/realFileSystem';
import { PyrightFileSystem } from './pyrightFileSystem';
export class BackgroundConsole implements ConsoleInterface {
// We always generate logs in the background. For the foreground,
// we'll decide based on user setting whether.
get level() {
return LogLevel.Log;
}
log(msg: string) {
this.post(LogLevel.Log, msg);
}
info(msg: string) {
this.post(LogLevel.Info, msg);
}
warn(msg: string) {
this.post(LogLevel.Warn, msg);
}
error(msg: string) {
this.post(LogLevel.Error, msg);
}
protected post(level: LogLevel, msg: string) {
parentPort?.postMessage({ requestType: 'log', data: { level: level, message: msg } });
}
}
export class BackgroundThreadBase {
private _console = new BackgroundConsole();
protected fs: FileSystem;
protected constructor(data: InitializationData) {
protected constructor(data: InitializationData, fileSystem?: FileSystem) {
setCancellationFolderName(data.cancellationFolderName);
// Stash the base directory into a global variable.
(global as any).__rootDirectory = data.rootDirectory;
this.fs = new PyrightFileSystem(createFromRealFileSystem(this.getConsole()));
this.fs = fileSystem ?? new PyrightFileSystem(createFromRealFileSystem(this.getConsole()));
}
protected log(level: LogLevel, msg: string) {
@ -34,23 +59,7 @@ export class BackgroundThreadBase {
}
protected getConsole() {
return {
log: (msg: string) => {
this.log(LogLevel.Log, msg);
},
info: (msg: string) => {
this.log(LogLevel.Info, msg);
},
warn: (msg: string) => {
this.log(LogLevel.Warn, msg);
},
error: (msg: string) => {
this.log(LogLevel.Error, msg);
},
// We always generate logs in the background. For the foreground,
// we'll decide decide based on user setting whether.
level: LogLevel.Log,
};
return this._console;
}
protected handleShutdown() {

View File

@ -86,7 +86,10 @@ export interface ReadOnlyFileSystem {
export interface FileSystem extends ReadOnlyFileSystem {
mkdirSync(path: string, options?: MkDirOptions): void;
writeFileSync(path: string, data: string | Buffer, encoding: BufferEncoding | null): void;
unlinkSync(path: string): void;
rmdirSync(path: string): void;
createFileSystemWatcher(paths: string[], listener: FileWatcherEventHandler): FileWatcher;
createReadStream(path: string): fs.ReadStream;
createWriteStream(path: string): fs.WriteStream;
@ -95,7 +98,6 @@ export interface FileSystem extends ReadOnlyFileSystem {
// The directory returned by tmpdir must exist and be the same each time tmpdir is called.
tmpdir(): string;
tmpfile(options?: TmpfileOptions): string;
dispose(): void;
}

View File

@ -288,6 +288,10 @@ class RealFileSystem implements FileSystem {
return stat;
}
rmdirSync(path: string): void {
yarnFS.rmdirSync(path);
}
unlinkSync(path: string) {
yarnFS.unlinkSync(path);
}

View File

@ -173,6 +173,7 @@ export interface LanguageServerInterface {
readonly window: WindowInterface;
readonly supportAdvancedEdits: boolean;
getWorkspaces(): Promise<Workspace[]>;
getWorkspaceForFile(filePath: string): Promise<Workspace>;
getSettings(workspace: Workspace): Promise<ServerSettings>;
createBackgroundAnalysis(serviceId: string): BackgroundAnalysisBase | undefined;
@ -200,7 +201,7 @@ export interface WorkspaceServices {
backgroundAnalysis: BackgroundAnalysisBase | undefined;
}
interface ClientCapabilities {
export interface ClientCapabilities {
hasConfigurationCapability: boolean;
hasVisualStudioExtensionsCapability: boolean;
hasWorkspaceFoldersCapability: boolean;
@ -376,7 +377,7 @@ export abstract class LanguageServerBase implements LanguageServerInterface {
// the extension directory. Otherwise the execution of
// python can have unintended and surprising results.
const moduleDirectory = this.fs.getModulePath();
if (moduleDirectory) {
if (moduleDirectory && this.fs.existsSync(moduleDirectory)) {
this.fs.chdir(moduleDirectory);
}
@ -437,7 +438,7 @@ export abstract class LanguageServerBase implements LanguageServerInterface {
return service;
}
async test_getWorkspaces() {
async getWorkspaces(): Promise<Workspace[]> {
const workspaces = [...this.workspaceFactory.items()];
for (const workspace of workspaces) {
await workspace.isInitialized.promise;

View File

@ -34,6 +34,102 @@ export enum DefinitionFilter {
PreferStubs = 'preferStubs',
}
export function addDeclarationsToDefinitions(
evaluator: TypeEvaluator,
sourceMapper: SourceMapper,
declarations: Declaration[] | undefined,
definitions: DocumentRange[]
) {
if (!declarations) {
return;
}
declarations.forEach((decl) => {
let resolvedDecl = evaluator.resolveAliasDeclaration(decl, /* resolveLocalNames */ true, {
allowExternallyHiddenAccess: true,
});
if (!resolvedDecl || !resolvedDecl.path) {
return;
}
// If the decl is an unresolved import, skip it.
if (resolvedDecl.type === DeclarationType.Alias && resolvedDecl.isUnresolved) {
return;
}
// If the resolved decl is still an alias, it means it
// resolved to a module. We need to apply loader actions
// to determine its path.
if (
resolvedDecl.type === DeclarationType.Alias &&
resolvedDecl.symbolName &&
resolvedDecl.submoduleFallback &&
resolvedDecl.submoduleFallback.path
) {
resolvedDecl = resolvedDecl.submoduleFallback;
}
_addIfUnique(definitions, {
path: resolvedDecl.path,
range: resolvedDecl.range,
});
if (isFunctionDeclaration(resolvedDecl)) {
// Handle overloaded function case
const functionType = evaluator.getTypeForDeclaration(resolvedDecl)?.type;
if (functionType && isOverloadedFunction(functionType)) {
for (const overloadDecl of functionType.overloads.map((o) => o.details.declaration).filter(isDefined)) {
_addIfUnique(definitions, {
path: overloadDecl.path,
range: overloadDecl.range,
});
}
}
}
if (!isStubFile(resolvedDecl.path)) {
return;
}
if (resolvedDecl.type === DeclarationType.Alias) {
// Add matching source module
sourceMapper
.findModules(resolvedDecl.path)
.map((m) => getFileInfo(m)?.filePath)
.filter(isDefined)
.forEach((f) => _addIfUnique(definitions, _createModuleEntry(f)));
return;
}
const implDecls = sourceMapper.findDeclarations(resolvedDecl);
for (const implDecl of implDecls) {
if (implDecl && implDecl.path) {
_addIfUnique(definitions, {
path: implDecl.path,
range: implDecl.range,
});
}
}
});
}
export function filterDefinitions(filter: DefinitionFilter, definitions: DocumentRange[]) {
if (filter === DefinitionFilter.All) {
return definitions;
}
// If go-to-declaration is supported, attempt to only show only pyi files in go-to-declaration
// and none in go-to-definition, unless filtering would produce an empty list.
const preferStubs = filter === DefinitionFilter.PreferStubs;
const wantedFile = (v: DocumentRange) => preferStubs === isStubFile(v.path);
if (definitions.find(wantedFile)) {
return definitions.filter(wantedFile);
}
return definitions;
}
class DefinitionProviderBase {
protected constructor(
protected readonly sourceMapper: SourceMapper,
@ -78,88 +174,11 @@ class DefinitionProviderBase {
return undefined;
}
if (this._filter === DefinitionFilter.All) {
return definitions;
}
// If go-to-declaration is supported, attempt to only show only pyi files in go-to-declaration
// and none in go-to-definition, unless filtering would produce an empty list.
const preferStubs = this._filter === DefinitionFilter.PreferStubs;
const wantedFile = (v: DocumentRange) => preferStubs === isStubFile(v.path);
if (definitions.find(wantedFile)) {
return definitions.filter(wantedFile);
}
return definitions;
return filterDefinitions(this._filter, definitions);
}
protected resolveDeclarations(declarations: Declaration[] | undefined, definitions: DocumentRange[]) {
if (declarations) {
declarations.forEach((decl) => {
let resolvedDecl = this.evaluator.resolveAliasDeclaration(decl, /* resolveLocalNames */ true, {
allowExternallyHiddenAccess: true,
});
if (resolvedDecl && resolvedDecl.path) {
// If the decl is an unresolved import, skip it.
if (resolvedDecl.type === DeclarationType.Alias && resolvedDecl.isUnresolved) {
return;
}
// If the resolved decl is still an alias, it means it
// resolved to a module. We need to apply loader actions
// to determine its path.
if (
resolvedDecl.type === DeclarationType.Alias &&
resolvedDecl.symbolName &&
resolvedDecl.submoduleFallback &&
resolvedDecl.submoduleFallback.path
) {
resolvedDecl = resolvedDecl.submoduleFallback;
}
_addIfUnique(definitions, {
path: resolvedDecl.path,
range: resolvedDecl.range,
});
if (isFunctionDeclaration(resolvedDecl)) {
// Handle overloaded function case
const functionType = this.evaluator.getTypeForDeclaration(resolvedDecl)?.type;
if (functionType && isOverloadedFunction(functionType)) {
for (const overloadDecl of functionType.overloads
.map((o) => o.details.declaration)
.filter(isDefined)) {
_addIfUnique(definitions, {
path: overloadDecl.path,
range: overloadDecl.range,
});
}
}
}
if (isStubFile(resolvedDecl.path)) {
if (resolvedDecl.type === DeclarationType.Alias) {
// Add matching source module
this.sourceMapper
.findModules(resolvedDecl.path)
.map((m) => getFileInfo(m)?.filePath)
.filter(isDefined)
.forEach((f) => _addIfUnique(definitions, _createModuleEntry(f)));
} else {
const implDecls = this.sourceMapper.findDeclarations(resolvedDecl);
for (const implDecl of implDecls) {
if (implDecl && implDecl.path) {
_addIfUnique(definitions, {
path: implDecl.path,
range: implDecl.range,
});
}
}
}
}
}
});
}
addDeclarationsToDefinitions(this.evaluator, this.sourceMapper, declarations, definitions);
}
}

View File

@ -46,23 +46,87 @@ import {
getTypeForToolTip,
} from './tooltipUtils';
export interface HoverTextPart {
python?: boolean;
text: string;
}
export interface HoverResults {
parts: HoverTextPart[];
range: Range;
}
export function convertHoverResults(hoverResults: HoverResults | null, format: MarkupKind): Hover | null {
if (!hoverResults) {
return null;
}
const markupString = hoverResults.parts
.map((part) => {
if (part.python) {
if (format === MarkupKind.Markdown) {
return '```python\n' + part.text + '\n```\n';
} else if (format === MarkupKind.PlainText) {
return part.text + '\n\n';
} else {
fail(`Unsupported markup type: ${format}`);
}
}
return part.text;
})
.join('')
.trimEnd();
return {
contents: {
kind: format,
value: markupString,
},
range: hoverResults.range,
};
}
export function addDocumentationResultsPart(docString: string | undefined, format: MarkupKind, parts: HoverTextPart[]) {
if (!docString) {
return;
}
if (format === MarkupKind.Markdown) {
const markDown = convertDocStringToMarkdown(docString);
if (parts.length > 0 && markDown.length > 0) {
parts.push({ text: '---\n' });
}
parts.push({ text: markDown, python: false });
return;
}
if (format === MarkupKind.PlainText) {
parts.push({ text: convertDocStringToPlainText(docString), python: false });
return;
}
fail(`Unsupported markup type: ${format}`);
}
export class HoverProvider {
private readonly _parseResults: ParseResults | undefined;
private readonly _sourceMapper: SourceMapper;
constructor(
private _program: ProgramView,
private _filePath: string,
private _position: Position,
private _format: MarkupKind,
private _token: CancellationToken
private readonly _program: ProgramView,
private readonly _filePath: string,
private readonly _position: Position,
private readonly _format: MarkupKind,
private readonly _token: CancellationToken
) {
this._parseResults = this._program.getParseResults(this._filePath);
this._sourceMapper = this._program.getSourceMapper(this._filePath, this._token, /* mapCompiled */ true);
}
getHover(): Hover | null {
return this._convertHoverResults(this._getHoverResult());
return convertHoverResults(this._getHoverResult(), this._format);
}
static getPrimaryDeclaration(declarations: Declaration[]) {
@ -469,30 +533,9 @@ export class HoverProvider {
const docString = getDocumentationPartsForTypeAndDecl(this._sourceMapper, type, resolvedDecl, this._evaluator, {
name,
});
if (docString) {
this._addDocumentationResultsPart(parts, docString);
return true;
}
return false;
}
private _addDocumentationResultsPart(parts: HoverTextPart[], docString?: string) {
if (docString) {
if (this._format === MarkupKind.Markdown) {
const markDown = convertDocStringToMarkdown(docString);
if (parts.length > 0 && markDown.length > 0) {
parts.push({ text: '---\n' });
}
this._addResultsPart(parts, markDown);
} else if (this._format === MarkupKind.PlainText) {
this._addResultsPart(parts, convertDocStringToPlainText(docString));
} else {
fail(`Unsupported markup type: ${this._format}`);
}
}
addDocumentationResultsPart(docString, this._format, parts);
return !!docString;
}
private _addResultsPart(parts: HoverTextPart[], text: string, python = false) {
@ -501,44 +544,4 @@ export class HoverProvider {
text,
});
}
private _convertHoverResults(hoverResults: HoverResults | null): Hover | null {
if (!hoverResults) {
return null;
}
const markupString = hoverResults.parts
.map((part) => {
if (part.python) {
if (this._format === MarkupKind.Markdown) {
return '```python\n' + part.text + '\n```\n';
} else if (this._format === MarkupKind.PlainText) {
return part.text + '\n\n';
} else {
fail(`Unsupported markup type: ${this._format}`);
}
}
return part.text;
})
.join('')
.trimEnd();
return {
contents: {
kind: this._format,
value: markupString,
},
range: hoverResults.range,
};
}
}
interface HoverTextPart {
python?: boolean;
text: string;
}
interface HoverResults {
parts: HoverTextPart[];
range: Range;
}

View File

@ -77,6 +77,10 @@ export class PyrightFileSystem
this.realFS.writeFileSync(this.getOriginalPath(path), data, encoding);
}
override rmdirSync(path: string): void {
this.realFS.rmdirSync(this.getOriginalPath(path));
}
override unlinkSync(path: string): void {
this.realFS.unlinkSync(this.getOriginalPath(path));
}

View File

@ -95,6 +95,10 @@ export class ReadOnlyAugmentedFileSystem implements FileSystem {
return this.realFS.statSync(this.getOriginalPath(path));
}
rmdirSync(path: string): void {
throw new Error('Operation is not allowed.');
}
unlinkSync(path: string): void {
throw new Error('Operation is not allowed.');
}

View File

@ -108,10 +108,15 @@ export class TestLanguageService implements LanguageServerInterface {
searchPathsToWatch: [],
};
}
decodeTextDocumentUri(uriString: string): string {
return this._uriParser.decodeTextDocumentUri(uriString);
}
getWorkspaces(): Promise<Workspace[]> {
return Promise.resolve([this._workspace, this._defaultWorkspace]);
}
getWorkspaceForFile(filePath: string): Promise<Workspace> {
if (filePath.startsWith(this._workspace.rootPath)) {
return Promise.resolve(this._workspace);

View File

@ -14,6 +14,7 @@ import {
FileSystem,
FileWatcher,
FileWatcherEventHandler,
FileWatcherEventType,
MkDirOptions,
TmpfileOptions,
} from '../../../common/fileSystem';
@ -31,6 +32,25 @@ export interface DiffOptions {
includeChangedFileWithSameContent?: boolean;
}
export class TestFileSystemWatcher implements FileWatcher {
private _paths: string[] = [];
constructor(paths: string[], private _listener: FileWatcherEventHandler) {
this._paths = paths.map((p) => pathUtil.normalizePath(p));
}
close() {
// Do nothing.
}
fireFileChange(path: string, eventType: FileWatcherEventType): boolean {
const normalized = pathUtil.normalizePath(path);
if (this._paths.some((p) => normalized.startsWith(p))) {
this._listener(eventType, normalized);
return true;
}
return false;
}
}
/**
* Represents a virtual POSIX-like file system.
*/
@ -53,8 +73,12 @@ export class TestFileSystem implements FileSystem {
private _shadowRoot: TestFileSystem | undefined;
private _dirStack: string[] | undefined;
private _tmpfileCounter = 0;
private _watchers: TestFileSystemWatcher[] = [];
private _id: number;
private static _nextId = 1;
constructor(ignoreCase: boolean, options: FileSystemOptions = {}) {
this._id = TestFileSystem._nextId++;
const { time = -1, files, meta } = options;
this.ignoreCase = ignoreCase;
this.stringComparer = this.ignoreCase
@ -315,11 +339,17 @@ export class TestFileSystem implements FileSystem {
}
createFileSystemWatcher(paths: string[], listener: FileWatcherEventHandler): FileWatcher {
return {
close: () => {
/* left empty */
},
};
const watcher = new TestFileSystemWatcher(paths, listener);
this._watchers.push(watcher);
return watcher;
}
fireFileWatcherEvent(path: string, event: FileWatcherEventType) {
for (const watcher of this._watchers) {
if (watcher.fireFileChange(path, event)) {
break;
}
}
}
getModulePath(): string {

View File

@ -296,11 +296,26 @@ export class WorkspaceFactory {
return workspace;
}
getWorkspaceForFileSync(filePath: string, pythonPath: string | undefined): Workspace {
// Find or create best match.
return this._getOrCreateBestWorkspaceFileSync(filePath, pythonPath);
}
async getContainingWorkspacesForFile(filePath: string): Promise<Workspace[]> {
// Wait for all workspaces to be initialized before attempting to find the best workspace. Otherwise
// the list of files won't be complete and the `contains` check might fail.
await Promise.all(this.items().map((w) => w.isInitialized.promise));
// Find or create best match.
const workspaces = this.getContainingWorkspacesForFileSync(filePath);
// The workspaces may have just been created, wait for them all to be initialized
await Promise.all(workspaces.map((w) => w.isInitialized.promise));
return workspaces;
}
getContainingWorkspacesForFileSync(filePath: string): Workspace[] {
// All workspaces that track the file should be considered.
let workspaces = this.items().filter((w) => w.service.isTracked(filePath));
@ -314,9 +329,6 @@ export class WorkspaceFactory {
workspaces = workspaces.filter((w) => w.pythonPathKind === WorkspacePythonPathKind.Immutable);
}
// The workspaces may have just been created, wait for them all to be initialized
await Promise.all(workspaces.map((w) => w.isInitialized.promise));
return workspaces;
}

File diff suppressed because it is too large Load Diff

View File

@ -25,13 +25,13 @@
"devDependencies": {
"@types/copy-webpack-plugin": "^10.1.0",
"@types/node": "^17.0.45",
"copy-webpack-plugin": "^10.2.4",
"copy-webpack-plugin": "^11.0.0",
"esbuild-loader": "^3.0.1",
"shx": "^0.3.4",
"ts-loader": "^9.4.2",
"ts-loader": "^9.4.3",
"typescript": "~4.4.4",
"webpack": "^5.82.0",
"webpack-cli": "^4.10.0"
"webpack": "^5.85.0",
"webpack-cli": "^5.1.1"
},
"files": [
"/dist",
@ -42,4 +42,4 @@
"pyright": "index.js",
"pyright-langserver": "langserver.index.js"
}
}
}

View File

@ -54,7 +54,7 @@ module.exports = (_, { mode }) => {
{
// Transform pre-compiled JS files to use syntax available in Node 12+.
// esbuild is fast, so let it run on all JS files rather than matching
// only known-bad libs. ts-loader does this for us for TypeScript files.
// only known-bad libs.
test: /\.js$/,
loader: 'esbuild-loader',
options: {

File diff suppressed because it is too large Load Diff

View File

@ -959,14 +959,14 @@
"@types/copy-webpack-plugin": "^10.1.0",
"@types/node": "^17.0.45",
"@types/vscode": "~1.78.0",
"copy-webpack-plugin": "^10.2.4",
"copy-webpack-plugin": "^11.0.0",
"detect-indent": "^6.1.0",
"esbuild-loader": "^3.0.1",
"shx": "^0.3.4",
"ts-loader": "^9.4.2",
"ts-loader": "^9.4.3",
"typescript": "~4.4.4",
"vsce": "^2.7.0",
"webpack": "^5.82.0",
"webpack-cli": "^4.10.0"
"webpack": "^5.85.0",
"webpack-cli": "^5.1.1"
}
}
}