microsoft/pyright
1
1
Fork 0
mirror of https://github.com/microsoft/pyright.git synced 2024-11-09 13:35:59 +03:00
Code Issues Projects Releases Wiki Activity

Added support for py.typed files to distinguish between packages that… (#1007)

Browse Source 
* Added support for py.typed files to distinguish between packages that claim to have inlined types versus those that don't. Also added support for "partial" stub packages (a "-stub" package that has a py.typed file that includes the line "partial\n" as described in PEP 561).

* Fixed style issue.

* Removed unnecessary truthy tests for results returned by resolveAbsoluteImport.

Co-authored-by: Eric Traut <erictr@microsoft.com>
This commit is contained in:
Eric Traut 2020-09-04 14:55:37 -07:00 committed by GitHub
parent 6e9f8795ea
commit 97d754e1b5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 161 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
packages/pyright-internal/src/analyzer/binder.ts
View File

@ -278,7 +278,11 @@ export class Binder extends ParseTreeWalker {
);
} else {
// Source found, but type stub is missing
if (!importResult.isStubFile && importResult.importType === ImportType.ThirdParty) {
if (
!importResult.isStubFile &&
importResult.importType === ImportType.ThirdParty &&
!importResult.isPyTypedPresent
) {
const diagnostic = this._addDiagnostic(
this._fileInfo.diagnosticRuleSet.reportMissingTypeStubs,
DiagnosticRule.reportMissingTypeStubs,

147
packages/pyright-internal/src/analyzer/importResolver.ts
View File

@ -54,6 +54,7 @@ type CachedImportResults = Map<string, ImportResult>;
const supportedNativeLibExtensions = ['.pyd', '.so', '.dylib'];
const supportedFileExtensions = ['.py', '.pyi', ...supportedNativeLibExtensions];
const stubsSuffix = '-stubs';
const pyTypedFileName = 'py.typed';
// Should we allow partial resolution for third-party packages? Some use tricks
// to populate their package namespaces, so we might be able to partially resolve
@ -62,6 +63,11 @@ const stubsSuffix = '-stubs';
// errors when these partial-resolutions fail.
const allowPartialResolutionForThirdPartyPackages = false;
interface PyTypedInfo {
isPyTypedPresent: boolean;
isPartiallyTyped: boolean;
}
export class ImportResolver {
private _configOptions: ConfigOptions;
private _cachedPythonSearchPaths = new Map<string, string[]>();
@ -465,9 +471,9 @@ export class ImportResolver {
importFailureInfo: string[],
allowPartial = false,
allowNativeLib = false,
allowStubPackages = false,
useStubPackage = false,
allowPyi = true
): ImportResult | undefined {
): ImportResult {
importFailureInfo.push(`Attempting to resolve using root path '${rootPath}'`);
// Starting at the specified path, walk the file system to find the
@ -478,6 +484,7 @@ export class ImportResolver {
let isStubFile = false;
let isNativeLib = false;
let implicitImports: ImplicitImport[] = [];
let packageDirectory: string | undefined;
// Handle the "from . import XXX" case.
if (moduleDescriptor.nameParts.length === 0) {
@ -501,28 +508,21 @@ export class ImportResolver {
implicitImports = this._findImplicitImports(dirPath, [pyFilePath, pyiFilePath]);
} else {
for (let i = 0; i < moduleDescriptor.nameParts.length; i++) {
const isFirstPart = i === 0;
const isLastPart = i === moduleDescriptor.nameParts.length - 1;
dirPath = combinePaths(dirPath, moduleDescriptor.nameParts[i]);
let foundDirectory = false;
if (allowPyi && allowStubPackages) {
// PEP 561 indicates that package authors can ship their stubs
// separately from their package implementation by appending
// the string '-stubs' to its top-level directory name. We'll
// look there first.
const stubsDirPath = dirPath + stubsSuffix;
foundDirectory =
this.fileSystem.existsSync(stubsDirPath) && isDirectory(this.fileSystem, stubsDirPath);
if (foundDirectory) {
dirPath = stubsDirPath;
}
if (useStubPackage && isFirstPart) {
dirPath += stubsSuffix;
}
if (!foundDirectory) {
foundDirectory = this.fileSystem.existsSync(dirPath) && isDirectory(this.fileSystem, dirPath);
}
const foundDirectory = this.fileSystem.existsSync(dirPath) && isDirectory(this.fileSystem, dirPath);
if (foundDirectory) {
if (isFirstPart) {
packageDirectory = dirPath;
}
if (!isLastPart) {
// We are not at the last part, and we found a directory,
// so continue to look for the next part.
@ -638,6 +638,7 @@ export class ImportResolver {
isStubFile,
isNativeLib,
implicitImports,
packageDirectory,
};
}
@ -661,6 +662,40 @@ export class ImportResolver {
return undefined;
}
private _getPyTypedInfo(dirPath: string): PyTypedInfo {
let isPyTypedPresent = false;
let isPartiallyTyped = false;
if (this.fileSystem.existsSync(dirPath) && isDirectory(this.fileSystem, dirPath)) {
const pyTypedPath = combinePaths(dirPath, pyTypedFileName);
if (this.fileSystem.existsSync(dirPath) && isFile(this.fileSystem, pyTypedPath)) {
isPyTypedPresent = true;
// Read the contents of the file as text.
const fileStats = this.fileSystem.statSync(pyTypedPath);
// Do a quick sanity check on the size before we attempt to read it. This
// file should always be really small - typically zero bytes in length.
if (fileStats.size > 0 && fileStats.size < 16 * 1024) {
const pyTypedContents = this.fileSystem.readFileSync(pyTypedPath, 'utf8');
// PEP 561 doesn't specify the format of "py.typed" in any detail other than
// to say that "If a stub package is partial it MUST include partial\n in a top
// level py.typed file."
if (pyTypedContents.match(/partial\n/) || pyTypedContents.match(/partial\r\n/)) {
isPartiallyTyped = true;
}
}
}
}
return {
isPyTypedPresent,
isPartiallyTyped,
};
}
private _lookUpResultsInCache(
execEnv: ExecutionEnvironment,
importName: string,
@ -760,7 +795,7 @@ export class ImportResolver {
importFailureInfo
);
if (typingsImport && typingsImport.isImportFound) {
if (typingsImport.isImportFound) {
// We will treat typings files as "local" rather than "third party".
typingsImport.importType = ImportType.Local;
typingsImport.isLocalTypingsFile = true;
@ -778,12 +813,12 @@ export class ImportResolver {
moduleDescriptor,
importName,
importFailureInfo,
undefined,
undefined,
undefined,
/* allowPartial */ undefined,
/* allowNativeLib */ undefined,
/* useStubPackage */ undefined,
allowPyi
);
if (localImport && localImport.isImportFound && !localImport.isNamespacePackage) {
if (localImport.isImportFound && !localImport.isNamespacePackage) {
return localImport;
}
bestResultSoFar = localImport;
@ -795,12 +830,12 @@ export class ImportResolver {
moduleDescriptor,
importName,
importFailureInfo,
undefined,
undefined,
undefined,
/* allowPartial */ undefined,
/* allowNativeLib */ undefined,
/* useStubPackage */ undefined,
allowPyi
);
if (localImport && localImport.isImportFound) {
if (localImport.isImportFound) {
return localImport;
}
@ -820,19 +855,53 @@ export class ImportResolver {
if (pythonSearchPaths.length > 0) {
for (const searchPath of pythonSearchPaths) {
importFailureInfo.push(`Looking in python search path '${searchPath}'`);
const thirdPartyImport = this.resolveAbsoluteImport(
searchPath,
moduleDescriptor,
importName,
importFailureInfo,
/* allowPartial */ allowPartialResolutionForThirdPartyPackages,
/* allowNativeLib */ true,
/* allowStubPackages */ true,
allowPyi
);
// Is there a "py.typed" file present?
const dirPath = combinePaths(searchPath, moduleDescriptor.nameParts[0]);
let pyTypedInfo: PyTypedInfo | undefined;
let thirdPartyImport: ImportResult | undefined;
if (allowPyi) {
pyTypedInfo = this._getPyTypedInfo(dirPath + stubsSuffix);
// Look for packaged stubs first. PEP 561 indicates that package authors can ship
// their stubs separately from their package implementation by appending the string
// '-stubs' to its top - level directory name. We'll look there first.
thirdPartyImport = this.resolveAbsoluteImport(
searchPath,
moduleDescriptor,
importName,
importFailureInfo,
/* allowPartial */ allowPartialResolutionForThirdPartyPackages,
/* allowNativeLib */ false,
/* useStubPackage */ true,
allowPyi
);
}
if (!thirdPartyImport?.isImportFound) {
// Either we didn't look for a packaged stub or we looked but didn't find one.
// If there was a packaged stub directory, we can stop searching unless
// it happened to be marked as "partially typed".
if (!thirdPartyImport?.packageDirectory || pyTypedInfo?.isPartiallyTyped) {
pyTypedInfo = this._getPyTypedInfo(dirPath);
thirdPartyImport = this.resolveAbsoluteImport(
searchPath,
moduleDescriptor,
importName,
importFailureInfo,
/* allowPartial */ allowPartialResolutionForThirdPartyPackages,
/* allowNativeLib */ true,
/* useStubPackage */ false,
allowPyi
);
}
}
if (thirdPartyImport) {
thirdPartyImport.importType = ImportType.ThirdParty;
thirdPartyImport.isPyTypedPresent = pyTypedInfo?.isPyTypedPresent;
if (thirdPartyImport.isImportFound && thirdPartyImport.isStubFile) {
return thirdPartyImport;
@ -953,7 +1022,7 @@ export class ImportResolver {
importName,
importFailureInfo
);
if (importInfo && importInfo.isImportFound) {
if (importInfo.isImportFound) {
importInfo.importType = isStdLib ? ImportType.BuiltIn : ImportType.ThirdParty;
return importInfo;
}
@ -1082,10 +1151,6 @@ export class ImportResolver {
// Now try to match the module parts from the current directory location.
const absImport = this.resolveAbsoluteImport(curDir, moduleDescriptor, importName, importFailureInfo);
if (!absImport) {
return undefined;
}
return this._filterImplicitImports(absImport, moduleDescriptor.importedSymbols);
}

7
packages/pyright-internal/src/analyzer/importResult.ts
View File

@ -73,4 +73,11 @@ export interface ImportResult {
// If resolved from a type hint (.pyi), then store the import result
// from .py here.
nonStubImportResult?: ImportResult;
// Is there a "py.typed" file (as described in PEP 561) present in
// the package that was used to resolve the import?
isPyTypedPresent?: boolean;
// The directory of the package, if found.
packageDirectory?: string;
}

50
packages/pyright-internal/src/analyzer/program.ts
View File

@ -76,6 +76,7 @@ export interface SourceFileInfo {
// Information about the source file
isTypeshedFile: boolean;
isThirdPartyImport: boolean;
isThirdPartyPyTypedPresent: boolean;
diagnosticsVersion?: number;
builtinsImport?: SourceFileInfo;
@ -112,6 +113,7 @@ export interface Indices {
interface UpdateImportInfo {
isTypeshedFile: boolean;
isThirdPartyImport: boolean;
isPyTypedPresent: boolean;
}
// Container for all of the files that are being analyzed. Files
@ -221,6 +223,7 @@ export class Program {
isOpenByClient: false,
isTypeshedFile: false,
isThirdPartyImport: false,
isThirdPartyPyTypedPresent: false,
diagnosticsVersion: undefined,
imports: [],
importedBy: [],
@ -249,6 +252,7 @@ export class Program {
isOpenByClient: true,
isTypeshedFile: false,
isThirdPartyImport: false,
isThirdPartyPyTypedPresent: false,
diagnosticsVersion: undefined,
imports: [],
importedBy: [],
@ -600,6 +604,7 @@ export class Program {
isOpenByClient: false,
isTypeshedFile: false,
isThirdPartyImport: false,
isThirdPartyPyTypedPresent: false,
diagnosticsVersion: undefined,
imports: [],
importedBy: [],
@ -1718,7 +1723,10 @@ export class Program {
return false;
}
let thirdPartyImportAllowed = this._configOptions.useLibraryCodeForTypes || false;
let thirdPartyImportAllowed =
this._configOptions.useLibraryCodeForTypes ||
(importResult.importType === ImportType.ThirdParty && !!importResult.isPyTypedPresent) ||
(importResult.importType === ImportType.Local && importer.isThirdPartyPyTypedPresent);
if (
importResult.importType === ImportType.ThirdParty ||
@ -1768,6 +1776,33 @@ export class Program {
// list of imports for this file.
const imports = sourceFileInfo.sourceFile.getImports();
// Create a local function that determines whether the import should
// be considered a "third-party import" and whether it is coming from
// a third-party package that claims to be typed. An import is
// considered third-party if it is external to the importer
// or is internal but the importer is itself a third-party package.
const getThirdPartyImportInfo = (importResult: ImportResult) => {
let isThirdPartyImport = false;
let isPyTypedPresent = false;
if (importResult.importType === ImportType.ThirdParty) {
isThirdPartyImport = true;
if (importResult.isPyTypedPresent) {
isPyTypedPresent = true;
}
} else if (sourceFileInfo.isThirdPartyImport && importResult.importType === ImportType.Local) {
isThirdPartyImport = true;
if (sourceFileInfo.isThirdPartyPyTypedPresent) {
isPyTypedPresent = true;
}
}
return {
isThirdPartyImport,
isPyTypedPresent,
};
};
// Create a map of unique imports, since imports can appear more than once.
const newImportPathMap = new Map<string, UpdateImportInfo>();
imports.forEach((importResult) => {
@ -1776,11 +1811,11 @@ export class Program {
if (importResult.resolvedPaths.length > 0) {
const filePath = importResult.resolvedPaths[importResult.resolvedPaths.length - 1];
if (filePath) {
const thirdPartyTypeInfo = getThirdPartyImportInfo(importResult);
newImportPathMap.set(filePath, {
isTypeshedFile: !!importResult.isTypeshedFile,
isThirdPartyImport:
importResult.importType === ImportType.ThirdParty ||
(sourceFileInfo.isThirdPartyImport && importResult.importType === ImportType.Local),
isThirdPartyImport: thirdPartyTypeInfo.isThirdPartyImport,
isPyTypedPresent: thirdPartyTypeInfo.isPyTypedPresent,
});
}
}
@ -1788,11 +1823,11 @@ export class Program {
importResult.implicitImports.forEach((implicitImport) => {
if (this._isImportAllowed(sourceFileInfo, importResult, implicitImport.isStubFile)) {
const thirdPartyTypeInfo = getThirdPartyImportInfo(importResult);
newImportPathMap.set(implicitImport.path, {
isTypeshedFile: !!importResult.isTypeshedFile,
isThirdPartyImport:
importResult.importType === ImportType.ThirdParty ||
(sourceFileInfo.isThirdPartyImport && importResult.importType === ImportType.Local),
isThirdPartyImport: thirdPartyTypeInfo.isThirdPartyImport,
isPyTypedPresent: thirdPartyTypeInfo.isPyTypedPresent,
});
}
});
@ -1847,6 +1882,7 @@ export class Program {
isOpenByClient: false,
isTypeshedFile: importInfo.isTypeshedFile,
isThirdPartyImport: importInfo.isThirdPartyImport,
isThirdPartyPyTypedPresent: importInfo.isPyTypedPresent,
diagnosticsVersion: undefined,
imports: [],
importedBy: [],