microsoft/pyright
1
1
Fork 0
mirror of https://github.com/microsoft/pyright.git synced 2024-09-11 07:55:56 +03:00
Code Issues Projects Releases Wiki Activity

libzip fixes, change function to protected in resolver, make watcher error a warning (#1967)

Browse Source 
Co-authored-by: HeeJae Chang <hechang@microsoft.com>
This commit is contained in:
Jake Bailey 2021-06-09 13:30:11 -07:00 committed by GitHub
parent e98146b5d3
commit e3f7cccc6e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 161 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build/updateDeps.js
View File

@ -10,7 +10,7 @@ async function main() {
transitive: { type: 'boolean' },
}).argv;
await updateAll(argv.transitive, [
await updateAll(!!argv.transitive, [
// These packages impact compatibility with VS Code and other users;
// ensure they remained pinned exactly.
'@types/vscode',

2
packages/pyright-internal/src/analyzer/docStringConversion.ts
View File

@ -103,6 +103,7 @@ interface RestTableState {
header: string;
inHeader: boolean;
}
class DocStringConverter {
private _builder = '';
private _skipAppendEmptyLine = true;
@ -635,6 +636,7 @@ class DocStringConverter {
if (EqualHeaderRegExp.test(line)) {
this._eatLine();
this._appendLine('\n<br/>\n');
this._popState();
this._tableState = undefined;
return;

17
packages/pyright-internal/src/analyzer/importResolver.ts
View File

@ -28,7 +28,6 @@ import {
getRelativePathComponentsFromDirectory,
isDirectory,
isFile,
normalizePathCase,
resolvePaths,
stripFileExtension,
stripTrailingDirectorySeparator,
@ -227,7 +226,7 @@ export class ImportResolver {
this._getCompletionSuggestionsTypeshedPath(execEnv, moduleDescriptor, false, suggestions, similarityLimit);
// Look for the import in the list of third-party packages.
const pythonSearchPaths = this._getPythonSearchPaths(execEnv, importFailureInfo);
const pythonSearchPaths = this.getPythonSearchPaths(execEnv, importFailureInfo);
for (const searchPath of pythonSearchPaths) {
this.getCompletionSuggestionsAbsolute(searchPath, moduleDescriptor, suggestions, similarityLimit);
}
@ -430,7 +429,7 @@ export class ImportResolver {
}
// Look for the import in the list of third-party packages.
const pythonSearchPaths = this._getPythonSearchPaths(execEnv, importFailureInfo);
const pythonSearchPaths = this.getPythonSearchPaths(execEnv, importFailureInfo);
for (const searchPath of pythonSearchPaths) {
const candidateModuleName = this._getModuleNameFromPath(searchPath, filePath);
@ -489,7 +488,7 @@ export class ImportResolver {
roots.push(typeshedPathEx);
}
const pythonSearchPaths = this._getPythonSearchPaths(execEnv, importFailureInfo);
const pythonSearchPaths = this.getPythonSearchPaths(execEnv, importFailureInfo);
if (pythonSearchPaths.length > 0) {
roots.push(...pythonSearchPaths);
}
@ -585,7 +584,7 @@ export class ImportResolver {
addPaths(execEnv.root);
execEnv.extraPaths.forEach((p) => addPaths(p));
addPaths(this.getTypeshedPathEx(execEnv, ignored));
this._getPythonSearchPaths(execEnv, ignored).forEach((p) => addPaths(p));
this.getPythonSearchPaths(execEnv, ignored).forEach((p) => addPaths(p));
this.fileSystem.processPartialStubPackages(paths, this.getImportRoots(execEnv));
this._invalidateFileSystemCache();
@ -1089,7 +1088,7 @@ export class ImportResolver {
}
// Look for the import in the list of third-party packages.
const pythonSearchPaths = this._getPythonSearchPaths(execEnv, importFailureInfo);
const pythonSearchPaths = this.getPythonSearchPaths(execEnv, importFailureInfo);
if (pythonSearchPaths.length > 0) {
for (const searchPath of pythonSearchPaths) {
importFailureInfo.push(`Looking in python search path '${searchPath}'`);
@ -1224,14 +1223,14 @@ export class ImportResolver {
return true;
}
private _getPythonSearchPaths(execEnv: ExecutionEnvironment, importFailureInfo: string[]) {
protected getPythonSearchPaths(execEnv: ExecutionEnvironment, importFailureInfo: string[]) {
const cacheKey = '<default>';
// Find the site packages for the configured virtual environment.
if (!this._cachedPythonSearchPaths.has(cacheKey)) {
let paths = (
PythonPathUtils.findPythonSearchPaths(this.fileSystem, this._configOptions, importFailureInfo) || []
).map((p) => normalizePathCase(this.fileSystem, p));
).map((p) => this.fileSystem.realCasePath(p));
// Remove duplicates (yes, it happens).
paths = [...new Set(paths)];
@ -1489,7 +1488,7 @@ export class ImportResolver {
typeshedPath = possibleTypeshedPath;
}
} else {
const pythonSearchPaths = this._getPythonSearchPaths(execEnv, importFailureInfo);
const pythonSearchPaths = this.getPythonSearchPaths(execEnv, importFailureInfo);
for (const searchPath of pythonSearchPaths) {
const possibleTypeshedPath = combinePaths(searchPath, 'typeshed');
if (this.dirExistsCached(possibleTypeshedPath)) {

134
packages/pyright-internal/src/common/fileSystem.ts
View File

@ -9,13 +9,14 @@
*/
// * NOTE * except tests, this should be only file that import "fs"
import { FakeFS, PortablePath, PosixFS, ppath, VirtualFS, ZipOpenFS } from '@yarnpkg/fslib';
import { FakeFS, NativePath, PortablePath, PosixFS, ppath, VirtualFS, ZipOpenFS } from '@yarnpkg/fslib';
import { getLibzipSync } from '@yarnpkg/libzip';
import * as chokidar from 'chokidar';
import type * as fs from 'fs';
import * as fs from 'fs';
import * as tmp from 'tmp';
import { ConsoleInterface, NullConsole } from './console';
import { getRootLength } from './pathUtils';
// Automatically remove files created by tmp at process exit.
tmp.setGracefulCleanup();
@ -74,6 +75,9 @@ export interface FileSystem {
// The directory returned by tmpdir must exist and be the same each time tmpdir is called.
tmpdir(): string;
tmpfile(options?: TmpfileOptions): string;
// Return path in casing on OS.
realCasePath(path: string): string;
}
export interface FileWatcherProvider {
@ -86,7 +90,8 @@ export function createFromRealFileSystem(
console?: ConsoleInterface,
fileWatcherProvider?: FileWatcherProvider
): FileSystem {
return new RealFileSystem(fileWatcherProvider ?? new ChokidarFileWatcherProvider(console ?? new NullConsole()));
console = console ?? new NullConsole();
return new RealFileSystem(fileWatcherProvider ?? new ChokidarFileWatcherProvider(console), console);
}
// File watchers can give "changed" event even for a file open. but for those cases,
@ -134,10 +139,47 @@ function getArchivePart(path: string) {
// Returns true if the specified path may be inside of a zip or egg file.
// These files don't really exist, and will fail if navigated to in the editor.
export function isInZipOrEgg(path: string) {
export function isInZipOrEgg(path: string): boolean {
return /[^\\/]\.(?:egg|zip)[\\/]/.test(path);
}
function hasZipOrEggExtension(p: string): boolean {
return p.endsWith(DOT_ZIP) || p.endsWith(DOT_EGG);
}
// "Magic" values for the zip file type. https://en.wikipedia.org/wiki/List_of_file_signatures
const zipMagic = [
Buffer.from([0x50, 0x4b, 0x03, 0x04]),
Buffer.from([0x50, 0x4b, 0x05, 0x06]),
Buffer.from([0x50, 0x4b, 0x07, 0x08]),
];
function hasZipMagic(fs: FakeFS<PortablePath>, p: PortablePath): boolean {
let fd: number | undefined;
try {
fd = fs.openSync(p, 'r');
const buffer = Buffer.alloc(4);
const bytesRead = fs.readSync(fd, buffer, 0, 4, 0);
if (bytesRead < 4) {
return false;
}
for (const magic of zipMagic) {
if (buffer.compare(magic) === 0) {
return true;
}
}
return false;
} catch {
return false;
} finally {
if (fd !== undefined) {
fs.closeSync(fd);
}
}
}
// Patch fslib's ZipOpenFS to also consider .egg files to be .zip files.
//
// For now, override findZip (even though it's private), with the intent
@ -153,7 +195,6 @@ class EggZipOpenFS extends ZipOpenFS {
private isZip!: Set<PortablePath>;
private notZip!: Set<PortablePath>;
// Exactly the same as ZipOpenFS, but uses our getArchivePart.
findZip(p: PortablePath) {
if (this.filter && !this.filter.test(p)) return null;
@ -173,6 +214,11 @@ class EggZipOpenFS extends ZipOpenFS {
this.notZip.add(filePath);
continue;
}
if (!hasZipMagic(this.baseFs, filePath)) {
this.notZip.add(filePath);
continue;
}
} catch {
return null;
}
@ -188,29 +234,47 @@ class EggZipOpenFS extends ZipOpenFS {
}
}
const yarnFS = new PosixFS(
new VirtualFS({
baseFs: new EggZipOpenFS({
class YarnFS extends PosixFS {
private readonly _eggZipOpenFS: EggZipOpenFS;
constructor() {
const eggZipOpenFS = new EggZipOpenFS({
// Note: libzip is a WASM module and can take a few milliseconds to load.
// The next version of fslib should allow this to be initialized lazily.
libzip: getLibzipSync(),
useCache: true,
maxOpenFiles: 80,
readOnlyArchives: true,
}),
})
);
});
class RealFileSystem implements FileSystem {
private _fileWatcherProvider: FileWatcherProvider;
private _tmpdir?: string;
super(
new VirtualFS({
baseFs: eggZipOpenFS,
})
);
constructor(fileWatcherProvider: FileWatcherProvider) {
this._fileWatcherProvider = fileWatcherProvider;
this._eggZipOpenFS = eggZipOpenFS;
}
isZip(p: NativePath): boolean {
return !!this._eggZipOpenFS.findZip(this.mapToBase(p));
}
}
const yarnFS = new YarnFS();
class RealFileSystem implements FileSystem {
private _tmpdir?: string;
constructor(private _fileWatcherProvider: FileWatcherProvider, private _console: ConsoleInterface) {}
existsSync(path: string) {
return yarnFS.existsSync(path);
try {
// Catch zip open errors. existsSync is assumed to never throw by callers.
return yarnFS.existsSync(path);
} catch {
return false;
}
}
mkdirSync(path: string, options?: MkDirOptions) {
@ -229,8 +293,8 @@ class RealFileSystem implements FileSystem {
return yarnFS.readdirSync(path, { withFileTypes: true }).map((entry): fs.Dirent => {
// Treat zip/egg files as directories.
// See: https://github.com/yarnpkg/berry/blob/master/packages/vscode-zipfs/sources/ZipFSProvider.ts
if (entry.name.endsWith(DOT_ZIP) || entry.name.endsWith(DOT_EGG)) {
if (entry.isFile()) {
if (hasZipOrEggExtension(entry.name)) {
if (entry.isFile() && yarnFS.isZip(path)) {
return {
name: entry.name,
isFile: () => false,
@ -265,8 +329,8 @@ class RealFileSystem implements FileSystem {
const stat = yarnFS.statSync(path);
// Treat zip/egg files as directories.
// See: https://github.com/yarnpkg/berry/blob/master/packages/vscode-zipfs/sources/ZipFSProvider.ts
if (path.endsWith(DOT_ZIP) || path.endsWith(DOT_EGG)) {
if (stat.isFile()) {
if (hasZipOrEggExtension(path)) {
if (stat.isFile() && yarnFS.isZip(path)) {
return {
...stat,
isFile: () => false,
@ -332,6 +396,34 @@ class RealFileSystem implements FileSystem {
const f = tmp.fileSync({ dir: this.tmpdir(), discardDescriptor: true, ...options });
return f.name;
}
realCasePath(path: string): string {
try {
// If it doesn't exist in the real FS, return path as it is.
if (!fs.existsSync(path)) {
return path;
}
// realpathSync.native will return casing as in OS rather than
// trying to preserve casing given.
const realPath = fs.realpathSync.native(path);
// path is not rooted, return as it is
const rootLength = getRootLength(realPath);
if (rootLength <= 0) {
return realPath;
}
// path is rooted, make sure we lower case the root part
// to follow vscode's behavior.
return realPath.substr(0, rootLength).toLowerCase() + realPath.substr(rootLength);
} catch (e) {
// Return as it is, if anything failed.
this._console.error(`Failed to get real file system casing for ${path}: ${e}`);
return path;
}
}
}
class ChokidarFileWatcherProvider implements FileWatcherProvider {

2
packages/pyright-internal/src/languageServerBase.ts
View File

@ -422,7 +422,7 @@ export abstract class LanguageServerBase implements LanguageServerInterface {
listener(event as FileWatcherEventType, filename)
);
} catch (e) {
this.console.error(`Exception received when installing recursive file system watcher: ${e}`);
this.console.warn(`Exception received when installing recursive file system watcher: ${e}`);
return undefined;
}
})

4
packages/pyright-internal/src/pyrightFileSystem.ts
View File

@ -159,6 +159,10 @@ export class PyrightFileSystem implements FileSystem {
return this._realFS.tmpfile(options);
}
realCasePath(path: string): string {
return this._realFS.realCasePath(path);
}
isPartialStubPackagesScanned(execEnv: ExecutionEnvironment): boolean {
return this.isPathScanned(execEnv.root);
}

16
packages/pyright-internal/src/tests/docStringConversion.test.ts
View File

@ -731,7 +731,9 @@ default_rng Default constructor for \`\`Generator\`\`
|Generator | |
|---------------|---------------------------------------------------------|
|Generator |Class implementing all of the random number distributions|
|default_rng |Default constructor for \`\`Generator\`\`|`;
|default_rng |Default constructor for \`\`Generator\`\`|
<br/>`;
_testConvertToMarkdown(docstring, markdown);
});
@ -749,7 +751,9 @@ rand Uniformly distributed values.
const markdown = `
|Compatibility <br>functions - removed <br>in the new API | <br> <br> |
|--------------------|---------------------------------------------------------|
|rand |Uniformly distributed values.|`;
|rand |Uniformly distributed values.|
<br/>`;
_testConvertToMarkdown(docstring, markdown);
});
@ -767,7 +771,9 @@ Scalar Type Array Type
const markdown = `|Scalar Type |Array Type |
|------------------------------|-------------------------------------|
|:class:\`pandas.Interval\` |:class:\`pandas.arrays.IntervalArray\`|
|:class:\`pandas.Period\` |:class:\`pandas.arrays.PeriodArray\`|`;
|:class:\`pandas.Period\` |:class:\`pandas.arrays.PeriodArray\`|
<br/>`;
_testConvertToMarkdown(docstring, markdown);
});
@ -793,7 +799,9 @@ dtype : str, np.dtype, or ExtensionDtype, optional
| Scalar Type | Array Type |
|------------------------------|-------------------------------------|
| :class:\`pandas.Interval\` | :class:\`pandas.arrays.IntervalArray\`|
| :class:\`pandas.Period\` | :class:\`pandas.arrays.PeriodArray\`|`;
| :class:\`pandas.Period\` | :class:\`pandas.arrays.PeriodArray\`|
<br/>`;
_testConvertToMarkdown(docstring, markdown);
});

4
packages/pyright-internal/src/tests/harness/vfs/filesystem.ts
View File

@ -335,6 +335,10 @@ export class TestFileSystem implements FileSystem {
return path;
}
realCasePath(path: string): string {
return path;
}
private _scan(path: string, stats: Stats, axis: Axis, traversal: Traversal, noFollow: boolean, results: string[]) {
if (axis === 'ancestors-or-self' || axis === 'self' || axis === 'descendants-or-self') {
if (!traversal.accept || traversal.accept(path, stats)) {

1
packages/pyright-internal/src/tests/samples/zipfs/bad.egg Normal file
View File

@ -0,0 +1 @@
This file isn't a zip.

1
packages/pyright-internal/src/tests/samples/zipfs/bad.zip Normal file
View File

@ -0,0 +1 @@
This file isn't a zip.

14
packages/pyright-internal/src/tests/zipfs.test.ts
View File

@ -88,3 +88,17 @@ function runTests(p: string): void {
describe('zip', () => runTests('./samples/zipfs/basic.zip'));
describe('egg', () => runTests('./samples/zipfs/basic.egg'));
function runBadTests(p: string): void {
const zipRoot = path.resolve(path.dirname(module.filename), p);
const fs = createFromRealFileSystem();
test('stat root', () => {
const stats = fs.statSync(zipRoot);
assert.strictEqual(stats.isDirectory(), false);
assert.strictEqual(stats.isFile(), true);
});
}
describe('zip', () => runBadTests('./samples/zipfs/bad.zip'));
describe('egg', () => runBadTests('./samples/zipfs/bad.egg'));