mirror of
https://github.com/microsoft/pyright.git
synced 2024-09-11 07:55:56 +03:00
libzip fixes, change function to protected in resolver, make watcher error a warning (#1967)
Co-authored-by: HeeJae Chang <hechang@microsoft.com>
This commit is contained in:
parent
e98146b5d3
commit
e3f7cccc6e
@ -10,7 +10,7 @@ async function main() {
|
||||
transitive: { type: 'boolean' },
|
||||
}).argv;
|
||||
|
||||
await updateAll(argv.transitive, [
|
||||
await updateAll(!!argv.transitive, [
|
||||
// These packages impact compatibility with VS Code and other users;
|
||||
// ensure they remained pinned exactly.
|
||||
'@types/vscode',
|
||||
|
@ -103,6 +103,7 @@ interface RestTableState {
|
||||
header: string;
|
||||
inHeader: boolean;
|
||||
}
|
||||
|
||||
class DocStringConverter {
|
||||
private _builder = '';
|
||||
private _skipAppendEmptyLine = true;
|
||||
@ -635,6 +636,7 @@ class DocStringConverter {
|
||||
|
||||
if (EqualHeaderRegExp.test(line)) {
|
||||
this._eatLine();
|
||||
this._appendLine('\n<br/>\n');
|
||||
this._popState();
|
||||
this._tableState = undefined;
|
||||
return;
|
||||
|
@ -28,7 +28,6 @@ import {
|
||||
getRelativePathComponentsFromDirectory,
|
||||
isDirectory,
|
||||
isFile,
|
||||
normalizePathCase,
|
||||
resolvePaths,
|
||||
stripFileExtension,
|
||||
stripTrailingDirectorySeparator,
|
||||
@ -227,7 +226,7 @@ export class ImportResolver {
|
||||
this._getCompletionSuggestionsTypeshedPath(execEnv, moduleDescriptor, false, suggestions, similarityLimit);
|
||||
|
||||
// Look for the import in the list of third-party packages.
|
||||
const pythonSearchPaths = this._getPythonSearchPaths(execEnv, importFailureInfo);
|
||||
const pythonSearchPaths = this.getPythonSearchPaths(execEnv, importFailureInfo);
|
||||
for (const searchPath of pythonSearchPaths) {
|
||||
this.getCompletionSuggestionsAbsolute(searchPath, moduleDescriptor, suggestions, similarityLimit);
|
||||
}
|
||||
@ -430,7 +429,7 @@ export class ImportResolver {
|
||||
}
|
||||
|
||||
// Look for the import in the list of third-party packages.
|
||||
const pythonSearchPaths = this._getPythonSearchPaths(execEnv, importFailureInfo);
|
||||
const pythonSearchPaths = this.getPythonSearchPaths(execEnv, importFailureInfo);
|
||||
for (const searchPath of pythonSearchPaths) {
|
||||
const candidateModuleName = this._getModuleNameFromPath(searchPath, filePath);
|
||||
|
||||
@ -489,7 +488,7 @@ export class ImportResolver {
|
||||
roots.push(typeshedPathEx);
|
||||
}
|
||||
|
||||
const pythonSearchPaths = this._getPythonSearchPaths(execEnv, importFailureInfo);
|
||||
const pythonSearchPaths = this.getPythonSearchPaths(execEnv, importFailureInfo);
|
||||
if (pythonSearchPaths.length > 0) {
|
||||
roots.push(...pythonSearchPaths);
|
||||
}
|
||||
@ -585,7 +584,7 @@ export class ImportResolver {
|
||||
addPaths(execEnv.root);
|
||||
execEnv.extraPaths.forEach((p) => addPaths(p));
|
||||
addPaths(this.getTypeshedPathEx(execEnv, ignored));
|
||||
this._getPythonSearchPaths(execEnv, ignored).forEach((p) => addPaths(p));
|
||||
this.getPythonSearchPaths(execEnv, ignored).forEach((p) => addPaths(p));
|
||||
|
||||
this.fileSystem.processPartialStubPackages(paths, this.getImportRoots(execEnv));
|
||||
this._invalidateFileSystemCache();
|
||||
@ -1089,7 +1088,7 @@ export class ImportResolver {
|
||||
}
|
||||
|
||||
// Look for the import in the list of third-party packages.
|
||||
const pythonSearchPaths = this._getPythonSearchPaths(execEnv, importFailureInfo);
|
||||
const pythonSearchPaths = this.getPythonSearchPaths(execEnv, importFailureInfo);
|
||||
if (pythonSearchPaths.length > 0) {
|
||||
for (const searchPath of pythonSearchPaths) {
|
||||
importFailureInfo.push(`Looking in python search path '${searchPath}'`);
|
||||
@ -1224,14 +1223,14 @@ export class ImportResolver {
|
||||
return true;
|
||||
}
|
||||
|
||||
private _getPythonSearchPaths(execEnv: ExecutionEnvironment, importFailureInfo: string[]) {
|
||||
protected getPythonSearchPaths(execEnv: ExecutionEnvironment, importFailureInfo: string[]) {
|
||||
const cacheKey = '<default>';
|
||||
|
||||
// Find the site packages for the configured virtual environment.
|
||||
if (!this._cachedPythonSearchPaths.has(cacheKey)) {
|
||||
let paths = (
|
||||
PythonPathUtils.findPythonSearchPaths(this.fileSystem, this._configOptions, importFailureInfo) || []
|
||||
).map((p) => normalizePathCase(this.fileSystem, p));
|
||||
).map((p) => this.fileSystem.realCasePath(p));
|
||||
|
||||
// Remove duplicates (yes, it happens).
|
||||
paths = [...new Set(paths)];
|
||||
@ -1489,7 +1488,7 @@ export class ImportResolver {
|
||||
typeshedPath = possibleTypeshedPath;
|
||||
}
|
||||
} else {
|
||||
const pythonSearchPaths = this._getPythonSearchPaths(execEnv, importFailureInfo);
|
||||
const pythonSearchPaths = this.getPythonSearchPaths(execEnv, importFailureInfo);
|
||||
for (const searchPath of pythonSearchPaths) {
|
||||
const possibleTypeshedPath = combinePaths(searchPath, 'typeshed');
|
||||
if (this.dirExistsCached(possibleTypeshedPath)) {
|
||||
|
@ -9,13 +9,14 @@
|
||||
*/
|
||||
|
||||
// * NOTE * except tests, this should be only file that import "fs"
|
||||
import { FakeFS, PortablePath, PosixFS, ppath, VirtualFS, ZipOpenFS } from '@yarnpkg/fslib';
|
||||
import { FakeFS, NativePath, PortablePath, PosixFS, ppath, VirtualFS, ZipOpenFS } from '@yarnpkg/fslib';
|
||||
import { getLibzipSync } from '@yarnpkg/libzip';
|
||||
import * as chokidar from 'chokidar';
|
||||
import type * as fs from 'fs';
|
||||
import * as fs from 'fs';
|
||||
import * as tmp from 'tmp';
|
||||
|
||||
import { ConsoleInterface, NullConsole } from './console';
|
||||
import { getRootLength } from './pathUtils';
|
||||
|
||||
// Automatically remove files created by tmp at process exit.
|
||||
tmp.setGracefulCleanup();
|
||||
@ -74,6 +75,9 @@ export interface FileSystem {
|
||||
// The directory returned by tmpdir must exist and be the same each time tmpdir is called.
|
||||
tmpdir(): string;
|
||||
tmpfile(options?: TmpfileOptions): string;
|
||||
|
||||
// Return path in casing on OS.
|
||||
realCasePath(path: string): string;
|
||||
}
|
||||
|
||||
export interface FileWatcherProvider {
|
||||
@ -86,7 +90,8 @@ export function createFromRealFileSystem(
|
||||
console?: ConsoleInterface,
|
||||
fileWatcherProvider?: FileWatcherProvider
|
||||
): FileSystem {
|
||||
return new RealFileSystem(fileWatcherProvider ?? new ChokidarFileWatcherProvider(console ?? new NullConsole()));
|
||||
console = console ?? new NullConsole();
|
||||
return new RealFileSystem(fileWatcherProvider ?? new ChokidarFileWatcherProvider(console), console);
|
||||
}
|
||||
|
||||
// File watchers can give "changed" event even for a file open. but for those cases,
|
||||
@ -134,10 +139,47 @@ function getArchivePart(path: string) {
|
||||
|
||||
// Returns true if the specified path may be inside of a zip or egg file.
|
||||
// These files don't really exist, and will fail if navigated to in the editor.
|
||||
export function isInZipOrEgg(path: string) {
|
||||
export function isInZipOrEgg(path: string): boolean {
|
||||
return /[^\\/]\.(?:egg|zip)[\\/]/.test(path);
|
||||
}
|
||||
|
||||
function hasZipOrEggExtension(p: string): boolean {
|
||||
return p.endsWith(DOT_ZIP) || p.endsWith(DOT_EGG);
|
||||
}
|
||||
|
||||
// "Magic" values for the zip file type. https://en.wikipedia.org/wiki/List_of_file_signatures
|
||||
const zipMagic = [
|
||||
Buffer.from([0x50, 0x4b, 0x03, 0x04]),
|
||||
Buffer.from([0x50, 0x4b, 0x05, 0x06]),
|
||||
Buffer.from([0x50, 0x4b, 0x07, 0x08]),
|
||||
];
|
||||
|
||||
function hasZipMagic(fs: FakeFS<PortablePath>, p: PortablePath): boolean {
|
||||
let fd: number | undefined;
|
||||
try {
|
||||
fd = fs.openSync(p, 'r');
|
||||
const buffer = Buffer.alloc(4);
|
||||
const bytesRead = fs.readSync(fd, buffer, 0, 4, 0);
|
||||
if (bytesRead < 4) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (const magic of zipMagic) {
|
||||
if (buffer.compare(magic) === 0) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
} catch {
|
||||
return false;
|
||||
} finally {
|
||||
if (fd !== undefined) {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Patch fslib's ZipOpenFS to also consider .egg files to be .zip files.
|
||||
//
|
||||
// For now, override findZip (even though it's private), with the intent
|
||||
@ -153,7 +195,6 @@ class EggZipOpenFS extends ZipOpenFS {
|
||||
private isZip!: Set<PortablePath>;
|
||||
private notZip!: Set<PortablePath>;
|
||||
|
||||
// Exactly the same as ZipOpenFS, but uses our getArchivePart.
|
||||
findZip(p: PortablePath) {
|
||||
if (this.filter && !this.filter.test(p)) return null;
|
||||
|
||||
@ -173,6 +214,11 @@ class EggZipOpenFS extends ZipOpenFS {
|
||||
this.notZip.add(filePath);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!hasZipMagic(this.baseFs, filePath)) {
|
||||
this.notZip.add(filePath);
|
||||
continue;
|
||||
}
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
@ -188,29 +234,47 @@ class EggZipOpenFS extends ZipOpenFS {
|
||||
}
|
||||
}
|
||||
|
||||
const yarnFS = new PosixFS(
|
||||
new VirtualFS({
|
||||
baseFs: new EggZipOpenFS({
|
||||
class YarnFS extends PosixFS {
|
||||
private readonly _eggZipOpenFS: EggZipOpenFS;
|
||||
|
||||
constructor() {
|
||||
const eggZipOpenFS = new EggZipOpenFS({
|
||||
// Note: libzip is a WASM module and can take a few milliseconds to load.
|
||||
// The next version of fslib should allow this to be initialized lazily.
|
||||
libzip: getLibzipSync(),
|
||||
useCache: true,
|
||||
maxOpenFiles: 80,
|
||||
readOnlyArchives: true,
|
||||
}),
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
class RealFileSystem implements FileSystem {
|
||||
private _fileWatcherProvider: FileWatcherProvider;
|
||||
private _tmpdir?: string;
|
||||
super(
|
||||
new VirtualFS({
|
||||
baseFs: eggZipOpenFS,
|
||||
})
|
||||
);
|
||||
|
||||
constructor(fileWatcherProvider: FileWatcherProvider) {
|
||||
this._fileWatcherProvider = fileWatcherProvider;
|
||||
this._eggZipOpenFS = eggZipOpenFS;
|
||||
}
|
||||
|
||||
isZip(p: NativePath): boolean {
|
||||
return !!this._eggZipOpenFS.findZip(this.mapToBase(p));
|
||||
}
|
||||
}
|
||||
|
||||
const yarnFS = new YarnFS();
|
||||
|
||||
class RealFileSystem implements FileSystem {
|
||||
private _tmpdir?: string;
|
||||
|
||||
constructor(private _fileWatcherProvider: FileWatcherProvider, private _console: ConsoleInterface) {}
|
||||
|
||||
existsSync(path: string) {
|
||||
return yarnFS.existsSync(path);
|
||||
try {
|
||||
// Catch zip open errors. existsSync is assumed to never throw by callers.
|
||||
return yarnFS.existsSync(path);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
mkdirSync(path: string, options?: MkDirOptions) {
|
||||
@ -229,8 +293,8 @@ class RealFileSystem implements FileSystem {
|
||||
return yarnFS.readdirSync(path, { withFileTypes: true }).map((entry): fs.Dirent => {
|
||||
// Treat zip/egg files as directories.
|
||||
// See: https://github.com/yarnpkg/berry/blob/master/packages/vscode-zipfs/sources/ZipFSProvider.ts
|
||||
if (entry.name.endsWith(DOT_ZIP) || entry.name.endsWith(DOT_EGG)) {
|
||||
if (entry.isFile()) {
|
||||
if (hasZipOrEggExtension(entry.name)) {
|
||||
if (entry.isFile() && yarnFS.isZip(path)) {
|
||||
return {
|
||||
name: entry.name,
|
||||
isFile: () => false,
|
||||
@ -265,8 +329,8 @@ class RealFileSystem implements FileSystem {
|
||||
const stat = yarnFS.statSync(path);
|
||||
// Treat zip/egg files as directories.
|
||||
// See: https://github.com/yarnpkg/berry/blob/master/packages/vscode-zipfs/sources/ZipFSProvider.ts
|
||||
if (path.endsWith(DOT_ZIP) || path.endsWith(DOT_EGG)) {
|
||||
if (stat.isFile()) {
|
||||
if (hasZipOrEggExtension(path)) {
|
||||
if (stat.isFile() && yarnFS.isZip(path)) {
|
||||
return {
|
||||
...stat,
|
||||
isFile: () => false,
|
||||
@ -332,6 +396,34 @@ class RealFileSystem implements FileSystem {
|
||||
const f = tmp.fileSync({ dir: this.tmpdir(), discardDescriptor: true, ...options });
|
||||
return f.name;
|
||||
}
|
||||
|
||||
realCasePath(path: string): string {
|
||||
try {
|
||||
// If it doesn't exist in the real FS, return path as it is.
|
||||
if (!fs.existsSync(path)) {
|
||||
return path;
|
||||
}
|
||||
|
||||
// realpathSync.native will return casing as in OS rather than
|
||||
// trying to preserve casing given.
|
||||
const realPath = fs.realpathSync.native(path);
|
||||
|
||||
// path is not rooted, return as it is
|
||||
const rootLength = getRootLength(realPath);
|
||||
if (rootLength <= 0) {
|
||||
return realPath;
|
||||
}
|
||||
|
||||
// path is rooted, make sure we lower case the root part
|
||||
// to follow vscode's behavior.
|
||||
return realPath.substr(0, rootLength).toLowerCase() + realPath.substr(rootLength);
|
||||
} catch (e) {
|
||||
// Return as it is, if anything failed.
|
||||
this._console.error(`Failed to get real file system casing for ${path}: ${e}`);
|
||||
|
||||
return path;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class ChokidarFileWatcherProvider implements FileWatcherProvider {
|
||||
|
@ -422,7 +422,7 @@ export abstract class LanguageServerBase implements LanguageServerInterface {
|
||||
listener(event as FileWatcherEventType, filename)
|
||||
);
|
||||
} catch (e) {
|
||||
this.console.error(`Exception received when installing recursive file system watcher: ${e}`);
|
||||
this.console.warn(`Exception received when installing recursive file system watcher: ${e}`);
|
||||
return undefined;
|
||||
}
|
||||
})
|
||||
|
@ -159,6 +159,10 @@ export class PyrightFileSystem implements FileSystem {
|
||||
return this._realFS.tmpfile(options);
|
||||
}
|
||||
|
||||
realCasePath(path: string): string {
|
||||
return this._realFS.realCasePath(path);
|
||||
}
|
||||
|
||||
isPartialStubPackagesScanned(execEnv: ExecutionEnvironment): boolean {
|
||||
return this.isPathScanned(execEnv.root);
|
||||
}
|
||||
|
@ -731,7 +731,9 @@ default_rng Default constructor for \`\`Generator\`\`
|
||||
|Generator | |
|
||||
|---------------|---------------------------------------------------------|
|
||||
|Generator |Class implementing all of the random number distributions|
|
||||
|default_rng |Default constructor for \`\`Generator\`\`|`;
|
||||
|default_rng |Default constructor for \`\`Generator\`\`|
|
||||
|
||||
<br/>`;
|
||||
|
||||
_testConvertToMarkdown(docstring, markdown);
|
||||
});
|
||||
@ -749,7 +751,9 @@ rand Uniformly distributed values.
|
||||
const markdown = `
|
||||
|Compatibility <br>functions - removed <br>in the new API | <br> <br> |
|
||||
|--------------------|---------------------------------------------------------|
|
||||
|rand |Uniformly distributed values.|`;
|
||||
|rand |Uniformly distributed values.|
|
||||
|
||||
<br/>`;
|
||||
|
||||
_testConvertToMarkdown(docstring, markdown);
|
||||
});
|
||||
@ -767,7 +771,9 @@ Scalar Type Array Type
|
||||
const markdown = `|Scalar Type |Array Type |
|
||||
|------------------------------|-------------------------------------|
|
||||
|:class:\`pandas.Interval\` |:class:\`pandas.arrays.IntervalArray\`|
|
||||
|:class:\`pandas.Period\` |:class:\`pandas.arrays.PeriodArray\`|`;
|
||||
|:class:\`pandas.Period\` |:class:\`pandas.arrays.PeriodArray\`|
|
||||
|
||||
<br/>`;
|
||||
|
||||
_testConvertToMarkdown(docstring, markdown);
|
||||
});
|
||||
@ -793,7 +799,9 @@ dtype : str, np.dtype, or ExtensionDtype, optional
|
||||
| Scalar Type | Array Type |
|
||||
|------------------------------|-------------------------------------|
|
||||
| :class:\`pandas.Interval\` | :class:\`pandas.arrays.IntervalArray\`|
|
||||
| :class:\`pandas.Period\` | :class:\`pandas.arrays.PeriodArray\`|`;
|
||||
| :class:\`pandas.Period\` | :class:\`pandas.arrays.PeriodArray\`|
|
||||
|
||||
<br/>`;
|
||||
|
||||
_testConvertToMarkdown(docstring, markdown);
|
||||
});
|
||||
|
@ -335,6 +335,10 @@ export class TestFileSystem implements FileSystem {
|
||||
return path;
|
||||
}
|
||||
|
||||
realCasePath(path: string): string {
|
||||
return path;
|
||||
}
|
||||
|
||||
private _scan(path: string, stats: Stats, axis: Axis, traversal: Traversal, noFollow: boolean, results: string[]) {
|
||||
if (axis === 'ancestors-or-self' || axis === 'self' || axis === 'descendants-or-self') {
|
||||
if (!traversal.accept || traversal.accept(path, stats)) {
|
||||
|
@ -0,0 +1 @@
|
||||
This file isn't a zip.
|
@ -0,0 +1 @@
|
||||
This file isn't a zip.
|
@ -88,3 +88,17 @@ function runTests(p: string): void {
|
||||
|
||||
describe('zip', () => runTests('./samples/zipfs/basic.zip'));
|
||||
describe('egg', () => runTests('./samples/zipfs/basic.egg'));
|
||||
|
||||
function runBadTests(p: string): void {
|
||||
const zipRoot = path.resolve(path.dirname(module.filename), p);
|
||||
const fs = createFromRealFileSystem();
|
||||
|
||||
test('stat root', () => {
|
||||
const stats = fs.statSync(zipRoot);
|
||||
assert.strictEqual(stats.isDirectory(), false);
|
||||
assert.strictEqual(stats.isFile(), true);
|
||||
});
|
||||
}
|
||||
|
||||
describe('zip', () => runBadTests('./samples/zipfs/bad.zip'));
|
||||
describe('egg', () => runBadTests('./samples/zipfs/bad.egg'));
|
||||
|
Loading…
Reference in New Issue
Block a user