Add extraPath support, accept minor versions in version_info, make completion extension async (#662)

.eslintrc.json

@@ -17,6 +17,7 @@
         "simple-import-sort"
     ],
     "rules": {
+        "eqeqeq": "error",
         "no-constant-condition": 0,
         "no-inner-declarations": 0,
         "no-undef": 0,

docs/settings.md

@@ -16,6 +16,8 @@ The Pyright VS Code extension honors the following settings.
 
 **python.analysis.autoSearchPaths** [boolean]: Determines whether pyright automatically adds common search paths like "src" if there are no execution environments defined in the config file.
 
+**python.analysis.extraPaths** [array of paths]: Paths to add to the default execution environment extra paths if there are no execution environments defined in the config file.
+
 **python.pythonPath** [path]: Path to Python interpreter.
 
 **python.venvPath** [path]: Path to folder with subdirectories that contain virtual environments.

server/.eslintrc.json

@@ -17,6 +17,7 @@
         "simple-import-sort"
     ],
     "rules": {
+        "eqeqeq": "error",
         "no-constant-condition": 0,
         "no-inner-declarations": 0,
         "no-undef": 0,

server/src/analyzer/program.ts

@@ -930,21 +930,21 @@ export class Program {
         });
     }
 
-    getCompletionsForPosition(
+    async getCompletionsForPosition(
         filePath: string,
         position: Position,
         workspacePath: string,
         token: CancellationToken
-    ): CompletionList | undefined {
-        return this._runEvaluatorWithCancellationToken(token, () => {
-            const sourceFileInfo = this._sourceFileMap.get(filePath);
-            if (!sourceFileInfo) {
-                return undefined;
-            }
+    ): Promise<CompletionList | undefined> {
+        const sourceFileInfo = this._sourceFileMap.get(filePath);
+        if (!sourceFileInfo) {
+            return undefined;
+        }
 
+        let completionList = this._runEvaluatorWithCancellationToken(token, () => {
             this._bindFile(sourceFileInfo);
 
-            let completionList = sourceFileInfo.sourceFile.getCompletionsForPosition(
+            return sourceFileInfo.sourceFile.getCompletionsForPosition(
                 position,
                 workspacePath,
                 this._configOptions,
@@ -954,25 +954,29 @@ export class Program {
                 () => this._buildModuleSymbolsMap(sourceFileInfo),
                 token
             );
-
-            if (completionList && this._extension?.completionListExtension) {
-                const pr = sourceFileInfo.sourceFile.getParseResults();
-                if (pr?.parseTree) {
-                    const offset = convertPositionToOffset(position, pr.tokenizerOutput.lines);
-                    if (offset) {
-                        completionList = this._extension.completionListExtension.updateCompletionList(
-                            completionList,
-                            pr.parseTree,
-                            filePath,
-                            offset,
-                            this._configOptions
-                        );
-                    }
-                }
-            }
-
-            return completionList;
         });
+
+        if (!completionList || !this._extension?.completionListExtension) {
+            return completionList;
+        }
+
+        const pr = sourceFileInfo.sourceFile.getParseResults();
+        const content = sourceFileInfo.sourceFile.getFileContents();
+        if (pr?.parseTree && content) {
+            const offset = convertPositionToOffset(position, pr.tokenizerOutput.lines);
+            if (offset) {
+                completionList = await this._extension.completionListExtension.updateCompletionList(
+                    completionList,
+                    pr.parseTree,
+                    content,
+                    offset,
+                    this._configOptions,
+                    token
+                );
+            }
+        }
+
+        return completionList;
     }
 
     resolveCompletionItem(filePath: string, completionItem: CompletionItem, token: CancellationToken) {

server/src/analyzer/service.ts

@@ -221,7 +221,7 @@ export class AnalyzerService {
         position: Position,
         workspacePath: string,
         token: CancellationToken
-    ): CompletionList | undefined {
+    ): Promise<CompletionList | undefined> {
         return this._program.getCompletionsForPosition(filePath, position, workspacePath, token);
     }
 
@@ -391,17 +391,24 @@ export class AnalyzerService {
                 }
 
                 // If the user has defined execution environments, then we ignore
-                // autoSearchPaths and leave it up to them to set extraPaths on them.
-                if (commandLineOptions.autoSearchPaths && configOptions.executionEnvironments.length === 0) {
-                    configOptions.addExecEnvironmentForAutoSearchPaths(this._fs);
+                // autoSearchPaths, extraPaths and leave it up to them to set
+                // extraPaths on the execution environments.
+                if (configOptions.executionEnvironments.length === 0) {
+                    configOptions.addExecEnvironmentForExtraPaths(
+                        this._fs,
+                        commandLineOptions.autoSearchPaths || false,
+                        commandLineOptions.extraPaths || []
+                    );
                 }
             }
             this._updateConfigFileWatcher();
             this._updateLibraryFileWatcher();
         } else {
-            if (commandLineOptions.autoSearchPaths) {
-                configOptions.addExecEnvironmentForAutoSearchPaths(this._fs);
-            }
+            configOptions.addExecEnvironmentForExtraPaths(
+                this._fs,
+                commandLineOptions.autoSearchPaths || false,
+                commandLineOptions.extraPaths || []
+            );
 
             configOptions.autoExcludeVenv = true;
         }

server/src/analyzer/staticExpressions.ts

@@ -117,7 +117,8 @@ export function evaluateStaticBoolLikeExpression(
 
 function _convertTupleToVersion(node: TupleNode): number | undefined {
     let comparisonVersion: number | undefined;
-    if (node.expressions.length === 2) {
+    // Ignore patch versions.
+    if (node.expressions.length >= 2) {
         if (
             node.expressions[0].nodeType === ParseNodeType.Number &&
             !node.expressions[0].isImaginary &&

server/src/analyzer/typeEvaluator.ts

@@ -7912,7 +7912,7 @@ export function createTypeEvaluator(importLookup: ImportLookup): TypeEvaluator {
                     const inferredYieldTypes: Type[] = [];
                     functionDecl.yieldExpressions.forEach((yieldNode) => {
                         if (isNodeReachable(yieldNode)) {
-                            if (yieldNode.nodeType == ParseNodeType.YieldFrom) {
+                            if (yieldNode.nodeType === ParseNodeType.YieldFrom) {
                                 const iteratorType = getTypeOfExpression(yieldNode.expression).type;
                                 const yieldType = getTypeFromIterable(
                                     iteratorType,

server/src/common/cancellationUtils.ts

@@ -207,7 +207,7 @@ export class OperationCanceledException extends ResponseError<void> {
     }
 
     static is(e: any) {
-        return e.code == ErrorCodes.RequestCancelled;
+        return e.code === ErrorCodes.RequestCancelled;
     }
 }
 

server/src/common/commandLineOptions.ts

@@ -61,6 +61,10 @@ export class CommandLineOptions {
     // execution environments.
     autoSearchPaths?: boolean;
 
+    // Extra paths to add to the default execution environment
+    // when user has not explicitly defined execution environments.
+    extraPaths?: string[];
+
     // Default type-checking rule set. Should be one of 'off',
     // 'basic', or 'strict'.
     typeCheckingMode?: string;

server/src/common/configOptions.ts

@@ -290,7 +290,7 @@ export function getNoTypeCheckingDiagnosticRuleSet(): DiagnosticRuleSet {
         enableTypeIgnoreComments: true,
         reportGeneralTypeIssues: 'none',
         reportTypeshedErrors: 'none',
-        reportMissingImports: 'none',
+        reportMissingImports: 'warning',
         reportMissingModuleSource: 'warning',
         reportMissingTypeStubs: 'none',
         reportImportCycles: 'none',
@@ -494,17 +494,31 @@ export class ConfigOptions {
         return execEnv;
     }
 
-    addExecEnvironmentForAutoSearchPaths(fs: FileSystem) {
-        // Auto-detect the common scenario where the sources are under the src folder
-        const srcPath = combinePaths(this.projectRoot, pathConsts.src);
-        if (fs.existsSync(srcPath) && !fs.existsSync(combinePaths(srcPath, '__init__.py'))) {
+    addExecEnvironmentForExtraPaths(fs: FileSystem, autoSearchPaths: boolean, extraPaths: string[]) {
+        const paths: string[] = [];
+
+        if (autoSearchPaths) {
+            // Auto-detect the common scenario where the sources are under the src folder
+            const srcPath = normalizePath(combinePaths(this.projectRoot, pathConsts.src));
+            if (fs.existsSync(srcPath) && !fs.existsSync(combinePaths(srcPath, '__init__.py'))) {
+                paths.push(srcPath);
+            }
+        }
+
+        if (extraPaths.length > 0) {
+            for (const p of extraPaths) {
+                paths.push(normalizePath(combinePaths(this.projectRoot, p)));
+            }
+        }
+
+        if (paths.length > 0) {
             const execEnv = new ExecutionEnvironment(
                 this.projectRoot,
                 this.defaultPythonVersion,
                 this.defaultPythonPlatform
             );
 
-            execEnv.extraPaths.push(srcPath);
+            execEnv.extraPaths.push(...paths);
 
             this.executionEnvironments.push(execEnv);
         }

server/src/common/extensibility.ts

@@ -6,7 +6,7 @@
 * Defines language service completion list extensibility.
 */
 
-import { CompletionList } from 'vscode-languageserver';
+import { CancellationToken, CompletionList } from 'vscode-languageserver';
 
 import { ModuleNode } from '../parser/parseNodes';
 import { ConfigOptions } from './configOptions';
@@ -21,6 +21,7 @@ export interface CompletionListExtension {
         ast: ModuleNode,
         content: string,
         position: number,
-        options: ConfigOptions
-    ): CompletionList;
+        options: ConfigOptions,
+        token: CancellationToken
+    ): Promise<CompletionList>;
 }

server/src/common/fileSystem.ts

@@ -36,9 +36,15 @@ export interface Stats {
     isSocket(): boolean;
 }
 
+export interface MkDirOptions {
+    recursive: boolean;
+    // Not supported on Windows so commented out.
+    // mode: string | number;
+}
+
 export interface FileSystem {
     existsSync(path: string): boolean;
-    mkdirSync(path: string): void;
+    mkdirSync(path: string, options?: MkDirOptions | number): void;
     chdir(path: string): void;
     readdirSync(path: string): string[];
     readFileSync(path: string, encoding?: null): Buffer;
@@ -84,8 +90,8 @@ class RealFileSystem implements FileSystem {
         return fs.existsSync(path);
     }
 
-    mkdirSync(path: string) {
-        fs.mkdirSync(path);
+    mkdirSync(path: string, options?: MkDirOptions | number) {
+        fs.mkdirSync(path, options);
     }
 
     chdir(path: string) {

server/src/languageServerBase.ts

@@ -81,6 +81,7 @@ export interface ServerSettings {
     disableLanguageServices?: boolean;
     disableOrganizeImports?: boolean;
     autoSearchPaths?: boolean;
+    extraPaths?: string[];
     watchForSourceChanges?: boolean;
     watchForLibraryChanges?: boolean;
 }
@@ -574,7 +575,7 @@ export abstract class LanguageServerBase implements LanguageServerInterface {
                 return;
             }
 
-            const completions = workspace.serviceInstance.getCompletionsForPosition(
+            const completions = await workspace.serviceInstance.getCompletionsForPosition(
                 filePath,
                 position,
                 workspace.rootPath,

server/src/languageService/analyzerServiceExecutor.ts

@@ -81,6 +81,7 @@ function _getCommandLineOptions(
     }
 
     commandLineOptions.autoSearchPaths = serverSettings.autoSearchPaths;
+    commandLineOptions.extraPaths = serverSettings.extraPaths;
 
     return commandLineOptions;
 }

server/src/languageService/referencesProvider.ts

@@ -160,7 +160,7 @@ export class ReferencesProvider {
         }
 
         // Special case module names, which don't have references.
-        if (node.parent?.nodeType == ParseNodeType.ModuleName) {
+        if (node.parent?.nodeType === ParseNodeType.ModuleName) {
             return undefined;
         }
 

server/src/server.ts

@@ -46,6 +46,11 @@ class PyrightServer extends LanguageServerBase {
                     serverSettings.typeshedPath = normalizeSlashes(typeshedPaths[0]);
                 }
                 serverSettings.autoSearchPaths = !!pythonAnalysisSection.autoSearchPaths;
+
+                const extraPaths = pythonAnalysisSection.extraPaths;
+                if (extraPaths && isArray(extraPaths) && extraPaths.length > 0) {
+                    serverSettings.extraPaths = extraPaths.map((p) => normalizeSlashes(p));
+                }
             } else {
                 serverSettings.autoSearchPaths = true;
             }

server/src/tests/checker.test.ts

@@ -1431,6 +1431,12 @@ test('Callable1', () => {
     validateResults(analysisResults, 3);
 });
 
+test('ThreePartVersion1', () => {
+    const analysisResults = TestUtils.typeAnalyzeSampleFiles(['threePartVersion1.py']);
+
+    validateResults(analysisResults, 0);
+});
+
 test('Unions1', () => {
     const configOptions = new ConfigOptions('.');
 

server/src/tests/config.test.ts

@@ -261,3 +261,27 @@ test('AutoSearchPathsOnWithConfigExecEnv', () => {
 
     assert.deepEqual(configOptions.executionEnvironments, expectedExecEnvs);
 });
+
+test('AutoSearchPathsOnAndExtraPaths', () => {
+    const cwd = normalizePath(combinePaths(process.cwd(), '../server/src/tests/samples/project_src_with_extra_paths'));
+    const nullConsole = new NullConsole();
+    const service = new AnalyzerService('<default>', createFromRealFileSystem(nullConsole), nullConsole);
+    const commandLineOptions = new CommandLineOptions(cwd, false);
+    commandLineOptions.autoSearchPaths = true;
+    commandLineOptions.extraPaths = ['src/_vendored'];
+    service.setOptions(commandLineOptions);
+
+    const configOptions = service.test_getConfigOptions(commandLineOptions);
+
+    // An execution environment is automatically created with src and src/_vendored as extra paths
+    const expectedExecEnvs = [
+        {
+            pythonPlatform: undefined,
+            pythonVersion: 776,
+            root: cwd,
+            extraPaths: [normalizePath(combinePaths(cwd, 'src')), normalizePath(combinePaths(cwd, 'src', '_vendored'))],
+        },
+    ];
+
+    assert.deepEqual(configOptions.executionEnvironments, expectedExecEnvs);
+});

server/src/tests/fourslash/completions.excluded.fourslash.ts

@@ -1,4 +1,5 @@
 /// <reference path="fourslash.ts" />
+// @asynctest: true
 
 // @filename: test.py
 //// a = 42

server/src/tests/fourslash/completions.fourslash.ts

@@ -1,4 +1,5 @@
 /// <reference path="fourslash.ts" />
+// @asynctest: true
 
 // @filename: test.py
 //// import time

server/src/tests/fourslash/completions.included.fourslash.ts

@@ -1,4 +1,5 @@
 /// <reference path="fourslash.ts" />
+// @asynctest: true
 
 // @filename: test.py
 //// a = 42

server/src/tests/fourslash/completions.libCodeAndStub.fourslash.skip.ts

@@ -1,4 +1,5 @@
 /// <reference path="fourslash.ts" />
+// @asynctest: true
 
 // @filename: mspythonconfig.json
 //// {

server/src/tests/fourslash/completions.libCodeNoStub.fourslash.ts

@@ -1,4 +1,5 @@
 /// <reference path="fourslash.ts" />
+// @asynctest: true
 
 // @filename: mspythonconfig.json
 //// {

server/src/tests/fourslash/completions.libStub.fourslash.ts

@@ -1,4 +1,5 @@
 /// <reference path="fourslash.ts" />
+// @asynctest: true
 
 // @filename: mspythonconfig.json
 //// {

server/src/tests/fourslash/fourslash.ts

@@ -130,7 +130,7 @@ declare namespace _ {
             map: {
                 [marker: string]: { completions: { label: string; documentation?: { kind: string; value: string } }[] };
             }
-        ): void;
+        ): Promise<void>;
         verifySignature(map: {
             [marker: string]: {
                 noSig?: boolean;

server/src/tests/harness/fourslash/runner.ts

@@ -67,23 +67,41 @@ export function runFourSlashTestContent(
 
 function runCode(code: string, state: TestState): void {
     // Compile and execute the test
-    const wrappedCode = `(function(helper, Consts) {
-${code}
-})`;
 
-    // TODO: figure out how to use this with async
     try {
-        const f = eval(wrappedCode);
-        f(state, Consts);
-
-        markDone();
+        if (state.asyncTest) {
+            runAsyncCode();
+        } else {
+            runPlainCode();
+        }
     } catch (error) {
         markDone(error);
     }
 
+    function runAsyncCode() {
+        const wrappedCode = `(async function(helper, Consts) {
+            ${code}
+            })`;
+        const f = eval(wrappedCode);
+        f(state, Consts)
+            .then(() => {
+                markDone();
+            })
+            .catch((e: any) => {
+                markDone(e);
+            });
+    }
+
+    function runPlainCode() {
+        const wrappedCode = `(function(helper, Consts) {
+            ${code}
+            })`;
+        const f = eval(wrappedCode);
+        f(state, Consts);
+        markDone();
+    }
+
     function markDone(...args: any[]) {
-        if (!state.asyncTest) {
-            state.markTestDone(...args);
-        }
+        state.markTestDone(...args);
     }
 }

server/src/tests/harness/fourslash/testState.ts

@@ -648,7 +648,7 @@ export class TestState {
             const actualText = textEdit.newText;
             const expectedText: string = Object.values(files)[0];
 
-            if (actualText != expectedText) {
+            if (actualText !== expectedText) {
                 this._raiseError(
                     `doesn't contain expected result: ${stringify(expectedText)}, actual: ${stringify(actualText)}`
                 );
@@ -774,10 +774,10 @@ export class TestState {
         this._verifyTextMatches(this._rangeText(this._getOnlyRange()), !!includeWhiteSpace, expectedText);
     }
 
-    verifyCompletion(
+    async verifyCompletion(
         verifyMode: 'exact' | 'included' | 'excluded',
         map: { [marker: string]: { completions: { label: string; documentation?: { kind: string; value: string } }[] } }
-    ) {
+    ): Promise<void> {
         this._analyze();
 
         for (const marker of this.getMarkers()) {
@@ -785,7 +785,7 @@ export class TestState {
             const expectedCompletions = map[this.getMarkerName(marker)].completions;
             const completionPosition = this._convertOffsetToPosition(filePath, marker.position);
 
-            const result = this.program.getCompletionsForPosition(
+            const result = await this.program.getCompletionsForPosition(
                 filePath,
                 completionPosition,
                 this.workspace.rootPath,

server/src/tests/samples/project_src_with_extra_paths/src/_vendored/vendored1.py

@@ -0,0 +1 @@
+MSG = 'hello'

server/src/tests/samples/project_src_with_extra_paths/src/module1.py

@@ -0,0 +1,2 @@
+from vendored1 import MSG
+print(MSG)

server/src/tests/samples/threePartVersion1.py

@@ -0,0 +1,26 @@
+import sys
+
+from datetime import datetime, timezone, timedelta
+from typing import overload, Optional
+
+# Overload was broken before 3.5.2.
+# This sort of hack is seen in some type-annotated code to prevent crashes.
+if sys.version_info < (3, 5, 2):
+    def overload(f):
+        return f
+
+
+@overload
+def from_json_timestamp(ts: int) -> datetime:
+    ...
+
+@overload
+def from_json_timestamp(ts: None) -> None:
+    ...
+
+def from_json_timestamp(ts: Optional[int]) -> Optional[datetime]:
+    return None if ts is None else (datetime(1970, 1, 1, tzinfo=timezone.utc) + timedelta(milliseconds=ts))
+
+result1: datetime = from_json_timestamp(2418049)
+result3: None = from_json_timestamp(None)
+