Added new top-level "extraPaths" config option for pythonconfig.json that specifies the default extraPaths to use when no execution environments apply to a file. Changed settings logic to use the new default extraPaths mechanism for the "python.analysis.extraPaths" setting. (#1632)

Co-authored-by: Eric Traut <erictr@microsoft.com>
This commit is contained in:
Eric Traut 2021-03-15 17:43:31 -07:00 committed by GitHub
parent a2d0fa84c6
commit cb4aad13dc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 133 additions and 125 deletions

View File

@ -24,6 +24,8 @@ Relative paths specified within the config file are relative to the config file
**verboseOutput** [boolean]: Specifies whether output logs should be verbose. This is useful when diagnosing certain problems like import resolution issues.
**extraPaths** [array of strings, optional]: Additional search paths that will be used when searching for modules imported by files.
**pythonVersion** [string, optional]: Specifies the version of Python that will be used to execute the source code. The version should be specified as a string in the format "M.m" where M is the major version and m is the minor (e.g. `"3.0"` or `"3.6"`). If a version is provided, pyright will generate errors if the source code makes use of language features that are not supported in that version. It will also tailor its use of type stub files, which conditionalizes type definitions based on the version. If no version is specified, pyright will use the version of the current python interpreter, if one is present.
**pythonPlatform** [string, optional]: Specifies the target platform that will be used to execute the source code. Should be one of `"Windows"`, `"Darwin"` or `"Linux"`. If specified, pyright will tailor its use of type stub files, which conditionalize type definitions based on the platform. If no platform is specified, pyright will use the current platform.
@ -150,7 +152,7 @@ The following settings can be specified for each execution environment.
**root** [string, required]: Root path for the code that will execute within this execution environment.
**extraPaths** [array of strings, optional]: Additional search paths (in addition to the root path) that will be used when searching for modules imported by files within this execution environment. Note that each files execution environment mapping is independent, so if file A is in one execution environment ane imports a second file B within a second execution environment, any imports from B will use the extraPaths in the second execution environment.
**extraPaths** [array of strings, optional]: Additional search paths (in addition to the root path) that will be used when searching for modules imported by files within this execution environment. If specified, this overrides the default extraPaths setting when resolving imports for files within this execution environment. Note that each files execution environment mapping is independent, so if file A is in one execution environment ane imports a second file B within a second execution environment, any imports from B will use the extraPaths in the second execution environment.
**pythonVersion** [string, optional]: The version of Python used for this execution environment. If not specified, the global `pythonVersion` setting is used instead.

View File

@ -471,6 +471,11 @@ export class AnalyzerService {
// by the config file, so initialize them upfront.
configOptions.defaultPythonPlatform = commandLineOptions.pythonPlatform;
configOptions.defaultPythonVersion = commandLineOptions.pythonVersion;
configOptions.ensureDefaultExtraPaths(
this._fs,
commandLineOptions.autoSearchPaths || false,
commandLineOptions.extraPaths
);
if (commandLineOptions.fileSpecs.length > 0) {
commandLineOptions.fileSpecs.forEach((fileSpec) => {
@ -526,25 +531,8 @@ export class AnalyzerService {
configOptions.autoExcludeVenv = true;
}
}
// If the user has defined execution environments, then we ignore
// autoSearchPaths, extraPaths and leave it up to them to set
// extraPaths on the execution environments.
if (configOptions.executionEnvironments.length === 0) {
configOptions.addExecEnvironmentForExtraPaths(
this._fs,
commandLineOptions.autoSearchPaths || false,
commandLineOptions.extraPaths || []
);
}
}
} else {
configOptions.addExecEnvironmentForExtraPaths(
this._fs,
commandLineOptions.autoSearchPaths || false,
commandLineOptions.extraPaths || []
);
configOptions.autoExcludeVenv = true;
configOptions.applyDiagnosticOverrides(commandLineOptions.diagnosticSeverityOverrides);
}

View File

@ -71,6 +71,7 @@ export function createConfigOptionsFrom(jsonObject: any): ConfigOptions {
configOptions.venv = jsonObject.venv;
configOptions.defaultPythonVersion = jsonObject.defaultPythonVersion;
configOptions.defaultPythonPlatform = jsonObject.defaultPythonPlatform;
configOptions.defaultExtraPaths = jsonObject.defaultExtraPaths;
configOptions.diagnosticRuleSet = jsonObject.diagnosticRuleSet;
configOptions.executionEnvironments = jsonObject.executionEnvironments;
configOptions.autoImportCompletions = jsonObject.autoImportCompletions;

View File

@ -39,10 +39,16 @@ export enum PythonPlatform {
export class ExecutionEnvironment {
// Default to "." which indicates every file in the project.
constructor(root: string, defaultPythonVersion?: PythonVersion, defaultPythonPlatform?: string) {
constructor(
root: string,
defaultPythonVersion: PythonVersion | undefined,
defaultPythonPlatform: string | undefined,
defaultExtraPaths: string[] | undefined
) {
this.root = root;
this.pythonVersion = defaultPythonVersion || latestStablePythonVersion;
this.pythonPlatform = defaultPythonPlatform;
this.extraPaths = defaultExtraPaths || [];
}
// Root directory for execution - absolute or relative to the
@ -616,6 +622,9 @@ export class ConfigOptions {
// Default pythonPlatform. Can be overridden by executionEnvironment.
defaultPythonPlatform?: string;
// Default extraPaths. Can be overridden by executionEnvironment.
defaultExtraPaths?: string[];
//---------------------------------------------------------------
// Internal-only switches
@ -645,44 +654,24 @@ export class ConfigOptions {
});
if (!execEnv) {
execEnv = new ExecutionEnvironment(this.projectRoot, this.defaultPythonVersion, this.defaultPythonPlatform);
execEnv = new ExecutionEnvironment(
this.projectRoot,
this.defaultPythonVersion,
this.defaultPythonPlatform,
this.defaultExtraPaths
);
}
return execEnv;
}
getDefaultExecEnvironment(): ExecutionEnvironment {
return new ExecutionEnvironment(this.projectRoot, this.defaultPythonVersion, this.defaultPythonPlatform);
}
addExecEnvironmentForExtraPaths(fs: FileSystem, autoSearchPaths: boolean, extraPaths: string[]) {
const paths: string[] = [];
if (autoSearchPaths) {
// Auto-detect the common scenario where the sources are under the src folder
const srcPath = resolvePaths(this.projectRoot, pathConsts.src);
if (fs.existsSync(srcPath) && !fs.existsSync(resolvePaths(srcPath, '__init__.py'))) {
paths.push(srcPath);
}
}
if (extraPaths.length > 0) {
for (const p of extraPaths) {
paths.push(resolvePaths(this.projectRoot, p));
}
}
if (paths.length > 0) {
const execEnv = new ExecutionEnvironment(
this.projectRoot,
this.defaultPythonVersion,
this.defaultPythonPlatform
);
execEnv.extraPaths.push(...paths);
this.executionEnvironments.push(execEnv);
}
return new ExecutionEnvironment(
this.projectRoot,
this.defaultPythonVersion,
this.defaultPythonPlatform,
this.defaultExtraPaths
);
}
// Initialize the structure from a JSON object.
@ -1195,6 +1184,23 @@ export class ConfigOptions {
}
}
// Read the default "extraPaths".
if (configObj.extraPaths !== undefined) {
this.defaultExtraPaths = [];
if (!Array.isArray(configObj.extraPaths)) {
console.error(`Config "extraPaths" field must contain an array.`);
} else {
const pathList = configObj.extraPaths as string[];
pathList.forEach((path, pathIndex) => {
if (typeof path !== 'string') {
console.error(`Config "extraPaths" field ${pathIndex} must be a string.`);
} else {
this.defaultExtraPaths!.push(normalizePath(combinePaths(this.projectRoot, path)));
}
});
}
}
// Read the default "pythonVersion".
if (configObj.pythonVersion !== undefined) {
if (typeof configObj.pythonVersion === 'string') {
@ -1362,6 +1368,28 @@ export class ConfigOptions {
}
}
ensureDefaultExtraPaths(fs: FileSystem, autoSearchPaths: boolean, extraPaths: string[] | undefined) {
const paths: string[] = [];
if (autoSearchPaths) {
// Auto-detect the common scenario where the sources are under the src folder
const srcPath = resolvePaths(this.projectRoot, pathConsts.src);
if (fs.existsSync(srcPath) && !fs.existsSync(resolvePaths(srcPath, '__init__.py'))) {
paths.push(srcPath);
}
}
if (extraPaths && extraPaths.length > 0) {
for (const p of extraPaths) {
paths.push(resolvePaths(this.projectRoot, p));
}
}
if (paths.length > 0) {
this.defaultExtraPaths = paths;
}
}
applyDiagnosticOverrides(diagnosticSeverityOverrides: DiagnosticSeverityOverridesMap | undefined) {
if (!diagnosticSeverityOverrides) {
return;
@ -1407,7 +1435,8 @@ export class ConfigOptions {
const newExecEnv = new ExecutionEnvironment(
this.projectRoot,
this.defaultPythonVersion,
this.defaultPythonPlatform
this.defaultPythonPlatform,
this.defaultExtraPaths
);
// Validate the root.

View File

@ -15,20 +15,19 @@ import { ConfigOptions, ExecutionEnvironment } from '../common/configOptions';
import { NullConsole } from '../common/console';
import { createFromRealFileSystem } from '../common/fileSystem';
import { combinePaths, getBaseFileName, normalizePath, normalizeSlashes } from '../common/pathUtils';
import { latestStablePythonVersion } from '../common/pythonVersion';
test('FindFilesWithConfigFile', () => {
const cwd = normalizePath(process.cwd());
const service = new AnalyzerService('<default>', createFromRealFileSystem(), new NullConsole());
const commandLineOptions = new CommandLineOptions(cwd, true);
const commandLineOptions = new CommandLineOptions(cwd, /* fromVsCodeExtension */ true);
commandLineOptions.configFilePath = 'src/tests/samples/project1';
const configOptions = service.test_getConfigOptions(commandLineOptions);
service.setOptions(commandLineOptions);
// The config file specifies a single file spec (a directory).
assert.equal(configOptions.include.length, 1, `failed creating options from ${cwd}`);
assert.equal(
assert.strictEqual(configOptions.include.length, 1, `failed creating options from ${cwd}`);
assert.strictEqual(
normalizeSlashes(configOptions.projectRoot),
normalizeSlashes(combinePaths(cwd, commandLineOptions.configFilePath))
);
@ -38,13 +37,13 @@ test('FindFilesWithConfigFile', () => {
// The config file specifies a subdirectory, so we should find
// only two of the three "*.py" files present in the project
// directory.
assert.equal(fileList.length, 2);
assert.strictEqual(fileList.length, 2);
});
test('FindFilesVirtualEnvAutoDetectExclude', () => {
const cwd = normalizePath(process.cwd());
const service = new AnalyzerService('<default>', createFromRealFileSystem(), new NullConsole());
const commandLineOptions = new CommandLineOptions(cwd, true);
const commandLineOptions = new CommandLineOptions(cwd, /* fromVsCodeExtension */ true);
commandLineOptions.configFilePath = 'src/tests/samples/project_with_venv_auto_detect_exclude';
service.setOptions(commandLineOptions);
@ -56,13 +55,13 @@ test('FindFilesVirtualEnvAutoDetectExclude', () => {
// There are 3 python files in the workspace, outside of myvenv
// There is 1 python file in myvenv, which should be excluded
const fileNames = fileList.map((p) => getBaseFileName(p)).sort();
assert.deepEqual(fileNames, ['sample1.py', 'sample2.py', 'sample3.py']);
assert.deepStrictEqual(fileNames, ['sample1.py', 'sample2.py', 'sample3.py']);
});
test('FindFilesVirtualEnvAutoDetectInclude', () => {
const cwd = normalizePath(process.cwd());
const service = new AnalyzerService('<default>', createFromRealFileSystem(), new NullConsole());
const commandLineOptions = new CommandLineOptions(cwd, true);
const commandLineOptions = new CommandLineOptions(cwd, /* fromVsCodeExtension */ true);
commandLineOptions.configFilePath = 'src/tests/samples/project_with_venv_auto_detect_include';
service.setOptions(commandLineOptions);
@ -74,14 +73,14 @@ test('FindFilesVirtualEnvAutoDetectInclude', () => {
// There is 1 more python file in excluded folder
// There is 1 python file in myvenv, which should be included
const fileNames = fileList.map((p) => getBaseFileName(p)).sort();
assert.deepEqual(fileNames, ['library1.py', 'sample1.py', 'sample2.py', 'sample3.py']);
assert.deepStrictEqual(fileNames, ['library1.py', 'sample1.py', 'sample2.py', 'sample3.py']);
});
test('FileSpecNotAnArray', () => {
const cwd = normalizePath(process.cwd());
const nullConsole = new NullConsole();
const service = new AnalyzerService('<default>', createFromRealFileSystem(nullConsole), nullConsole);
const commandLineOptions = new CommandLineOptions(cwd, false);
const commandLineOptions = new CommandLineOptions(cwd, /* fromVsCodeExtension */ false);
commandLineOptions.configFilePath = 'src/tests/samples/project2';
service.setOptions(commandLineOptions);
@ -95,7 +94,7 @@ test('FileSpecNotAString', () => {
const cwd = normalizePath(process.cwd());
const nullConsole = new NullConsole();
const service = new AnalyzerService('<default>', createFromRealFileSystem(nullConsole), nullConsole);
const commandLineOptions = new CommandLineOptions(cwd, false);
const commandLineOptions = new CommandLineOptions(cwd, /* fromVsCodeExtension */ false);
commandLineOptions.configFilePath = 'src/tests/samples/project3';
service.setOptions(commandLineOptions);
@ -109,7 +108,7 @@ test('SomeFileSpecsAreInvalid', () => {
const cwd = normalizePath(process.cwd());
const nullConsole = new NullConsole();
const service = new AnalyzerService('<default>', createFromRealFileSystem(nullConsole), nullConsole);
const commandLineOptions = new CommandLineOptions(cwd, false);
const commandLineOptions = new CommandLineOptions(cwd, /* fromVsCodeExtension */ false);
commandLineOptions.configFilePath = 'src/tests/samples/project4';
service.setOptions(commandLineOptions);
@ -117,9 +116,9 @@ test('SomeFileSpecsAreInvalid', () => {
// The config file specifies four file specs in the include array
// and one in the exclude array.
assert.equal(configOptions.include.length, 4, `failed creating options from ${cwd}`);
assert.equal(configOptions.exclude.length, 1);
assert.equal(
assert.strictEqual(configOptions.include.length, 4, `failed creating options from ${cwd}`);
assert.strictEqual(configOptions.exclude.length, 1);
assert.strictEqual(
normalizeSlashes(configOptions.projectRoot),
normalizeSlashes(combinePaths(cwd, commandLineOptions.configFilePath))
);
@ -127,14 +126,14 @@ test('SomeFileSpecsAreInvalid', () => {
const fileList = service.test_getFileNamesFromFileSpecs();
// We should receive two final files that match the include/exclude rules.
assert.equal(fileList.length, 2);
assert.strictEqual(fileList.length, 2);
});
test('ConfigBadJson', () => {
const cwd = normalizePath(process.cwd());
const nullConsole = new NullConsole();
const service = new AnalyzerService('<default>', createFromRealFileSystem(nullConsole), nullConsole);
const commandLineOptions = new CommandLineOptions(cwd, false);
const commandLineOptions = new CommandLineOptions(cwd, /* fromVsCodeExtension */ false);
commandLineOptions.configFilePath = 'src/tests/samples/project5';
service.setOptions(commandLineOptions);
@ -149,21 +148,31 @@ test('FindExecEnv1', () => {
const configOptions = new ConfigOptions(cwd);
// Build a config option with three execution environments.
const execEnv1 = new ExecutionEnvironment('src/foo');
const execEnv1 = new ExecutionEnvironment(
'src/foo',
/* defaultPythonVersion */ undefined,
/* defaultPythonPlatform */ undefined,
/* defaultExtraPaths */ undefined
);
configOptions.executionEnvironments.push(execEnv1);
const execEnv2 = new ExecutionEnvironment('src');
const execEnv2 = new ExecutionEnvironment(
'src',
/* defaultPythonVersion */ undefined,
/* defaultPythonPlatform */ undefined,
/* defaultExtraPaths */ undefined
);
configOptions.executionEnvironments.push(execEnv2);
const file1 = normalizeSlashes(combinePaths(cwd, 'src/foo/bar.py'));
assert.equal(configOptions.findExecEnvironment(file1), execEnv1);
assert.strictEqual(configOptions.findExecEnvironment(file1), execEnv1);
const file2 = normalizeSlashes(combinePaths(cwd, 'src/foo2/bar.py'));
assert.equal(configOptions.findExecEnvironment(file2), execEnv2);
assert.strictEqual(configOptions.findExecEnvironment(file2), execEnv2);
// If none of the execution environments matched, we should get
// a default environment with the root equal to that of the config.
const file4 = '/nothing/bar.py';
const defaultExecEnv = configOptions.findExecEnvironment(file4);
assert.equal(normalizeSlashes(defaultExecEnv.root), normalizeSlashes(configOptions.projectRoot));
assert.strictEqual(normalizeSlashes(defaultExecEnv.root), normalizeSlashes(configOptions.projectRoot));
});
test('PythonPlatform', () => {
@ -183,104 +192,82 @@ test('PythonPlatform', () => {
configOptions.initializeFromJson(json, undefined, nullConsole);
const env = configOptions.executionEnvironments[0];
assert.equal(env.pythonPlatform, 'platform');
assert.strictEqual(env.pythonPlatform, 'platform');
});
test('AutoSearchPathsOn', () => {
const cwd = normalizePath(combinePaths(process.cwd(), 'src/tests/samples/project_src'));
const nullConsole = new NullConsole();
const service = new AnalyzerService('<default>', createFromRealFileSystem(nullConsole), nullConsole);
const commandLineOptions = new CommandLineOptions(cwd, false);
const commandLineOptions = new CommandLineOptions(cwd, /* fromVsCodeExtension */ false);
commandLineOptions.autoSearchPaths = true;
service.setOptions(commandLineOptions);
const configOptions = service.test_getConfigOptions(commandLineOptions);
// An execution environment is automatically created with src folder as extra path
const expectedExecEnvs = [
{
pythonPlatform: undefined,
pythonVersion: latestStablePythonVersion,
root: cwd,
extraPaths: [normalizePath(combinePaths(cwd, 'src'))],
},
];
assert.deepEqual(configOptions.executionEnvironments, expectedExecEnvs);
const expectedExtraPaths = [normalizePath(combinePaths(cwd, 'src'))];
assert.deepStrictEqual(configOptions.defaultExtraPaths, expectedExtraPaths);
});
test('AutoSearchPathsOff', () => {
const cwd = normalizePath(combinePaths(process.cwd(), 'src/tests/samples/project_src'));
const nullConsole = new NullConsole();
const service = new AnalyzerService('<default>', createFromRealFileSystem(nullConsole), nullConsole);
const commandLineOptions = new CommandLineOptions(cwd, false);
const commandLineOptions = new CommandLineOptions(cwd, /* fromVsCodeExtension */ false);
commandLineOptions.autoSearchPaths = false;
service.setOptions(commandLineOptions);
const configOptions = service.test_getConfigOptions(commandLineOptions);
assert.deepEqual(configOptions.executionEnvironments, []);
assert.deepStrictEqual(configOptions.executionEnvironments, []);
});
test('AutoSearchPathsOnSrcIsPkg', () => {
const cwd = normalizePath(combinePaths(process.cwd(), 'src/tests/samples/project_src_is_pkg'));
const nullConsole = new NullConsole();
const service = new AnalyzerService('<default>', createFromRealFileSystem(nullConsole), nullConsole);
const commandLineOptions = new CommandLineOptions(cwd, false);
const commandLineOptions = new CommandLineOptions(cwd, /* fromVsCodeExtension */ false);
commandLineOptions.autoSearchPaths = true;
service.setOptions(commandLineOptions);
const configOptions = service.test_getConfigOptions(commandLineOptions);
// The src folder is a package (has __init__.py) and so should not be automatically added as extra path
assert.deepEqual(configOptions.executionEnvironments, []);
assert.deepStrictEqual(configOptions.executionEnvironments, []);
});
test('AutoSearchPathsOnWithConfigExecEnv', () => {
const cwd = normalizePath(combinePaths(process.cwd(), 'src/tests/samples/project_src_with_config_exec_env'));
const cwd = normalizePath(combinePaths(process.cwd(), 'src/tests/samples/project_src_with_config_extra_paths'));
const nullConsole = new NullConsole();
const service = new AnalyzerService('<default>', createFromRealFileSystem(nullConsole), nullConsole);
const commandLineOptions = new CommandLineOptions(cwd, false);
const commandLineOptions = new CommandLineOptions(cwd, /* fromVsCodeExtension */ false);
commandLineOptions.configFilePath = combinePaths(cwd, 'mspythonconfig.json');
commandLineOptions.autoSearchPaths = true;
service.setOptions(commandLineOptions);
const configOptions = service.test_getConfigOptions(commandLineOptions);
// Execution environments are found in the config file, we do not modify them
// (so auto search path option is ignored)
const expectedExecEnvs = [
{
pythonPlatform: 'Windows',
pythonVersion: 0x0305,
root: cwd,
extraPaths: [],
},
];
// The extraPaths in the config file should override the setting.
const expectedExtraPaths: string[] = [];
assert.deepEqual(configOptions.executionEnvironments, expectedExecEnvs);
assert.deepStrictEqual(configOptions.defaultExtraPaths, expectedExtraPaths);
});
test('AutoSearchPathsOnAndExtraPaths', () => {
const cwd = normalizePath(combinePaths(process.cwd(), 'src/tests/samples/project_src_with_extra_paths'));
const cwd = normalizePath(combinePaths(process.cwd(), 'src/tests/samples/project_src_with_config_no_extra_paths'));
const nullConsole = new NullConsole();
const service = new AnalyzerService('<default>', createFromRealFileSystem(nullConsole), nullConsole);
const commandLineOptions = new CommandLineOptions(cwd, false);
const commandLineOptions = new CommandLineOptions(cwd, /* fromVsCodeExtension */ false);
commandLineOptions.autoSearchPaths = true;
commandLineOptions.extraPaths = ['src/_vendored'];
service.setOptions(commandLineOptions);
const configOptions = service.test_getConfigOptions(commandLineOptions);
// An execution environment is automatically created with src and src/_vendored as extra paths
const expectedExecEnvs = [
{
pythonPlatform: undefined,
pythonVersion: latestStablePythonVersion,
root: cwd,
extraPaths: [normalizePath(combinePaths(cwd, 'src')), normalizePath(combinePaths(cwd, 'src', '_vendored'))],
},
const expectedExtraPaths: string[] = [
normalizePath(combinePaths(cwd, 'src')),
normalizePath(combinePaths(cwd, 'src', '_vendored')),
];
assert.deepEqual(configOptions.executionEnvironments, expectedExecEnvs);
assert.deepStrictEqual(configOptions.defaultExtraPaths, expectedExtraPaths);
});

View File

@ -1,9 +0,0 @@
{
"executionEnvironments": [
{
"root": ".",
"pythonVersion": "3.5",
"pythonPlatform": "Windows"
}
]
}

View File

@ -70,7 +70,12 @@ export function parseText(
export function parseSampleFile(
fileName: string,
diagSink: DiagnosticSink,
execEnvironment = new ExecutionEnvironment('.')
execEnvironment = new ExecutionEnvironment(
'.',
/* defaultPythonVersion */ undefined,
/* defaultPythonPlatform */ undefined,
/* defaultExtraPaths */ undefined
)
): FileParseResult {
const text = readSampleFile(fileName);
const parseOptions = new ParseOptions();