Added a new configuration switch disableBytesTypePromotions that controls whether bytearray and memoryview should be implicit subtypes of bytes. This switch defaults to false in basic type checking mode and true in strict mode. Eventually, we will probably have it default to true in all modes. (#6024)

Co-authored-by: Eric Traut <erictr@microsoft.com>
2024-10-26 10:55:06 +03:00 · 2023-09-25 12:00:40 -07:00 · 2023-09-25 12:00:40 -07:00 · fe1e16ec1e
commit fe1e16ec1e
parent 21aee77eb5
13 changed files with 79 additions and 15 deletions
--- a/docs/configuration.md
+++ b/docs/configuration.md
@ -58,7 +58,9 @@ The following settings control pyright’s diagnostic output (warnings or errors

 <a name="deprecateTypingAliases"></a> **deprecateTypingAliases** [boolean]: PEP 585 indicates that aliases to types in standard collections that were introduced solely to support generics are deprecated as of Python 3.9. This switch controls whether these are treated as deprecated. This applies only when pythonVersion is 3.9 or newer. The default value for this setting is `false` but may be switched to `true` in the future.

-<a name="enableExperimentalFeatures"></a> **enableExperimentalFeatures** [boolean]: Enables a set of experimental (mostly undocumented) features that correspond to proposed or exploratory changes to the Python typing standard. These features will likely change or be removed, so they should not be used except for experimentation purposes.
+<a name="enableExperimentalFeatures"></a> **enableExperimentalFeatures** [boolean]: Enables a set of experimental (mostly undocumented) features that correspond to proposed or exploratory changes to the Python typing standard. These features will likely change or be removed, so they should not be used except for experimentation purposes. The default value for this setting is `false`.
+
+<a name="disableBytesTypePromotions"></a> **disableBytesTypePromotions** [boolean]: Disables legacy behavior where `bytearray` and `memoryview` are considered subtypes of `bytes`. [PEP 688](https://peps.python.org/pep-0688/#no-special-meaning-for-bytes) deprecates this behavior, but this switch is provided to restore the older behavior. The default value for this setting is `false`.

 <a name="reportGeneralTypeIssues"></a> **reportGeneralTypeIssues** [boolean or string, optional]: Generate or suppress diagnostics for general type inconsistencies, unsupported operations, argument/parameter mismatches, etc. This covers all of the basic type-checking rules not covered by other rules. It does not include syntax errors. The default value for this setting is `"error"`.

@ -303,6 +305,7 @@ The following table lists the default severity levels for each diagnostic rule w
 | analyzeUnannotatedFunctions               | true       | true       | true       |
 | strictParameterNoneValue                  | true       | true       | true       |
 | enableTypeIgnoreComments                  | true       | true       | true       |
+| disableBytesTypePromotions                | false      | false      | true       |
 | strictListInference                       | false      | false      | true       |
 | strictDictionaryInference                 | false      | false      | true       |
 | strictSetInference                        | false      | false      | true       |
--- a/packages/pyright-internal/src/analyzer/patternMatching.ts
+++ b/packages/pyright-internal/src/analyzer/patternMatching.ts
@ -653,7 +653,7 @@ function narrowTypeBasedOnClassPattern(
    // If this is a class (but not a type alias that refers to a class),
    // specialize it with Unknown type arguments.
    if (isClass(exprType) && !exprType.typeAliasInfo) {
-        exprType = ClassType.cloneForPromotionType(exprType, /* isTypeArgumentExplicit */ false);
+        exprType = ClassType.cloneRemoveTypePromotions(exprType);
        exprType = specializeClassType(exprType);
    }

--- a/packages/pyright-internal/src/analyzer/typeEvaluator.ts
+++ b/packages/pyright-internal/src/analyzer/typeEvaluator.ts
@ -424,6 +424,7 @@ const nonSubscriptableBuiltinTypes: Map<string, PythonVersion> = new Map([
 const typePromotions: Map<string, string[]> = new Map([
    ['builtins.float', ['builtins.int']],
    ['builtins.complex', ['builtins.float', 'builtins.int']],
+    ['builtins.bytes', ['builtins.bytearray', 'builtins.memoryview']],
 ]);

 interface SymbolResolutionStackEntry {
@ -1188,6 +1189,21 @@ export function createTypeEvaluator(importLookup: ImportLookup, evaluatorOptions
            validateTypeIsInstantiable(typeResult, flags, node);
        }

+        // Should we disable type promotions for bytes?
+        if (
+            isInstantiableClass(typeResult.type) &&
+            typeResult.type.includePromotions &&
+            !typeResult.type.includeSubclasses &&
+            ClassType.isBuiltIn(typeResult.type, 'bytes')
+        ) {
+            if (AnalyzerNodeInfo.getFileInfo(node).diagnosticRuleSet.disableBytesTypePromotions) {
+                typeResult = {
+                    ...typeResult,
+                    type: ClassType.cloneRemoveTypePromotions(typeResult.type),
+                };
+            }
+        }
+
        writeTypeCache(node, typeResult, flags, inferenceContext, /* allowSpeculativeCaching */ true);

        // If there was an expected type, make sure that the result type is compatible.
@ -1540,6 +1556,10 @@ export function createTypeEvaluator(importLookup: ImportLookup, evaluatorOptions
                    type: getBuiltInObject(node, isBytes ? 'bytes' : 'str'),
                    isIncomplete,
                };
+
+                if (isClass(typeResult.type) && typeResult.type.includePromotions) {
+                    typeResult.type = ClassType.cloneRemoveTypePromotions(typeResult.type);
+                }
            }
        } else {
            typeResult = {
@ -3550,13 +3570,17 @@ export function createTypeEvaluator(importLookup: ImportLookup, evaluatorOptions
    }

    // If the type includes promotion types, expand these to their constituent types.
-    function expandPromotionTypes(node: ParseNode, type: Type): Type {
+    function expandPromotionTypes(node: ParseNode, type: Type, excludeBytes = false): Type {
        return mapSubtypes(type, (subtype) => {
            if (!isClass(subtype) || !subtype.includePromotions) {
                return subtype;
            }

-            const typesToCombine: Type[] = [ClassType.cloneForPromotionType(subtype, /* includePromotions */ false)];
+            if (excludeBytes && ClassType.isBuiltIn(subtype, 'bytes')) {
+                return subtype;
+            }
+
+            const typesToCombine: Type[] = [ClassType.cloneRemoveTypePromotions(subtype)];

            const promotionTypeNames = typePromotions.get(subtype.details.fullName);
            if (promotionTypeNames) {
@ -3565,10 +3589,7 @@ export function createTypeEvaluator(importLookup: ImportLookup, evaluatorOptions
                    let promotionSubtype = getBuiltInType(node, nameSplit[nameSplit.length - 1]);

                    if (promotionSubtype && isInstantiableClass(promotionSubtype)) {
-                        promotionSubtype = ClassType.cloneForPromotionType(
-                            promotionSubtype,
-                            /* includePromotions */ false
-                        );
+                        promotionSubtype = ClassType.cloneRemoveTypePromotions(promotionSubtype);

                        if (isClassInstance(subtype)) {
                            promotionSubtype = ClassType.cloneAsInstance(promotionSubtype);
@ -5169,8 +5190,10 @@ export function createTypeEvaluator(importLookup: ImportLookup, evaluatorOptions
            baseType = makeTopLevelTypeVarsConcrete(baseType);
        }

-        // Do union expansion for promotion types.
-        baseType = expandPromotionTypes(node, baseType);
+        // Do union expansion for promotion types. Exclude bytes here because
+        // this creates too much noise. Users who want stricter checks for bytes
+        // can use "disableBytesTypePromotions".
+        baseType = expandPromotionTypes(node, baseType, /* excludeBytes */ true);

        switch (baseType.category) {
            case TypeCategory.Any:
@ -14496,7 +14519,7 @@ export function createTypeEvaluator(importLookup: ImportLookup, evaluatorOptions
    function cloneBuiltinObjectWithLiteral(node: ParseNode, builtInName: string, value: LiteralValue): Type {
        const type = getBuiltInObject(node, builtInName);
        if (isClassInstance(type)) {
-            return ClassType.cloneWithLiteral(type, value);
+            return ClassType.cloneWithLiteral(ClassType.cloneRemoveTypePromotions(type), value);
        }

        return UnknownType.create();
@ -21745,6 +21768,15 @@ export function createTypeEvaluator(importLookup: ImportLookup, evaluatorOptions
                destType: destErrorTypeText,
            })
        );
+
+        // Tell the user about the disableBytesTypePromotions if that is involved.
+        if (ClassType.isBuiltIn(destType, 'bytes')) {
+            const promotions = typePromotions.get(destType.details.fullName);
+            if (promotions && promotions.some((name) => name === srcType.details.fullName)) {
+                diag?.addMessage(Localizer.DiagnosticAddendum.bytesTypePromotions());
+            }
+        }
+
        return false;
    }

--- a/packages/pyright-internal/src/analyzer/types.ts
+++ b/packages/pyright-internal/src/analyzer/types.ts
@ -827,9 +827,13 @@ export namespace ClassType {
        return newClassType;
    }

-    export function cloneForPromotionType(classType: ClassType, includePromotions: boolean): ClassType {
+    export function cloneRemoveTypePromotions(classType: ClassType): ClassType {
+        if (!classType.includePromotions) {
+            return classType;
+        }
+
        const newClassType = TypeBase.cloneType(classType);
-        newClassType.includePromotions = includePromotions;
+        delete newClassType.includePromotions;
        return newClassType;
    }

--- a/packages/pyright-internal/src/common/configOptions.ts
+++ b/packages/pyright-internal/src/common/configOptions.ts
@ -118,6 +118,9 @@ export interface DiagnosticRuleSet {
    // Enable support for type: ignore comments?
    enableTypeIgnoreComments: boolean;

+    // No longer treat bytearray and memoryview as subclasses of bytes?
+    disableBytesTypePromotions: boolean;
+
    // Treat old typing aliases as deprecated if pythonVersion >= 3.9?
    deprecateTypingAliases: boolean;

@ -345,6 +348,7 @@ export function getBooleanDiagnosticRules(includeNonOverridable = false) {
        DiagnosticRule.strictParameterNoneValue,
        DiagnosticRule.enableExperimentalFeatures,
        DiagnosticRule.deprecateTypingAliases,
+        DiagnosticRule.disableBytesTypePromotions,
    ];

    if (includeNonOverridable) {
@ -449,6 +453,7 @@ export function getOffDiagnosticRuleSet(): DiagnosticRuleSet {
        enableExperimentalFeatures: false,
        enableTypeIgnoreComments: true,
        deprecateTypingAliases: false,
+        disableBytesTypePromotions: false,
        reportGeneralTypeIssues: 'none',
        reportPropertyTypeMismatch: 'none',
        reportFunctionMemberAccess: 'none',
@ -533,6 +538,7 @@ export function getBasicDiagnosticRuleSet(): DiagnosticRuleSet {
        enableExperimentalFeatures: false,
        enableTypeIgnoreComments: true,
        deprecateTypingAliases: false,
+        disableBytesTypePromotions: false,
        reportGeneralTypeIssues: 'error',
        reportPropertyTypeMismatch: 'none',
        reportFunctionMemberAccess: 'none',
@ -617,6 +623,7 @@ export function getStrictDiagnosticRuleSet(): DiagnosticRuleSet {
        enableExperimentalFeatures: false,
        enableTypeIgnoreComments: true, // Not overridden by strict mode
        deprecateTypingAliases: false,
+        disableBytesTypePromotions: true,
        reportGeneralTypeIssues: 'error',
        reportPropertyTypeMismatch: 'none',
        reportFunctionMemberAccess: 'error',
--- a/packages/pyright-internal/src/common/diagnosticRules.ts
+++ b/packages/pyright-internal/src/common/diagnosticRules.ts
@ -19,6 +19,7 @@ export enum DiagnosticRule {
    enableExperimentalFeatures = 'enableExperimentalFeatures',
    enableTypeIgnoreComments = 'enableTypeIgnoreComments',
    deprecateTypingAliases = 'deprecateTypingAliases',
+    disableBytesTypePromotions = 'disableBytesTypePromotions',

    reportGeneralTypeIssues = 'reportGeneralTypeIssues',
    reportPropertyTypeMismatch = 'reportPropertyTypeMismatch',
--- a/packages/pyright-internal/src/localization/localize.ts
+++ b/packages/pyright-internal/src/localization/localize.ts
@ -1145,6 +1145,7 @@ export namespace Localizer {
            new ParameterizedString<{ baseClass: string; type: string }>(
                getRawString('DiagnosticAddendum.baseClassOverridesType')
            );
+        export const bytesTypePromotions = () => getRawString('DiagnosticAddendum.bytesTypePromotions');
        export const conditionalRequiresBool = () =>
            new ParameterizedString<{ operandType: string; boolReturnType: string }>(
                getRawString('DiagnosticAddendum.conditionalRequiresBool')
--- a/packages/pyright-internal/src/localization/package.nls.en-us.json
+++ b/packages/pyright-internal/src/localization/package.nls.en-us.json
@ -585,6 +585,7 @@
        "baseClassIncompatibleSubclass": "Base class \"{baseClass}\" derives from \"{subclass}\" which is incompatible with type \"{type}\"",
        "baseClassOverriddenType": "Base class \"{baseClass}\" provides type \"{type}\", which is overridden",
        "baseClassOverridesType": "Base class \"{baseClass}\" overrides with type \"{type}\"",
+        "bytesTypePromotions": "Set disableBytesTypePromotions to false to enable type promotion behavior for \"bytearray\" and \"memoryview\"",
        "conditionalRequiresBool": "Method __bool__ for type \"{operandType}\" returns type \"{boolReturnType}\" rather than \"bool\"",
        "dataClassFieldLocation": "Field declaration",
        "dataClassFrozen": "\"{name}\" is frozen",
--- a/packages/pyright-internal/src/tests/samples/classes8.py
+++ b/packages/pyright-internal/src/tests/samples/classes8.py
@ -42,6 +42,6 @@ def thing(value: AnyStr):
            if isinstance(file.name, str):
                if file.name.endswith(".xml"):
                    ...
-            else:
+            elif isinstance(file.name, bytes):
                if file.name.endswith(b".xml"):
                    ...
--- a/packages/pyright-internal/src/tests/samples/typePromotions1.py
+++ b/packages/pyright-internal/src/tests/samples/typePromotions1.py
@ -10,6 +10,11 @@ def func1(float_val: float, int_val: int):
    v3: complex = int_val


+def func2(mem_view_val: memoryview, byte_array_val: bytearray):
+    v1: bytes = mem_view_val
+    v2: bytes = byte_array_val
+
+
 class IntSubclass(int):
    ...

--- a/packages/pyright-internal/src/tests/typeEvaluator2.test.ts
+++ b/packages/pyright-internal/src/tests/typeEvaluator2.test.ts
@ -473,7 +473,10 @@ test('ConstrainedTypeVar14', () => {
 });

 test('ConstrainedTypeVar15', () => {
-    const analysisResults = TestUtils.typeAnalyzeSampleFiles(['constrainedTypeVar15.py']);
+    const configOptions = new ConfigOptions('.');
+    configOptions.diagnosticRuleSet.disableBytesTypePromotions = true;
+
+    const analysisResults = TestUtils.typeAnalyzeSampleFiles(['constrainedTypeVar15.py'], configOptions);

    TestUtils.validateResults(analysisResults, 0);
 });
--- a/packages/pyright-internal/src/tests/typeEvaluator4.test.ts
+++ b/packages/pyright-internal/src/tests/typeEvaluator4.test.ts
@ -784,6 +784,7 @@ test('Generic3', () => {

 test('Unions1', () => {
    const configOptions = new ConfigOptions('.');
+    configOptions.diagnosticRuleSet.disableBytesTypePromotions = true;

    // Analyze with Python 3.9 settings. This will generate errors.
    configOptions.defaultPythonVersion = PythonVersion.V3_9;
--- a/packages/vscode-pyright/schemas/pyrightconfig.schema.json
+++ b/packages/vscode-pyright/schemas/pyrightconfig.schema.json
@ -112,6 +112,12 @@
      ],
      "pattern": "^(.*)$"
    },
+    "disableBytesTypePromotions": {
+      "$id": "#/properties/disableBytesTypePromotions",
+      "type": "boolean",
+      "title": "Do not treat `bytearray` and `memoryview` as implicit subtypes of `bytes`",
+      "default": false
+    },
    "strictListInference": {
      "$id": "#/properties/strictListInference",
      "type": "boolean",