Fixed a bug that led to extremely long type analysis times when determining type compatibility between an recursive type alias and a recursive protocol. This addresses https://github.com/microsoft/pyright/issues/4353.

packages/pyright-internal/src/analyzer/typeEvaluator.ts

@@ -21420,14 +21420,12 @@ export function createTypeEvaluator(importLookup: ImportLookup, evaluatorOptions
         const transformedDestType = transformPossibleRecursiveTypeAlias(destType);
         const transformedSrcType = transformPossibleRecursiveTypeAlias(srcType);
 
-        // Did both the source and dest include recursive type aliases?
-        // If so, we are potentially dealing with different recursive type
-        // aliases that are defined in the same way.
+        // Did either the source or dest include recursive type aliases?
+        // If so, we could be dealing with different recursive type aliases
+        // or a recursive type alias and a recursive protocol definition.
         if (
-            transformedDestType !== destType &&
-            transformedSrcType !== srcType &&
-            isUnion(transformedDestType) &&
-            isUnion(transformedSrcType)
+            (transformedDestType !== destType && isUnion(transformedDestType)) ||
+            (transformedSrcType !== srcType && isUnion(transformedSrcType))
         ) {
             // Use a smaller recursive limit in this case to prevent runaway recursion.
             if (recursionCount > maxRecursiveTypeAliasRecursionCount) {

packages/pyright-internal/src/tests/samples/recursiveTypeAlias12.py

@@ -0,0 +1,24 @@
+# This sample tests the case where a recursive type alias is evaluated
+# for type compatibility with a recursive protocol. We want to make sure
+# this doesn't lead to extremely long evaluation times or stack overflows.
+
+from collections.abc import Callable
+from types import FrameType
+from typing import Any, Protocol, Self, TypeAlias
+
+
+class TraceFunctionProto(Protocol):
+    def __call__(self, frame: FrameType, event: str, arg: Any) -> Self | None:
+        ...
+
+TraceFunction: TypeAlias = Callable[[FrameType, str, Any], "TraceFunction | None"]
+
+def settrace(tf: TraceFunction | None) -> None: ...
+
+def func1(frame: FrameType, event: str, arg: Any) -> TraceFunction:
+    ...
+
+def func2(tf: TraceFunctionProto | None):
+    settrace(tf)
+    settrace(func1)
+

packages/pyright-internal/src/tests/typeEvaluator3.test.ts

@@ -637,6 +637,12 @@ test('RecursiveTypeAlias11', () => {
     TestUtils.validateResults(analysisResults, 0);
 });
 
+test('RecursiveTypeAlias12', () => {
+    const analysisResults = TestUtils.typeAnalyzeSampleFiles(['recursiveTypeAlias12.py']);
+
+    TestUtils.validateResults(analysisResults, 0);
+});
+
 test('Classes1', () => {
     const analysisResults = TestUtils.typeAnalyzeSampleFiles(['classes1.py']);