Fixed a bug that results in an incorrect type evaluation of a higher-order function that is passed an overloaded function. This addresses #8898. (#8900)

packages/pyright-internal/src/analyzer/typeUtils.ts

@@ -4197,6 +4197,12 @@ class ApplySolvedTypeVarsTransformer extends TypeVarTransformer {
             return callback();
         }
 
+        // Handle the case where we're already processing one of the signature contexts
+        // and are called recursively. Don't loop over all the signature contexts again.
+        if (this._activeConstraintSetIndex !== undefined) {
+            return callback();
+        }
+
         // Loop through all of the signature contexts in the type var context
         // to create an overload type.
         const overloadTypes = solutionSets.map((_, index) => {

packages/pyright-internal/src/tests/samples/solverHigherOrder14.py

@@ -0,0 +1,34 @@
+# This sample tests the case where a function with an overload
+# is passed to a higher-order function and the return type uses
+# a nested Callable type.
+
+from typing import overload, Callable
+
+
+def func1[A, B](f: Callable[[A], B]) -> Callable[[Callable[[], A]], B]: ...
+
+
+@overload
+def func2(v: int) -> None: ...
+
+
+@overload
+def func2(v: str) -> None: ...
+
+
+def func2(v: int | str) -> None:
+    pass
+
+
+def func3() -> int:
+    return 1
+
+
+v1 = func1(func2)
+reveal_type(v1, expected_text="Overload[(() -> int) -> None, (() -> str) -> None]")
+
+v2 = v1(func3)
+reveal_type(v2, expected_text="None")
+
+v3 = v1(lambda: 1)
+reveal_type(v3, expected_text="None")

packages/pyright-internal/src/tests/typeEvaluator2.test.ts

@@ -915,6 +915,12 @@ test('SolverHigherOrder13', () => {
     TestUtils.validateResults(analysisResults, 0);
 });
 
+test('SolverHigherOrder14', () => {
+    const analysisResults = TestUtils.typeAnalyzeSampleFiles(['solverHigherOrder14.py']);
+
+    TestUtils.validateResults(analysisResults, 0);
+});
+
 test('SolverLiteral1', () => {
     const analysisResults = TestUtils.typeAnalyzeSampleFiles(['solverLiteral1.py']);