Do not permit inferred-open-in-output-position extension variables to grow

Closes #4852
Closes #4845

crates/compiler/unify/src/unify.rs

@@ -2405,14 +2405,22 @@ fn close_uninhabited_extended_union(subs: &mut Subs, mut var: Variable) {
     }
 }
 
+enum UnifySides<T, U> {
+    Left(T, U),
+    Right(U, T),
+}
+
 #[must_use]
 fn unify_tag_ext<M: MetaCollector>(
     env: &mut Env,
     pool: &mut Pool,
-    ext: TagExt,
-    var: Variable,
+    vars: UnifySides<TagExt, Variable>,
     mode: Mode,
 ) -> Outcome<M> {
+    let (ext, var, flip_for_unify) = match vars {
+        UnifySides::Left(ext, var) => (ext, var, false),
+        UnifySides::Right(var, ext) => (ext, var, true),
+    };
     let legal_unification = match ext {
         TagExt::Openness(_) => {
             // Openness extensions can either unify with empty tag unions (marking them as closed),
@@ -2434,12 +2442,24 @@ fn unify_tag_ext<M: MetaCollector>(
         TagExt::Any(_) => true,
     };
     if legal_unification {
-        unify_pool(env, pool, ext.var(), var, mode)
+        if flip_for_unify {
+            unify_pool(env, pool, var, ext.var(), mode)
+        } else {
+            unify_pool(env, pool, ext.var(), var, mode)
+        }
     } else {
         mismatch!()
     }
 }
 
+#[must_use]
+fn merge_tag_exts(ext1: TagExt, ext2: TagExt) -> TagExt {
+    match (ext1, ext2) {
+        (_, TagExt::Openness(v)) | (TagExt::Openness(v), _) => TagExt::Openness(v),
+        (TagExt::Any(v), TagExt::Any(_)) => TagExt::Any(v),
+    }
+}
+
 #[allow(clippy::too_many_arguments)]
 #[must_use]
 fn unify_tag_unions<M: MetaCollector>(
@@ -2509,7 +2529,7 @@ fn unify_tag_unions<M: MetaCollector>(
                 ctx,
                 shared_tags,
                 OtherTags2::Empty,
-                ext1,
+                merge_tag_exts(ext1, ext2),
                 recursion_var,
             );
 
@@ -2538,7 +2558,8 @@ fn unify_tag_unions<M: MetaCollector>(
                 ext1 = new_ext;
             }
 
-            let ext_outcome = unify_tag_ext(env, pool, ext1, extra_tags_in_2, ctx.mode);
+            let ext_outcome =
+                unify_tag_ext(env, pool, UnifySides::Left(ext1, extra_tags_in_2), ctx.mode);
 
             if !ext_outcome.mismatches.is_empty() {
                 return ext_outcome;
@@ -2590,7 +2611,12 @@ fn unify_tag_unions<M: MetaCollector>(
                 ext2 = new_ext;
             }
 
-            let ext_outcome = unify_tag_ext(env, pool, ext2, extra_tags_in_1, ctx.mode);
+            let ext_outcome = unify_tag_ext(
+                env,
+                pool,
+                UnifySides::Right(extra_tags_in_1, ext2),
+                ctx.mode,
+            );
 
             if !ext_outcome.mismatches.is_empty() {
                 return ext_outcome;
@@ -2659,7 +2685,7 @@ fn unify_tag_unions<M: MetaCollector>(
         let mut total_outcome = Outcome::default();
         let snapshot = env.subs.snapshot();
 
-        let ext1_outcome = unify_tag_ext(env, pool, ext1, sub2, ctx.mode);
+        let ext1_outcome = unify_tag_ext(env, pool, UnifySides::Left(ext1, sub2), ctx.mode);
         if !ext1_outcome.mismatches.is_empty() {
             env.subs.rollback_to(snapshot);
             return ext1_outcome;
@@ -2667,7 +2693,7 @@ fn unify_tag_unions<M: MetaCollector>(
         total_outcome.union(ext1_outcome);
 
         if ctx.mode.is_eq() {
-            let ext2_outcome = unify_tag_ext(env, pool, ext2, sub1, ctx.mode);
+            let ext2_outcome = unify_tag_ext(env, pool, UnifySides::Right(sub1, ext2), ctx.mode);
             if !ext2_outcome.mismatches.is_empty() {
                 env.subs.rollback_to(snapshot);
                 return ext2_outcome;

crates/reporting/tests/test_reporting.rs

@@ -12848,8 +12848,7 @@ I recommend using camelCase. It's the standard style in Roc code!
             str = List.concat ["blah"] empty
 
             {one, str}
-        "#
-        ),
+        "#),
     @r###"
     ── TYPE MISMATCH ───────────────────────────────────────── /code/proj/Main.roc ─
 
@@ -12867,4 +12866,166 @@ I recommend using camelCase. It's the standard style in Roc code!
         List Str
     "###
     );
+
+    test_report!(
+        implicit_inferred_open_in_output_position_cannot_grow,
+        indoc!(
+            r#"
+            app "test" provides [main] to "./platform"
+
+            main : {} -> [One]
+            main = \{} ->
+                if Bool.true
+                then One
+                else Two
+            "#
+        ),
+    @r###"
+    ── TYPE MISMATCH ───────────────────────────────────────── /code/proj/Main.roc ─
+
+    Something is off with the `else` branch of this `if` expression:
+
+    3│  main : {} -> [One]
+    4│  main = \{} ->
+    5│      if Bool.true
+    6│      then One
+    7│      else Two
+                 ^^^
+
+    This `Two` tag has the type:
+
+        [Two]
+
+    But the type annotation on `main` says it should be:
+
+        [One]
+    "###
+    );
+
+    test_report!(
+        implicit_inferred_open_in_output_position_cannot_grow_alias,
+        indoc!(
+            r#"
+            app "test" provides [main] to "./platform"
+
+            R : [One]
+
+            main : {} -> R
+            main = \{} ->
+                if Bool.true
+                then One
+                else Two
+            "#
+        ),
+    @r###"
+    ── TYPE MISMATCH ───────────────────────────────────────── /code/proj/Main.roc ─
+
+    Something is off with the `else` branch of this `if` expression:
+
+    5│  main : {} -> R
+    6│  main = \{} ->
+    7│      if Bool.true
+    8│      then One
+    9│      else Two
+                 ^^^
+
+    This `Two` tag has the type:
+
+        [Two]
+
+    But the type annotation on `main` says it should be:
+
+        [One]
+    "###
+    );
+
+    test_report!(
+        implicit_inferred_open_in_output_position_cannot_grow_nested,
+        indoc!(
+            r#"
+            app "test" provides [main] to "./platform"
+
+            main : List [One, Two] -> List [One]
+            main = \tags ->
+                List.map tags \tag ->
+                    when tag is
+                        One -> One
+                        Two -> Two
+            "#
+        ),
+    @r###"
+    ── TYPE MISMATCH ───────────────────────────────────────── /code/proj/Main.roc ─
+
+    Something is off with the body of the `main` definition:
+
+    3│   main : List [One, Two] -> List [One]
+    4│   main = \tags ->
+    5│>      List.map tags \tag ->
+    6│>          when tag is
+    7│>              One -> One
+    8│>              Two -> Two
+
+    This `map` call produces:
+
+        List [One, Two]
+
+    But the type annotation on `main` says it should be:
+
+        List [One]
+    "###
+    );
+
+    test_report!(
+        implicit_inferred_open_in_output_position_cannot_grow_nested_alias,
+        indoc!(
+            r#"
+            app "test" provides [main] to "./platform"
+
+            R : [One]
+
+            main : List [One, Two] -> List R
+            main = \tags ->
+                List.map tags \tag ->
+                    when tag is
+                        One -> One
+                        Two -> Two
+            "#
+        ),
+    @r###"
+    ── TYPE MISMATCH ───────────────────────────────────────── /code/proj/Main.roc ─
+
+    Something is off with the body of the `main` definition:
+
+     5│   main : List [One, Two] -> List R
+     6│   main = \tags ->
+     7│>      List.map tags \tag ->
+     8│>          when tag is
+     9│>              One -> One
+    10│>              Two -> Two
+
+    This `map` call produces:
+
+        List [One, Two]
+
+    But the type annotation on `main` says it should be:
+
+        List [One]
+    "###
+    );
+
+    test_no_problem!(
+        explicit_inferred_open_in_output_position_can_grow,
+        indoc!(
+            r#"
+            app "test" provides [main] to "./platform"
+
+            main : List [One, Two] -> List [One]_
+            main = \tags ->
+                List.map tags \tag ->
+                    when tag is
+                        One -> One
+                        Two -> Two
+            "#
+        )
+    );
 }