Support record up/downgrades in Java codegen (#17486)

* Support record up/downgrades in Java codegen

This adds enough upgrading support to the Java codegen to use it
against the current upgrading PoCs. This is backwards compatible so I
enabled it in all cases instead of trying to add a flag somewhere.

language-support/java/bindings/BUILD.bazel

@@ -79,6 +79,7 @@ da_java_library(
         "@maven//:io_grpc_grpc_stub",
         "@maven//:javax_annotation_javax_annotation_api",
         "@maven//:org_checkerframework_checker_qual",
+        "@maven//:org_slf4j_slf4j_api",
     ],
 )
 

language-support/java/bindings/src/main/java/com/daml/ledger/javaapi/data/codegen/PrimitiveValueDecoders.java

@@ -6,9 +6,12 @@ package com.daml.ledger.javaapi.data.codegen;
 import com.daml.ledger.javaapi.data.*;
 import java.time.Instant;
 import java.time.LocalDate;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * {@link ValueDecoder}s for Daml types that are not code-generated.
@@ -16,6 +19,9 @@ import java.util.Optional;
  * @see ValueDecoder
  */
 public final class PrimitiveValueDecoders {
+
+  private static final Logger logger = LoggerFactory.getLogger(PrimitiveValueDecoders.class);
+
   // constructing not allowed
   private PrimitiveValueDecoders() {}
 
@@ -116,7 +122,7 @@ public final class PrimitiveValueDecoders {
    * @hidden
    */
   public static List<com.daml.ledger.javaapi.data.DamlRecord.Field> recordCheck(
-      int expectedFields, Value maybeRecord) {
+      int expectedFields, int trailingOptionalFields, Value maybeRecord) {
     var record =
         maybeRecord
             .asRecord()
@@ -124,9 +130,65 @@ public final class PrimitiveValueDecoders {
                 () -> new IllegalArgumentException("Contracts must be constructed from Records"));
     var fields = record.getFields();
     var numberOfFields = fields.size();
-    if (numberOfFields != expectedFields)
-      throw new IllegalArgumentException(
-          "Expected " + expectedFields + " arguments, got " + numberOfFields);
+    if (numberOfFields == expectedFields) {
+      return fields;
+    }
+    if (numberOfFields > expectedFields) {
+      // Downgrade, check that the additional fields are empty optionals.
+      for (var i = expectedFields; i < numberOfFields; i++) {
+        final var field = fields.get(i);
+        final var optValue = field.getValue().asOptional();
+        if (optValue.isEmpty()) {
+          throw new IllegalArgumentException(
+              "Expected "
+                  + expectedFields
+                  + " arguments, got "
+                  + numberOfFields
+                  + " and field "
+                  + i
+                  + " is not optional: "
+                  + field);
+        }
+        final var value = optValue.get();
+        if (!value.isEmpty()) {
+          throw new IllegalArgumentException(
+              "Expected "
+                  + expectedFields
+                  + " arguments, got "
+                  + numberOfFields
+                  + " and field "
+                  + i
+                  + " is Optional but not empty: "
+                  + field);
+        }
+      }
+      logger.trace(
+          "Downgrading record, dropping {} trailing optional fields",
+          numberOfFields - expectedFields);
+      return fields.subList(0, expectedFields);
+    }
+    if (numberOfFields < expectedFields) {
+      // Upgrade, add empty optionals to the end.
+      if ((expectedFields - numberOfFields) <= trailingOptionalFields) {
+        final var newFields = new ArrayList<>(fields);
+        for (var i = 0; i < expectedFields - numberOfFields; i++) {
+          newFields.add(new com.daml.ledger.javaapi.data.DamlRecord.Field(DamlOptional.EMPTY));
+        }
+        logger.trace(
+            "Upgrading record, appending {} empty optional fields",
+            expectedFields - numberOfFields);
+        return newFields;
+      } else {
+        throw new IllegalArgumentException(
+            "Expected "
+                + expectedFields
+                + " arguments, got "
+                + numberOfFields
+                + " and only the last "
+                + trailingOptionalFields
+                + " of the expected type are optionals");
+      }
+    }
     return fields;
   }
 

language-support/java/codegen/src/it/daml/Lib_1_11.daml

@@ -18,3 +18,4 @@ import Tests.TextMapTest()
 import Tests.GenMapTest()
 import Tests.ContractKeys()
 import Tests.ParametersOrder()
+import Tests.UpgradeTest()

language-support/java/codegen/src/it/daml/Lib_1_12.daml

@@ -18,3 +18,4 @@ import Tests.TextMapTest()
 import Tests.GenMapTest()
 import Tests.ContractKeys()
 import Tests.ParametersOrder()
+import Tests.UpgradeTest()

language-support/java/codegen/src/it/daml/Lib_1_6.daml

@@ -16,3 +16,4 @@ import Tests.Escape()
 import Tests.TextMapTest()
 import Tests.ContractKeys()
 import Tests.ParametersOrder()
+import Tests.UpgradeTest()

language-support/java/codegen/src/it/daml/Lib_1_7.daml

@@ -17,3 +17,4 @@ import Tests.Escape()
 import Tests.TextMapTest()
 import Tests.ContractKeys()
 import Tests.ParametersOrder()
+import Tests.UpgradeTest()

language-support/java/codegen/src/it/daml/Lib_1_8.daml

@@ -17,3 +17,4 @@ import Tests.Escape()
 import Tests.TextMapTest()
 import Tests.ContractKeys()
 import Tests.ParametersOrder()
+import Tests.UpgradeTest()

language-support/java/codegen/src/it/daml/Lib_1_dev.daml

@@ -18,3 +18,4 @@ import Tests.TextMapTest()
 import Tests.GenMapTest()
 import Tests.ContractKeys()
 import Tests.ParametersOrder()
+import Tests.UpgradeTest()

language-support/java/codegen/src/it/daml/Tests/UpgradeTest.daml

@@ -0,0 +1,29 @@
+-- Copyright (c) 2023 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+-- SPDX-License-Identifier: Apache-2.0
+
+
+module Tests.UpgradeTest where
+
+data NoOptional = NoOptional
+  with
+    a : Text
+    b : Text
+  deriving (Show, Eq)
+
+data OptionalAtEnd = OptionalAtEnd
+  with
+    a : Text
+    b : Text
+    c : Optional Text
+    d : Optional Text
+  deriving (Show, Eq)
+
+-- Not part of the test but the codegen filters out
+-- data definitions which are not used in a template
+template UpgradeTestTemplate
+    with
+        p : Party
+        noOptional : NoOptional
+        optionalAtEnd : OptionalAtEnd
+    where
+        signatory p

language-support/java/codegen/src/it/java/com/daml/AllGenericTests.java

@@ -21,5 +21,6 @@ import org.junit.runners.Suite;
   TemplateMethodTest.class,
   TextMapTest.class,
   VariantTest.class,
+  UpgradeTest.class,
 })
 public class AllGenericTests {}

language-support/java/codegen/src/it/java/com/daml/UpgradeTest.java

@@ -0,0 +1,92 @@
+// Copyright (c) 2023 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package com.daml;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import com.daml.ledger.javaapi.data.*;
+import java.util.Optional;
+import org.junit.jupiter.api.Test;
+import org.junit.platform.runner.JUnitPlatform;
+import org.junit.runner.RunWith;
+import tests.upgradetest.*;
+
+@RunWith(JUnitPlatform.class)
+public class UpgradeTest {
+
+  @Test
+  void exactMatch() {
+    DamlRecord record =
+        new DamlRecord(
+            new DamlRecord.Field(new Text("abc")), new DamlRecord.Field(new Text("def")));
+    NoOptional actual = NoOptional.fromValue(record);
+
+    NoOptional expected = new NoOptional("abc", "def");
+
+    assertEquals(actual, expected);
+  }
+
+  @Test
+  void downgradeEmptyOptional() {
+    DamlRecord record =
+        new DamlRecord(
+            new DamlRecord.Field(new Text("abc")),
+            new DamlRecord.Field(new Text("def")),
+            new DamlRecord.Field(DamlOptional.EMPTY),
+            new DamlRecord.Field(DamlOptional.EMPTY));
+    NoOptional actual = NoOptional.fromValue(record);
+
+    NoOptional expected = new NoOptional("abc", "def");
+
+    assertEquals(actual, expected);
+  }
+
+  @Test
+  void downgradeNonEmptyOptionalFails() {
+    DamlRecord record =
+        new DamlRecord(
+            new DamlRecord.Field(new Text("abc")),
+            new DamlRecord.Field(new Text("def")),
+            new DamlRecord.Field(DamlOptional.of(Unit.getInstance())));
+    assertThrows(IllegalArgumentException.class, () -> NoOptional.fromValue(record));
+  }
+
+  @Test
+  void downgradeNonOptionalFails() {
+    DamlRecord record =
+        new DamlRecord(
+            new DamlRecord.Field(new Text("abc")),
+            new DamlRecord.Field(new Text("def")),
+            new DamlRecord.Field(Unit.getInstance()));
+    assertThrows(IllegalArgumentException.class, () -> NoOptional.fromValue(record));
+  }
+
+  @Test
+  void upgradeOptionalFieldsTwoMissingOptionals() {
+    DamlRecord record =
+        new DamlRecord(
+            new DamlRecord.Field(new Text("abc")), new DamlRecord.Field(new Text("def")));
+    OptionalAtEnd actual = OptionalAtEnd.fromValue(record);
+    OptionalAtEnd expected = new OptionalAtEnd("abc", "def", Optional.empty(), Optional.empty());
+    assertEquals(actual, expected);
+  }
+
+  @Test
+  void upgradeOptionalFieldsOneMissingOptional() {
+    DamlRecord record =
+        new DamlRecord(
+            new DamlRecord.Field(new Text("abc")),
+            new DamlRecord.Field(new Text("def")),
+            new DamlRecord.Field(DamlOptional.of(new Text("ghi"))));
+    OptionalAtEnd actual = OptionalAtEnd.fromValue(record);
+    OptionalAtEnd expected = new OptionalAtEnd("abc", "def", Optional.of("ghi"), Optional.empty());
+    assertEquals(actual, expected);
+  }
+
+  @Test
+  void upgradeNonOptionalFields() {
+    DamlRecord record = new DamlRecord(new DamlRecord.Field(new Text("abc")));
+    assertThrows(IllegalArgumentException.class, () -> NoOptional.fromValue(record));
+  }
+}

language-support/java/codegen/src/main/scala/com/digitalasset/daml/lf/codegen/backend/java/inner/FromValueGenerator.scala

@@ -87,15 +87,24 @@ private[inner] object FromValueGenerator extends StrictLogging {
       .generate(typeParameters)
       .valueDecoderParameterSpecs
 
+    def isOptional(t: Type) =
+      t match {
+        case TypePrim(PrimTypeOptional, _) => true
+        case _ => false
+      }
+
+    val optionalFieldsSize = fields.reverse.takeWhile(f => isOptional(f.damlType)).size
+
     val fromValueCode = CodeBlock
       .builder()
       .add(recordValueExtractor("value$", "recordValue$"))
       .addStatement(
-        "$T fields$$ = $T.recordCheck($L,$WrecordValue$$)",
+        "$T fields$$ = $T.recordCheck($L,$L,$WrecordValue$$)",
         ParameterizedTypeName
           .get(classOf[java.util.List[_]], classOf[javaapi.data.DamlRecord.Field]),
         classOf[PrimitiveValueDecoders],
         fields.size,
+        optionalFieldsSize,
       )
 
     fields.iterator.zip(accessors).foreach { case (FieldInfo(_, damlType, javaName, _), accessor) =>