json-api: in-memory comparison query (#3405)

* comparison query parser given scalar parser

- written in half-error-propagation style to better suit other potential
  error features

* factor dupes in RangeExpr

* text range parsing

* interpreting Range into in-memory predicate; points out bad dedupe from earlier

* make reuse of the scalar extractors much nicer

* express date, time, int64 as factored-out range exprs

* express numeric as factored-out range expr

* factor \&/ usage

* refactor LF value extractors for reuse in range queries

* factor mkRange usage

* totally deconstruct the int64 and text cases

* totally deconstruct the date and timestamp cases

* totally deconstruct the numeric case

* document comparison queries

* use Utf8.Ordering for text comparison queries

* int64 range query tests

* more int64 range query tests

* date, string, numeric range query tests

* include line # in query test successes table

* timestamp range query tests

* add release note

* remove duplicate changelog entry from #3425

daml-lf/data/src/main/scala/com/digitalasset/daml/lf/data/Time.scala

@@ -3,6 +3,10 @@
 
 package com.digitalasset.daml.lf.data
 
+import scalaz.Order
+import scalaz.std.anyVal._
+import scalaz.syntax.order._
+
 import java.time.format.DateTimeFormatter
 import java.time.temporal.ChronoField
 import java.time.{Instant, LocalDate, ZoneId}
@@ -68,6 +72,12 @@ object Time {
     def assertFromDaysSinceEpoch(days: Int): Date =
       assertRight(fromDaysSinceEpoch(days))
 
+    implicit val `Time.Date Order`: Order[Date] = new Order[Date] {
+      override def equalIsNatural = true
+      override def equal(x: Date, y: Date) = x == y
+      override def order(x: Date, y: Date) = x.days ?|? y.days
+    }
+
   }
 
   case class Timestamp private (micros: Long) extends Ordered[Timestamp] {
@@ -144,6 +154,11 @@ object Time {
     def assertFromInstant(i: Instant): Timestamp =
       assertFromLong(assertMicrosFromInstant(i))
 
+    implicit val `Time.Timestamp Order`: Order[Timestamp] = new Order[Timestamp] {
+      override def equalIsNatural = true
+      override def equal(x: Timestamp, y: Timestamp) = x == y
+      override def order(x: Timestamp, y: Timestamp) = x.micros ?|? y.micros
+    }
   }
 
 }

docs/source/json-api/search-query-language.rst

@@ -42,8 +42,32 @@ Example: ``{ favorites: ["vanilla", "chocolate"] }``
 
 A JSON object, when considered with a record type, is always interpreted
 as a field equality query. Its type context is thus mutually exclusive
-with `the forthcoming comparison queries
-<https://github.com/digital-asset/daml/issues/2780>`_.
+with comparison queries.
+
+Comparison query
+****************
+
+Match values on comparison operators for int64, numeric, text, date, and
+time values. Instead of a value, a key can be an object with one or more
+operators: ``{ <op>: value }`` where ``<op>`` can be:
+
+- ``"%lt"`` for less than
+- ``"%gt"`` for greater than
+- ``"%lte"`` for less than or equal to
+- ``"%gte"`` for greater than or equal to
+
+``"%lt"`` and ``"%lte"`` may not be used at the same time, and likewise
+with ``"%lt"`` and ``"%lte"``, but all other combinations are allowed.
+
+Example:  ``{ "person" { "dob": { "%lt": "2000-01-01", "%gte": "1980-01-01" } } }``
+
+- Match: ``{ person: { dob: "1986-06-21" } }``
+- No match: ``{ person: { dob: "1976-06-21" } }``
+- No match: ``{ person: { dob: "2006-06-21" } }``
+
+These operators cannot occur in objects interpreted in a record context,
+nor may other keys than these four operators occur where they are legal,
+so there is no ambiguity with field equality.
 
 Appendix: Type-aware queries
 ****************************

ledger-service/http-json/src/main/scala/com/digitalasset/http/query/ValuePredicate.scala

@@ -6,13 +6,19 @@ package query
 
 import util.IdentifierConverters.lfIdentifier
 
-import com.digitalasset.daml.lf.data.{ImmArray, Numeric, Ref, SortedLookupList, Time}
+import com.digitalasset.daml.lf.data.{ImmArray, Numeric, Ref, SortedLookupList, Time, Utf8}
 import ImmArray.ImmArraySeq
 import com.digitalasset.daml.lf.data.ScalazEqual._
 import com.digitalasset.daml.lf.iface
 import com.digitalasset.daml.lf.value.{Value => V}
 import iface.{Type => Ty}
-import scalaz.\&/
+
+import scalaz.{Order, \&/, \/, \/-}
+import scalaz.Tags.Conjunction
+import scalaz.std.anyVal._
+import scalaz.syntax.apply._
+import scalaz.syntax.order._
+import scalaz.syntax.tag._
 import scalaz.syntax.std.option._
 import scalaz.syntax.std.string._
 import spray.json._
@@ -67,7 +73,13 @@ sealed abstract class ValuePredicate extends Product with Serializable {
         oq map go cata (csq => { case V.ValueOptional(Some(v)) => csq(v); case _ => false },
         { case V.ValueOptional(None) => true; case _ => false })
 
-      case Range(_, _) => predicateParseError("range not supported yet")
+      case range: Range[a] =>
+        implicit val ord: Order[a] = range.ord
+        range.project andThen { a =>
+          range.ltgt.bifoldMap {
+            case (incl, ceil) => Conjunction(if (incl) a <= ceil else a < ceil)
+          } { case (incl, floor) => Conjunction(if (incl) a >= floor else a > floor) }.unwrap
+        } orElse { case _ => false }
     }
     go(this)
   }
@@ -84,8 +96,8 @@ object ValuePredicate {
   final case class ListMatch(elems: Vector[ValuePredicate]) extends ValuePredicate
   final case class VariantMatch(elem: (Ref.Name, ValuePredicate)) extends ValuePredicate
   final case class OptionalMatch(elem: Option[ValuePredicate]) extends ValuePredicate
-  // boolean is whether inclusive (lte vs lt)
-  final case class Range(ltgt: (Boolean, LfV) \&/ (Boolean, LfV), typ: Ty) extends ValuePredicate
+  final case class Range[A](ltgt: Boundaries[A], ord: Order[A], project: LfV PartialFunction A)
+      extends ValuePredicate
 
   private[http] def fromTemplateJsObject(
       it: Map[String, JsValue],
@@ -104,14 +116,8 @@ object ValuePredicate {
             val ddt = defs(id).getOrElse(predicateParseError(s"Type $id not found"))
             fromCon(it, id, tc instantiate ddt)
         }
-        case iface.TypeNumeric(scale) => {
-          case JsString(q) =>
-            val nq = Numeric checkWithinBoundsAndRound (scale, BigDecimal(q)) fold (predicateParseError, identity)
-            Literal { case V.ValueNumeric(v) if nq == (v setScale scale) => }
-          case JsNumber(q) =>
-            val nq = Numeric checkWithinBoundsAndRound (scale, q) fold (predicateParseError, identity)
-            Literal { case V.ValueNumeric(v) if nq == (v setScale scale) => }
-        }
+        case iface.TypeNumeric(scale) =>
+          numericRangeExpr(scale).toQueryParser
         case iface.TypeVar(_) => predicateParseError("no vars allowed!")
       }(fallback = illTypedQuery(it, typ))
 
@@ -188,25 +194,10 @@ object ValuePredicate {
       }
       (typ.typ, it).match2 {
         case Bool => { case JsBoolean(q) => Literal { case V.ValueBool(v) if q == v => } }
-        case Int64 => {
-          case JsNumber(q) if q.isValidLong =>
-            val lq = q.toLongExact
-            Literal { case V.ValueInt64(v) if lq == (v: Long) => }
-          case JsString(q) =>
-            val lq: Long = q.parseLong.fold(e => throw e, identity)
-            Literal { case V.ValueInt64(v) if lq == (v: Long) => }
-        }
-        case Text => { case JsString(q) => Literal { case V.ValueText(v) if q == v => } }
-        case Date => {
-          case JsString(q) =>
-            val dq = Time.Date fromString q fold (predicateParseError(_), identity)
-            Literal { case V.ValueDate(v) if dq == v => }
-        }
-        case Timestamp => {
-          case JsString(q) =>
-            val tq = Time.Timestamp fromString q fold (predicateParseError(_), identity)
-            Literal { case V.ValueTimestamp(v) if tq == v => }
-        }
+        case Int64 => Int64RangeExpr.toQueryParser
+        case Text => TextRangeExpr.toQueryParser(Order fromScalaOrdering Utf8.Ordering)
+        case Date => DateRangeExpr.toQueryParser
+        case Timestamp => TimestampRangeExpr.toQueryParser
         case Party => {
           case JsString(q) => Literal { case V.ValueParty(v) if q == (v: String) => }
         }
@@ -242,9 +233,111 @@ object ValuePredicate {
     }) getOrElse predicateParseError(s"No record type found for $typ")
   }
 
-  private[this] def illTypedQuery(it: JsValue, typ: Any): Nothing =
+  private[this] val Int64RangeExpr = RangeExpr({
+    case JsNumber(q) if q.isValidLong =>
+      q.toLongExact
+    case JsString(q) =>
+      q.parseLong.fold(e => throw e, identity)
+  }, { case V.ValueInt64(v) => v })
+  private[this] val TextRangeExpr = RangeExpr({ case JsString(s) => s }, {
+    case V.ValueText(v) => v
+  })
+  private[this] val DateRangeExpr = RangeExpr({
+    case JsString(q) =>
+      Time.Date fromString q fold (predicateParseError(_), identity)
+  }, { case V.ValueDate(v) => v })
+  private[this] val TimestampRangeExpr = RangeExpr({
+    case JsString(q) =>
+      Time.Timestamp fromString q fold (predicateParseError(_), identity)
+  }, { case V.ValueTimestamp(v) => v })
+  private[this] def numericRangeExpr(scale: Numeric.Scale) =
+    RangeExpr(
+      {
+        case JsString(q) =>
+          Numeric checkWithinBoundsAndRound (scale, BigDecimal(q)) fold (predicateParseError, identity)
+        case JsNumber(q) =>
+          Numeric checkWithinBoundsAndRound (scale, q) fold (predicateParseError, identity)
+      }, { case V.ValueNumeric(v) => v setScale scale }
+    )
+
+  private[this] implicit val `jBD order`: Order[java.math.BigDecimal] =
+    Order.fromScalaOrdering
+
+  private[this] type Inclusive = Boolean
+  private[this] final val Inclusive = true
+  private[this] final val Exclusive = false
+
+  type Boundaries[+A] = (Inclusive, A) \&/ (Inclusive, A)
+
+  private[this] final case class RangeExpr[A](
+      scalar: JsValue PartialFunction A,
+      lfvScalar: LfV PartialFunction A) {
+    import RangeExpr._
+
+    private def scalarE(it: JsValue): PredicateParseError \/ A =
+      scalar.lift(it) \/> predicateParseError(s"invalid boundary $it")
+
+    object Scalar {
+      @inline def unapply(it: JsValue): Option[A] = scalar.lift(it)
+    }
+
+    @SuppressWarnings(Array("org.wartremover.warts.Any"))
+    def unapply(it: JsValue): Option[PredicateParseError \/ Boundaries[A]] =
+      it match {
+        case JsObject(fields) if fields.keySet exists keys =>
+          def badRangeSyntax(s: String): PredicateParseError \/ Nothing =
+            predicateParseError(s"Invalid range query, as $s: $it")
+
+          def side(exK: String, inK: String) = {
+            assert(keys(exK) && keys(inK)) // forgot to update 'keys' when changing the keys
+            (fields get exK, fields get inK) match {
+              case (Some(excl), None) => scalarE(excl) map (a => Some((Exclusive, a)))
+              case (None, Some(incl)) => scalarE(incl) map (a => Some((Inclusive, a)))
+              case (None, None) => \/-(None)
+              case (Some(_), Some(_)) => badRangeSyntax(s"only one of $exK, $inK may be used")
+            }
+          }
+
+          val strays = fields.keySet diff keys
+          Some(
+            if (strays.nonEmpty) badRangeSyntax(s"extra invalid keys $strays included")
+            else {
+              val left = side("%lt", "%lte")
+              val right = side("%gt", "%gte")
+              import \&/._
+              ^(left, right) {
+                case (Some(l), Some(r)) => Both(l, r)
+                case (Some(l), None) => This(l)
+                case (None, Some(r)) => That(r)
+                case (None, None) => sys.error("impossible; denied by 'fields.keySet exists keys'")
+              }
+            })
+        case _ => None
+      }
+
+    def toLiteral(q: A) = Literal { case v if lfvScalar.lift(v) contains q => }
+
+    def toRange(ltgt: Boundaries[A])(implicit A: Order[A]) = Range(ltgt, A, lfvScalar)
+
+    /** Match both the literal and range query cases. */
+    def toQueryParser(implicit A: Order[A]): JsValue PartialFunction ValuePredicate = {
+      val Self = this;
+      {
+        case Scalar(q) => toLiteral(q)
+        case Self(eoIor) => eoIor.map(toRange).merge
+      }
+    }
+  }
+
+  private[this] object RangeExpr {
+    private val keys = Set("%lt", "%lte", "%gt", "%gte")
+  }
+
+  private type PredicateParseError = Nothing
+
+  private[this] def illTypedQuery(it: JsValue, typ: Any): PredicateParseError =
     predicateParseError(s"$it is not a query that can match type $typ")
 
-  private def predicateParseError(s: String): Nothing =
+  private def predicateParseError(s: String): PredicateParseError =
     sys.error(s)
 }

ledger-service/http-json/src/test/scala/com/digitalasset/http/query/ValuePredicateTest.scala

@@ -5,13 +5,14 @@ package com.digitalasset.http
 package query
 
 import json.JsonProtocol.LfValueCodec.{apiValueToJsValue, jsValueToApiValue}
-import com.digitalasset.daml.lf.data.{ImmArray, Numeric, Ref, SortedLookupList}
+import com.digitalasset.daml.lf.data.{Decimal, ImmArray, Numeric, Ref, SortedLookupList, Time}
 import ImmArray.ImmArraySeq
 import com.digitalasset.daml.lf.iface
 import com.digitalasset.daml.lf.value.{Value => V}
 import com.digitalasset.daml.lf.value.TypedValueGenerators.{genAddend, ValueAddend => VA}
 
 import org.scalacheck.{Arbitrary, Gen}
+import org.scalactic.source
 import org.scalatest.prop.{GeneratorDrivenPropertyChecks, TableDrivenPropertyChecks}
 import org.scalatest.{Matchers, WordSpec}
 import spray.json._
@@ -42,8 +43,9 @@ class ValuePredicateTest
           .DefDataType(ImmArraySeq.empty, iface.Record(ImmArraySeq((dummyFieldName, ty))))).lift)
 
   "fromJsObject" should {
-    def c(query: String, ty: VA)(expected: ty.Inj[Cid], shouldMatch: Boolean) =
-      (query.parseJson, ty, ty.inj(expected), shouldMatch)
+    def c(query: String, ty: VA)(expected: ty.Inj[Cid], shouldMatch: Boolean)(
+        implicit pos: source.Position) =
+      (pos.lineNumber, query.parseJson, ty, ty.inj(expected), shouldMatch)
 
     object VAs {
       val oi = VA.optional(VA.int64)
@@ -68,12 +70,43 @@ class ValuePredicateTest
       ("[[42]]", VAs.oooi),
     )
 
+    val excl4143 = """{"%gt": 41, "%lt": 43}"""
+    val incl42 = """{"%gte": 42, "%lte": 42}"""
+    val VAtestNumeric = VA.numeric(Decimal.scale)
+
     val successes = Table(
-      ("query", "type", "expected", "should match?"),
+      ("line#", "query", "type", "expected", "should match?"),
       c("\"foo\"", VA.text)("foo", true),
       c("\"foo\"", VA.text)("bar", false),
       c("42", VA.int64)(42, true),
       c("42", VA.int64)(43, false),
+      c(excl4143, VA.int64)(42, true),
+      c(excl4143, VA.int64)(41, false),
+      c(excl4143, VA.int64)(43, false),
+      c(incl42, VA.int64)(42, true),
+      c(incl42, VA.int64)(41, false),
+      c(incl42, VA.int64)(43, false),
+      c(excl4143, VAtestNumeric)(Numeric assertFromString "42.", true),
+      c(excl4143, VAtestNumeric)(Numeric assertFromString "41.", false),
+      c(excl4143, VAtestNumeric)(Numeric assertFromString "43.", false),
+      c(incl42, VAtestNumeric)(Numeric assertFromString "42.", true),
+      c(incl42, VAtestNumeric)(Numeric assertFromString "41.", false),
+      c(incl42, VAtestNumeric)(Numeric assertFromString "43.", false),
+      c("""{"%lte": 42}""", VA.int64)(42, true),
+      c("""{"%lte": 42}""", VA.int64)(43, false),
+      c("""{"%gte": 42}""", VA.int64)(42, true),
+      c("""{"%gte": 42}""", VA.int64)(41, false),
+      c("""{"%lt": 42}""", VA.int64)(41, true),
+      c("""{"%lt": 42}""", VA.int64)(42, false),
+      c("""{"%gt": 42}""", VA.int64)(43, true),
+      c("""{"%gt": 42}""", VA.int64)(42, false),
+      c("""{"%gt": "bar", "%lt": "foo"}""", VA.text)("baz", true),
+      c("""{"%gte": "1980-01-01", "%lt": "2000-01-01"}""", VA.date)(
+        Time.Date assertFromString "1986-06-21",
+        true),
+      c("""{"%gte": "1980-01-01T00:00:00Z", "%lt": "2000-01-01T00:00:00Z"}""", VA.timestamp)(
+        Time.Timestamp assertFromString "1986-06-21T00:00:00Z",
+        true),
       c("""{"a": 1, "b": 2}""", VA.map(VA.int64))(SortedLookupList(Map("a" -> 1, "b" -> 2)), true),
       c("""{"a": 1, "b": 2}""", VA.map(VA.int64))(SortedLookupList(Map("a" -> 1, "c" -> 2)), false),
       c("""{"a": 1, "b": 2}""", VA.map(VA.int64))(SortedLookupList(Map()), false),
@@ -89,14 +122,15 @@ class ValuePredicateTest
       vp.toFunPredicate(wrappedExpected) shouldBe true
     }
 
-    "examine simple fields literally" in forAll(successes) { (query, va, expected, shouldMatch) =>
-      val (wrappedExpected, defs) = valueAndTypeInObject(expected, va.t)
-      val vp = ValuePredicate.fromJsObject(
-        Map((dummyFieldName: String) -> query),
-        dummyTypeCon,
-        defs
-      )
-      vp.toFunPredicate(wrappedExpected) shouldBe shouldMatch
+    "examine simple fields literally" in forAll(successes) {
+      (_, query, va, expected, shouldMatch) =>
+        val (wrappedExpected, defs) = valueAndTypeInObject(expected, va.t)
+        val vp = ValuePredicate.fromJsObject(
+          Map((dummyFieldName: String) -> query),
+          dummyTypeCon,
+          defs
+        )
+        vp.toFunPredicate(wrappedExpected) shouldBe shouldMatch
     }
 
     "examine all sorts of primitives literally" in forAll(genAddend, minSuccessful(100)) { va =>

unreleased.rst

@@ -31,5 +31,6 @@ HEAD — ongoing
   instead of the contract id.
 - [DAML Triggers] ``getTemplates`` has been renamed to ``getContracts`` to describe its behavior more accurately.
   ``getTemplates`` still exists as a compatiblity helper but it is deprecated and will be removed in a future SDK release.
-- [JSON API - Experimental] Fix to support Archive choice. See issue #3219.
-- [JSON API - Experimental] Implement replay on database consistency violation, See issue #3387.
+- [JSON API - Experimental] Implement replay on database consistency violation, See issue #3387.
+- [JSON API - Experimental] Comparison/range queries supported.
+  See `issue #2780 <https://github.com/digital-asset/daml/issues/2780>`__.