From 0d495ffd97b9872412ab01593c407901ca370152 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Thu, 25 Apr 2024 13:22:50 +0200 Subject: [PATCH] Make conversion of double to BigDecimal exact (#9740) Resolves #9607 by computing `Number.hash` by converting given number to `Float` first and then computing the hash. Also the conversion from `Float.to Decimal` is exact - done via `new BigDecimal(double)`. There is `Decimal.new` that handles the user-friendly conversion. However as a result `Decimal.from 2.1 != Decimal.new 2.1` - that's the only way to ensure consistency between hash code and conversions. --- .vscode/settings.json | 4 +- .../Base/0.0.0-dev/src/Data/Decimal.enso | 9 ++- .../Test/0.0.0-dev/src/Extensions.enso | 4 ++ .../interpreter/test/ValuesGenerator.java | 9 +++ .../builtin/meta/EqualsSimpleNode.java | 35 +++++++++--- .../expression/builtin/meta/HashCodeNode.java | 55 +++---------------- .../builtin/number/utils/BigIntegerOps.java | 9 +++ .../runtime/number/EnsoBigInteger.java | 2 +- .../org/enso/base/numeric/Decimal_Utils.java | 27 ++------- test/Base_Tests/src/Data/Decimal_Spec.enso | 42 +++++++++----- .../Column_Operations_Spec.enso | 2 +- .../Join/Union_Spec.enso | 3 +- .../src/Database/Postgres_Spec.enso | 8 +-- .../src/In_Memory/Integer_Overflow_Spec.enso | 16 +++--- 14 files changed, 114 insertions(+), 111 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 3b160f135b..0baf6fa290 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -23,5 +23,7 @@ ], "files.watcherExclude": { "**/target": true - } + }, + "metals.inlayHints.implicitArguments.enable": true, + "metals.inlayHints.implicitConversions.enable": true } diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Decimal.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Decimal.enso index 0580595b3b..8d8ff58317 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Decimal.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Decimal.enso @@ -12,7 +12,7 @@ import project.Warning.Warning from project.Data.Boolean import Boolean, False, True from project.Data.Numbers import Float, Integer, Number, Number_Parse_Error from project.Data.Numeric.Internal.Decimal_Internal import Decimal_Comparator -from project.Data.Ordering import Comparable, Ordering +from project.Data.Ordering import Comparable, Ordering, Default_Comparator from project.Errors.Common import Arithmetic_Error, Loss_Of_Numeric_Precision, Out_Of_Range, Unsupported_Argument_Types from project.Widget_Helpers import make_number_format_selector @@ -1059,7 +1059,12 @@ Comparable.from (_ : Number) = Decimal_Comparator Decimal.from (that : Text) = Decimal.from_string that ## PRIVATE -Decimal.from (that : Number) = Decimal.new that +Decimal.from (that : Integer) = Decimal.new that +Decimal.from (that : Float) = + is_exceptional = that.is_nan || that.is_infinite + if is_exceptional then Error.throw (Illegal_Argument.Error "Cannot convert "+that.to_text+" to a Decimal") else + handle_java_exception <| attach_loss_of_numeric_precision that <| + Decimal.Value <| Decimal_Utils.fromFloatExact that ## PRIVATE Float.from (that : Decimal) = that.to_float diff --git a/distribution/lib/Standard/Test/0.0.0-dev/src/Extensions.enso b/distribution/lib/Standard/Test/0.0.0-dev/src/Extensions.enso index b0a4700ac1..f10e9493bb 100644 --- a/distribution/lib/Standard/Test/0.0.0-dev/src/Extensions.enso +++ b/distribution/lib/Standard/Test/0.0.0-dev/src/Extensions.enso @@ -304,6 +304,10 @@ Number.should_equal self that epsilon=0 frames_to_skip=0 = msg = self.to_text + " did not equal " + that.to_text + " (at " + loc + ")." Test.fail msg +Decimal.should_equal : Number -> Float-> Float -> Integer -> Spec_Result +Decimal.should_equal self that epsilon=0 frames_to_skip=0 = + self.to_float . should_equal that.to_float epsilon frames_to_skip+1 + ## Asserts that `self` value is not an error. It returns the original value, so that it can be inspected further. diff --git a/engine/runtime-integration-tests/src/test/java/org/enso/interpreter/test/ValuesGenerator.java b/engine/runtime-integration-tests/src/test/java/org/enso/interpreter/test/ValuesGenerator.java index c251993ab9..d457271924 100644 --- a/engine/runtime-integration-tests/src/test/java/org/enso/interpreter/test/ValuesGenerator.java +++ b/engine/runtime-integration-tests/src/test/java/org/enso/interpreter/test/ValuesGenerator.java @@ -369,6 +369,12 @@ public final class ValuesGenerator { if (languages.contains(Language.ENSO)) { collect.add(v(null, "", "42").type()); collect.add(v(null, "", "6.7").type()); + collect.add(v(null, "", "9223372036854776000").type()); + collect.add(v(null, "", "9223372036854775999").type()); + collect.add(v(null, "", "9223372036854775808").type()); + collect.add(v(null, "", "9223372036854776000.0").type()); + collect.add(v(null, "", "9223372036854775999.0").type()); + collect.add(v(null, "", "9223372036854775808.0").type()); collect.add(v(null, "import Standard.Base.Data.Numbers", "40321 * 43202").type()); collect.add( v( @@ -392,6 +398,9 @@ public final class ValuesGenerator { collect.add(ctx.asValue((short) 44)); collect.add(ctx.asValue((int) 5432)); collect.add(ctx.asValue((long) 5435432)); + long over52bit = 2 ^ 56 + 777; + collect.add(ctx.asValue(over52bit)); + collect.add(ctx.asValue((double) over52bit)); collect.add(ctx.asValue((float) Math.PI)); collect.add(ctx.asValue((double) Math.E)); collect.add(ctx.asValue(Double.NaN)); diff --git a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/meta/EqualsSimpleNode.java b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/meta/EqualsSimpleNode.java index 46c6b1c47b..b18d30ad79 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/meta/EqualsSimpleNode.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/meta/EqualsSimpleNode.java @@ -12,6 +12,7 @@ import com.oracle.truffle.api.interop.TruffleObject; import com.oracle.truffle.api.interop.UnsupportedMessageException; import com.oracle.truffle.api.library.CachedLibrary; import com.oracle.truffle.api.nodes.Node; +import java.math.BigDecimal; import java.math.BigInteger; import org.enso.interpreter.node.expression.builtin.number.utils.BigIntegerOps; import org.enso.interpreter.runtime.EnsoContext; @@ -153,21 +154,32 @@ public abstract class EqualsSimpleNode extends Node { boolean equalsDoubleInterop( double self, Object other, - @Shared("interop") @CachedLibrary(limit = "10") InteropLibrary interop) { + @Shared("interop") @CachedLibrary(limit = "10") InteropLibrary iop) { try { - if (interop.fitsInDouble(other)) { - return self == interop.asDouble(other); + if (iop.fitsInDouble(other)) { + return self == iop.asDouble(other); + } + if (iop.fitsInBigInteger(other)) { + if (Double.isNaN(self)) { + return false; + } + if (Double.isInfinite(self)) { + return false; + } + return equalsDoubleBigInteger(self, asBigInteger(iop, other)); + } else { + return false; } - var otherBig = interop.asBigInteger(other); - return self == asDouble(otherBig); } catch (UnsupportedMessageException ex) { return false; } } @TruffleBoundary - private static double asDouble(BigInteger big) { - return big.doubleValue(); + private static boolean equalsDoubleBigInteger(double self, BigInteger big) { + var selfDecimal = new BigDecimal(self); + var otherDecimal = new BigDecimal(big); + return selfDecimal.equals(otherDecimal); } @Specialization(guards = {"isBigInteger(iop, self)", "isBigInteger(iop, other)"}) @@ -180,12 +192,17 @@ public abstract class EqualsSimpleNode extends Node { } @Specialization(guards = "isBigInteger(iop, self)") - @TruffleBoundary boolean equalsBigIntDouble( Object self, double other, @Shared("interop") @CachedLibrary(limit = "10") InteropLibrary iop) { - return asBigInteger(iop, self).doubleValue() == other; + if (Double.isNaN(other)) { + return false; + } + if (Double.isInfinite(other)) { + return false; + } + return equalsDoubleBigInteger(other, asBigInteger(iop, self)); } @Specialization(guards = "isBigInteger(iop, self)") diff --git a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/meta/HashCodeNode.java b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/meta/HashCodeNode.java index 5af09ca58e..e951e25da2 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/meta/HashCodeNode.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/meta/HashCodeNode.java @@ -22,7 +22,6 @@ import com.oracle.truffle.api.nodes.ExplodeLoop; import com.oracle.truffle.api.nodes.Node; import com.oracle.truffle.api.profiles.ConditionProfile; import com.oracle.truffle.api.profiles.LoopConditionProfile; -import java.math.BigDecimal; import java.time.LocalDateTime; import java.time.ZonedDateTime; import java.util.Arrays; @@ -31,7 +30,6 @@ import org.enso.interpreter.dsl.BuiltinMethod; import org.enso.interpreter.node.callable.InvokeCallableNode.ArgumentsExecutionMode; import org.enso.interpreter.node.callable.InvokeCallableNode.DefaultsExecutionMode; import org.enso.interpreter.node.callable.dispatch.InvokeFunctionNode; -import org.enso.interpreter.node.expression.builtin.number.utils.BigIntegerOps; import org.enso.interpreter.node.expression.builtin.ordering.CustomComparatorNode; import org.enso.interpreter.node.expression.builtin.ordering.CustomComparatorNodeGen; import org.enso.interpreter.node.expression.builtin.ordering.HashCallbackNode; @@ -89,70 +87,31 @@ public abstract class HashCodeNode extends Node { public abstract long execute(@AcceptsError Object object); - /** Specializations for primitive values * */ - @Specialization - long hashCodeForShort(short s) { - return s; - } - - @Specialization - long hashCodeForByte(byte b) { - return b; - } - @Specialization long hashCodeForLong(long l) { // By casting long to double, we lose some precision on purpose return hashCodeForDouble((double) l); } - @Specialization - long hashCodeForInt(int i) { - return hashCodeForLong(i); - } - - @Specialization - long hashCodeForFloat(float f) { - return Float.hashCode(f); - } - @Specialization long hashCodeForDouble(double d) { - if (Double.isNaN(d)) { - // NaN is Incomparable, just return a "random" constant - return 456879; - } else if (d % 1.0 != 0 || BigIntegerOps.fitsInLong(d)) { - return Double.hashCode(d); - } else { - return bigDoubleHash(d); - } - } - - @TruffleBoundary - private static long bigDoubleHash(double d) { - try { - return BigDecimal.valueOf(d).toBigIntegerExact().hashCode(); - } catch (ArithmeticException e) { - throw EnsoContext.get(null).raiseAssertionPanic(null, null, e); - } + return Double.hashCode(d); } @Specialization @TruffleBoundary - long hashCodeForBigInteger(EnsoBigInteger bigInteger) { - return bigInteger.getValue().hashCode(); + long hashCodeForBigInteger( + EnsoBigInteger bigInteger, + @Shared("interop") @CachedLibrary(limit = "10") InteropLibrary interop) { + return hashCodeForDouble(bigInteger.getValue().doubleValue()); } - @Specialization(guards = {"interop.fitsInLong(v) || interop.fitsInBigInteger(v)"}) + @Specialization(guards = {"interop.fitsInBigInteger(v)"}) @TruffleBoundary long hashCodeForBigInteger( Object v, @Shared("interop") @CachedLibrary(limit = "10") InteropLibrary interop) { try { - if (interop.fitsInLong(v)) { - return hashCodeForLong(interop.asLong(v)); - } else { - return interop.asBigInteger(v).hashCode(); - } + return hashCodeForDouble(interop.asBigInteger(v).doubleValue()); } catch (UnsupportedMessageException ex) { throw EnsoContext.get(this).raiseAssertionPanic(this, "Expecting BigInteger", ex); } diff --git a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/number/utils/BigIntegerOps.java b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/number/utils/BigIntegerOps.java index 6889d28d21..039d8916a8 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/number/utils/BigIntegerOps.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/number/utils/BigIntegerOps.java @@ -238,6 +238,15 @@ public class BigIntegerOps { return decimal <= Long.MAX_VALUE && decimal >= Long.MIN_VALUE; } + private static boolean fitsInLongCompat(double decimal) { + var nulaMinus = Double.doubleToRawLongBits(-0d); + if (nulaMinus == Double.doubleToRawLongBits(decimal)) { + return false; + } + var converted = (long) decimal; + return converted != Long.MAX_VALUE && converted == decimal; + } + public static boolean fitsInInt(long number) { return number >= Integer.MIN_VALUE && number <= Integer.MAX_VALUE; } diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/number/EnsoBigInteger.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/number/EnsoBigInteger.java index 9937458c96..00dc63ae93 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/number/EnsoBigInteger.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/number/EnsoBigInteger.java @@ -26,7 +26,7 @@ public final class EnsoBigInteger implements EnsoObject { * @param value the value to wrap. */ public EnsoBigInteger(BigInteger value) { - assert (value.bitLength() > 63); + assert (value.bitLength() > 63) : "Too small BigInteger: " + value; this.value = value; } diff --git a/std-bits/base/src/main/java/org/enso/base/numeric/Decimal_Utils.java b/std-bits/base/src/main/java/org/enso/base/numeric/Decimal_Utils.java index a818750db8..ab26435d6e 100644 --- a/std-bits/base/src/main/java/org/enso/base/numeric/Decimal_Utils.java +++ b/std-bits/base/src/main/java/org/enso/base/numeric/Decimal_Utils.java @@ -52,13 +52,17 @@ public class Decimal_Utils { } } - public static BigDecimal fromFloat(Double d) { + public static BigDecimal fromFloat(double d) { // According to the BigInteger Javadocs, valueOf is preferred because "the // value returned is equal to that resulting from constructing a BigDecimal // from the result of using Double.toString(double)." return BigDecimal.valueOf(d); } + public static BigDecimal fromFloatExact(double d) { + return new BigDecimal(d); + } + public static ConversionResult fromFloat(Double d, MathContext mc) { // We do not check for precision loss here because we always attach a // warning when converting from float. @@ -71,26 +75,7 @@ public class Decimal_Utils { } public static int hashCodeOf(BigDecimal bd) { - boolean isFractional = bd.remainder(BigDecimal.ONE).compareTo(BigDecimal.ZERO) != 0; - boolean fitsInLong = fitsInLong(bd); - if (isFractional || fitsInLong) { - double d = bd.doubleValue(); - var fitsInDouble = Double.isFinite(d); - if (fitsInDouble) { - return Double.hashCode(d); - } else { - // Infinite values here just means finite values outside the double - // range. In this path, the values must be fractional, and so cannot - // coincide with a value of any other type (including BigInteger), so we - // can hash it however we want. - assert isFractional; - return bd.hashCode(); - } - } else { - // Will not throw ArithmeticException since the value has a 0 fractional part. - assert !isFractional; - return bd.toBigIntegerExact().hashCode(); - } + return Double.hashCode(bd.doubleValue()); } public static BigDecimal round(BigDecimal bd, int decimalPlaces, boolean useBankers) { diff --git a/test/Base_Tests/src/Data/Decimal_Spec.enso b/test/Base_Tests/src/Data/Decimal_Spec.enso index aa65de087a..5d31eeeb20 100644 --- a/test/Base_Tests/src/Data/Decimal_Spec.enso +++ b/test/Base_Tests/src/Data/Decimal_Spec.enso @@ -301,8 +301,8 @@ add_specs suite_builder = values.map pr-> v = pr.at 0 - d = Decimal.new v - expected = pr.at 1 + d = Decimal.new v . to_float + expected = pr.at 1 . to_float d . should_equal expected expected . should_equal d @@ -367,13 +367,23 @@ add_specs suite_builder = suite_builder.group "edge cases" group_builder-> group_builder.specify "can support values outside the double range" <| d = Decimal.new Float.max_value + (d == Float.max_value) . should_be_false + ((d * d) == Float.max_value) . should_be_false + ((-(d * d)) == -Float.max_value) . should_be_false + Comparable.hash_builtin (d * d) . should_equal 2146435072 + Comparable.hash_builtin -(d * d) . should_equal -1048576 + Comparable.hash_builtin ((d * d) + 0.1) . should_equal 2146435072 + Comparable.hash_builtin ((-(d * d)) + 0.1) . should_equal -1048576 + + group_builder.specify "from conversion supports values outside the double range" <| + d = Float.max_value.to Decimal (d == Float.max_value) . should_be_true ((d * d) == Float.max_value) . should_be_false ((-(d * d)) == -Float.max_value) . should_be_false - Comparable.hash_builtin (d * d) . should_equal -1851772048 - Comparable.hash_builtin -(d * d) . should_equal 1851772048 - Comparable.hash_builtin ((d * d) + 0.1) . should_equal -1176480326 - Comparable.hash_builtin ((-(d * d)) + 0.1) . should_equal -534461530 + Comparable.hash_builtin (d * d) . should_equal 2146435072 + Comparable.hash_builtin -(d * d) . should_equal -1048576 + Comparable.hash_builtin ((d * d) + 0.1) . should_equal 2146435072 + Comparable.hash_builtin ((-(d * d)) + 0.1) . should_equal -1048576 suite_builder.group "arithmetic" group_builder-> group_builder.specify "should allow arithmetic with Decimals, without Math_Context" <| @@ -577,12 +587,12 @@ add_specs suite_builder = f.should_be_a Float f.should_equal 56.34 - group_builder.specify "Decimal.to_float should compare correctly with the original Decimal" <| + group_builder.specify "Decimal.to_float cannot compare correctly with the original Decimal" <| huge_a = Decimal.new "3.4E300" huge_a_float = Float.from huge_a - (huge_a_float == huge_a) . should_be_true - (huge_a == huge_a_float) . should_be_true + (huge_a_float == huge_a) . should_be_false + (huge_a == huge_a_float) . should_be_false group_builder.specify "Decimal.to_integer should compare correctly with the original Decimal" <| huge_a = Decimal.new "3.4E320" @@ -604,14 +614,15 @@ add_specs suite_builder = cases = [] + [[5, 3, 2], [5.0, 3.0, 2.0], [3.5, 2, 1.5], [10.5, 1.0, 0.5], [3, 1, 0], [3.0, 1.0, 0]] + [[-5, 3, -2], [-5.0, 3.0, -2.0], [-3.5, 2, -1.5], [-10.5, 1.0, -0.5], [-3, 1, 0], [-3.0, 1.0, 0]] - + [[2.7, 0.5, 0.2], [2.7, 0.4, 0.3], [-2.7, 0.5, -0.2], [-2.7, 0.4, -0.3]] + + [[2.7, 0.5, 0.2], [2.7, 0.4, 0.3, 0.001], [-2.7, 0.5, -0.2], [-2.7, 0.4, -0.3, 0.001]] + [["9223372036854775807", 10, 7], ["9223372036854775808", 10, 8], ["922337203685477580000000000008", 10, 8]] + [["-9223372036854775808", 10, -8], ["-9223372036854775809", 10, -9], ["-922337203685477580000000000008", 10, -8]] - cases.map c-> + cases.map c-> Test.with_clue ("["+c.to_text+"] ") <| base = c.at 0 modulus = c.at 1 residue = c.at 2 - (Decimal.new base % modulus) . should_equal residue + epsilon = if c.length > 3 then c.at 3 else 0.0 + (Decimal.new base % modulus) . should_equal epsilon=epsilon residue (Decimal.new base % Decimal.new modulus) . should_equal residue group_builder.specify "should define remainder (example)" <| @@ -734,7 +745,10 @@ add_specs suite_builder = Decimal.new "0" . signum . should_equal 0 suite_builder.group "rounding" group_builder-> - do_round n dp=0 use_bankers=False = Decimal.new n . round dp use_bankers + do_round n dp=0 use_bankers=False = + d = Decimal.new n . round dp use_bankers + d.to_float + Round_Spec.add_specs group_builder do_round group_builder.specify "Large values" <| @@ -916,7 +930,7 @@ add_specs suite_builder = 12 . min (Decimal.new "13") . should_equal (Decimal.new "12") Decimal.new "12.1" . min 12.3 . should_equal (Decimal.new "12.1") - 12.1 . min (Decimal.new "12.3") . should_equal (Decimal.new "12.1") + 12.1 . min (Decimal.new "12.3") . should_equal (Decimal.from 12.1) Decimal.new "0" . min (Decimal.new "-1") . should_equal (Decimal.new "-1") 0 . min (Decimal.new "-1") . should_equal (Decimal.new "-1") diff --git a/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso index 25235bc67d..20c80f4d2d 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso @@ -1072,7 +1072,7 @@ add_specs suite_builder setup = decimal_col.value_type.is_decimal . should_be_true decimal_col2 = decimal_col + decimal_col*decimal_col [(.floor), (.ceil), (.truncate), (x-> x.round 0), (x-> x.round 2)].each op-> - op decimal_col2 . to_vector . should_equal [i1 + i1*i1] + op decimal_col2 . to_vector . should_equal [i1 + i1*i1 . to_float] group_builder.specify "should allow Nothing/NULL" <| table = table_builder [["x", [Nothing, 0.51, 0.59, 3.51, Nothing, 3.59, -0.51, -0.59, -3.51, -3.59]]] diff --git a/test/Table_Tests/src/Common_Table_Operations/Join/Union_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Join/Union_Spec.enso index 5681a108e5..e2c1e3d37b 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Join/Union_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Join/Union_Spec.enso @@ -384,8 +384,7 @@ run_union_tests group_builder setup call_union = t4 = call_union [t2, t1, t3] allow_type_widening=True t4.at "X" . value_type . should_equal Value_Type.Float - # Inexact float equality will make this pass: - t4.at "X" . to_vector . should_equal [1.5, 2.5, 3.5, 1, (2^62)-1, 3, (2^100)+1, 2^10, 2] + t4.at "X" . to_vector . should_equal [1.5, 2.5, 3.5, 1, (2^62)-1, 3, (2^100)+1 . to_float, 2^10, 2] w = Problems.expect_only_warning Loss_Of_Integer_Precision t4 # Losing precision on (2^62)-1 and 2^100+1. diff --git a/test/Table_Tests/src/Database/Postgres_Spec.enso b/test/Table_Tests/src/Database/Postgres_Spec.enso index 23f1201570..39aae27335 100644 --- a/test/Table_Tests/src/Database/Postgres_Spec.enso +++ b/test/Table_Tests/src/Database/Postgres_Spec.enso @@ -402,7 +402,7 @@ postgres_specific_spec suite_builder create_connection_fn db_name setup = t2.at "Y" . value_type . scale . should_equal Nothing t2.at "X" . to_vector . should_equal [10, x] # Only works by approximation: - t2.at "Y" . to_vector . should_equal [20, x+10] + t2.at "Y" . to_vector . should_equal [20, x+10 . to_float] t2.at "Y" . cast Value_Type.Char . to_vector . should_equal ["20", (x+10).to_text] m2 = t2.remove_warnings.read @@ -410,7 +410,7 @@ postgres_specific_spec suite_builder create_connection_fn db_name setup = # As noted above - once operations are performed, the scale=0 may be lost and the column will be approximated as a float. m2.at "Y" . value_type . should_equal Value_Type.Float m2.at "X" . to_vector . should_equal [10, x] - m2.at "Y" . to_vector . should_equal [20, x+10] + m2.at "Y" . to_vector . should_equal [20, x+10 . to_float] w2 = Problems.expect_only_warning Inexact_Type_Coercion m2 w2.requested_type . should_equal (Value_Type.Decimal precision=Nothing scale=Nothing) w2.actual_type . should_equal Value_Type.Float @@ -425,7 +425,7 @@ postgres_specific_spec suite_builder create_connection_fn db_name setup = t3 . at "X" . value_type . precision . should_equal Nothing t3 . at "X" . value_type . scale . should_equal Nothing # Works but only relying on imprecise float equality: - t3 . at "X" . to_vector . should_equal [super_large] + t3 . at "X" . to_vector . should_equal [super_large . to_float] w3 = Problems.expect_only_warning Inexact_Type_Coercion t3 w3.requested_type . should_equal (Value_Type.Decimal precision=Nothing scale=0) w3.actual_type . should_equal (Value_Type.Decimal precision=Nothing scale=Nothing) @@ -433,7 +433,7 @@ postgres_specific_spec suite_builder create_connection_fn db_name setup = m4 = t3.remove_warnings.read # Because we no longer have a set scale, we cannot get a BigInteger column back - we'd need BigDecimal, but that is not fully supported yet in Enso - so we get the closest approximation - the imprecise Float. m4 . at "X" . value_type . should_equal Value_Type.Float - m4 . at "X" . to_vector . should_equal [super_large] + m4 . at "X" . to_vector . should_equal [super_large . to_float] w4 = Problems.expect_only_warning Inexact_Type_Coercion m4 w4.requested_type . should_equal (Value_Type.Decimal precision=Nothing scale=Nothing) w4.actual_type . should_equal Value_Type.Float diff --git a/test/Table_Tests/src/In_Memory/Integer_Overflow_Spec.enso b/test/Table_Tests/src/In_Memory/Integer_Overflow_Spec.enso index f3818e7af8..546b93dad0 100644 --- a/test/Table_Tests/src/In_Memory/Integer_Overflow_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Integer_Overflow_Spec.enso @@ -261,7 +261,7 @@ add_specs suite_builder = Problems.assume_no_problems c2 c2.value_type . should_equal Value_Type.Float # This will be an inexact equality, based on float rounding. - c2.to_vector . should_equal [x, 1.5] + c2.to_vector . map .to_float . should_equal [x . to_float, 1.5] c3 = Column.from_vector "X" [2^70, 1] Problems.assume_no_problems c3 @@ -346,8 +346,8 @@ add_specs suite_builder = (x * y) . to_vector . should_equal [10^30, 2^71, Nothing, 3 * 4] (x % y) . to_vector . should_equal [0, 0, Nothing, 3] (x % z) . to_vector . should_equal [1.0, 1.5, Nothing, 3.0] - (x / y) . to_vector . should_equal [10^30, 2^69, Nothing, 0.75] - (x ^ y) . to_vector . should_equal [10^30, 2^140, Nothing, 3^4] + (x / y) . to_vector . should_equal [10^30 . to_float, 2^69 . to_float, Nothing, 0.75] + (x ^ y) . to_vector . should_equal [10^30 . to_float, 2^140 . to_float, Nothing, 3^4] (x == y) . to_vector . should_equal [False, False, Nothing, False] (x < y) . to_vector . should_equal [False, False, Nothing, True] (x >= y) . to_vector . should_equal [True, True, Nothing, False] @@ -357,7 +357,7 @@ add_specs suite_builder = (x.min y) . to_vector . should_equal [1, 2, 3, 3] (x.max y) . to_vector . should_equal [10^30, 2^70, 3, 4] (x.min z) . to_vector . should_equal [1.5, 2.5, 3.5, 3] - (x.max [y, z]) . to_vector . should_equal [10^30, 2^70, 3.5, 4.5] + (x.max [y, z]) . to_vector . should_equal [10^30 . to_float, 2^70 . to_float, 3.5, 4.5] (y * x) . to_vector . should_equal [10^30, 2^71, Nothing, 3 * 4] (y % x) . to_vector . should_equal [1, 2, Nothing, 1] @@ -372,22 +372,22 @@ add_specs suite_builder = (y.min x) . to_vector . should_equal [1, 2, 3, 3] (y.max x) . to_vector . should_equal [10^30, 2^70, 3, 4] (z.min x) . to_vector . should_equal [1.5, 2.5, 3.5, 3] - (z.max x) . to_vector . should_equal [10^30, 2^70, 3.5, 4.5] + (z.max x) . to_vector . should_equal [10^30 . to_float, 2^70 . to_float, 3.5, 4.5] x.fill_nothing y . to_vector . should_equal [10^30, 2^70, 3, 3] - x.fill_nothing z . to_vector . should_equal [10^30, 2^70, 3.5, 3] + x.fill_nothing z . to_vector . should_equal [10^30 . to_float, 2^70 . to_float, 3.5, 3] y.fill_nothing x . to_vector . should_equal [1, 2, 3, 4] y2 = (y < 2).iif Nothing y y2.to_vector . should_equal [Nothing, 2, 3, 4] y2.fill_nothing x . to_vector . should_equal [10^30, 2, 3, 4] - ((z < 2).iif Nothing z).fill_nothing x . to_vector . should_equal [10^30, 2.5, 3.5, 4.5] + ((z < 2).iif Nothing z).fill_nothing x . to_vector . should_equal [10^30 . to_float, 2.5, 3.5, 4.5] r3 = x.fill_nothing 1.5 r3.value_type . should_equal Value_Type.Float # 2^70 is not exactly representable as a Float. (2^70 + 0.0).truncate . should_not_equal (2^70) Problems.expect_only_warning Loss_Of_Integer_Precision r3 - r3.to_vector . should_equal [10^30, 2^70, 1.5, 3] + r3.to_vector . should_equal [10^30 . to_float, 2^70 . to_float, 1.5, 3] r4 = x.fill_nothing 23 r4.value_type . should_be_a (Value_Type.Decimal ...)