Make conversion of double to BigDecimal exact (#9740)

Resolves #9607 by computing `Number.hash` by converting given number to `Float` first and then computing the hash. Also the conversion from `Float.to Decimal` is exact - done via `new BigDecimal(double)`. There is `Decimal.new` that handles the user-friendly conversion. However as a result `Decimal.from 2.1 != Decimal.new 2.1` - that's the only way to ensure consistency between hash code and conversions.
This commit is contained in:
Jaroslav Tulach 2024-04-25 13:22:50 +02:00 committed by GitHub
parent 5807c5c112
commit 0d495ffd97
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 114 additions and 111 deletions

View File

@ -23,5 +23,7 @@
], ],
"files.watcherExclude": { "files.watcherExclude": {
"**/target": true "**/target": true
} },
"metals.inlayHints.implicitArguments.enable": true,
"metals.inlayHints.implicitConversions.enable": true
} }

View File

@ -12,7 +12,7 @@ import project.Warning.Warning
from project.Data.Boolean import Boolean, False, True from project.Data.Boolean import Boolean, False, True
from project.Data.Numbers import Float, Integer, Number, Number_Parse_Error from project.Data.Numbers import Float, Integer, Number, Number_Parse_Error
from project.Data.Numeric.Internal.Decimal_Internal import Decimal_Comparator from project.Data.Numeric.Internal.Decimal_Internal import Decimal_Comparator
from project.Data.Ordering import Comparable, Ordering from project.Data.Ordering import Comparable, Ordering, Default_Comparator
from project.Errors.Common import Arithmetic_Error, Loss_Of_Numeric_Precision, Out_Of_Range, Unsupported_Argument_Types from project.Errors.Common import Arithmetic_Error, Loss_Of_Numeric_Precision, Out_Of_Range, Unsupported_Argument_Types
from project.Widget_Helpers import make_number_format_selector from project.Widget_Helpers import make_number_format_selector
@ -1059,7 +1059,12 @@ Comparable.from (_ : Number) = Decimal_Comparator
Decimal.from (that : Text) = Decimal.from_string that Decimal.from (that : Text) = Decimal.from_string that
## PRIVATE ## PRIVATE
Decimal.from (that : Number) = Decimal.new that Decimal.from (that : Integer) = Decimal.new that
Decimal.from (that : Float) =
is_exceptional = that.is_nan || that.is_infinite
if is_exceptional then Error.throw (Illegal_Argument.Error "Cannot convert "+that.to_text+" to a Decimal") else
handle_java_exception <| attach_loss_of_numeric_precision that <|
Decimal.Value <| Decimal_Utils.fromFloatExact that
## PRIVATE ## PRIVATE
Float.from (that : Decimal) = that.to_float Float.from (that : Decimal) = that.to_float

View File

@ -304,6 +304,10 @@ Number.should_equal self that epsilon=0 frames_to_skip=0 =
msg = self.to_text + " did not equal " + that.to_text + " (at " + loc + ")." msg = self.to_text + " did not equal " + that.to_text + " (at " + loc + ")."
Test.fail msg Test.fail msg
Decimal.should_equal : Number -> Float-> Float -> Integer -> Spec_Result
Decimal.should_equal self that epsilon=0 frames_to_skip=0 =
self.to_float . should_equal that.to_float epsilon frames_to_skip+1
## Asserts that `self` value is not an error. ## Asserts that `self` value is not an error.
It returns the original value, so that it can be inspected further. It returns the original value, so that it can be inspected further.

View File

@ -369,6 +369,12 @@ public final class ValuesGenerator {
if (languages.contains(Language.ENSO)) { if (languages.contains(Language.ENSO)) {
collect.add(v(null, "", "42").type()); collect.add(v(null, "", "42").type());
collect.add(v(null, "", "6.7").type()); collect.add(v(null, "", "6.7").type());
collect.add(v(null, "", "9223372036854776000").type());
collect.add(v(null, "", "9223372036854775999").type());
collect.add(v(null, "", "9223372036854775808").type());
collect.add(v(null, "", "9223372036854776000.0").type());
collect.add(v(null, "", "9223372036854775999.0").type());
collect.add(v(null, "", "9223372036854775808.0").type());
collect.add(v(null, "import Standard.Base.Data.Numbers", "40321 * 43202").type()); collect.add(v(null, "import Standard.Base.Data.Numbers", "40321 * 43202").type());
collect.add( collect.add(
v( v(
@ -392,6 +398,9 @@ public final class ValuesGenerator {
collect.add(ctx.asValue((short) 44)); collect.add(ctx.asValue((short) 44));
collect.add(ctx.asValue((int) 5432)); collect.add(ctx.asValue((int) 5432));
collect.add(ctx.asValue((long) 5435432)); collect.add(ctx.asValue((long) 5435432));
long over52bit = 2 ^ 56 + 777;
collect.add(ctx.asValue(over52bit));
collect.add(ctx.asValue((double) over52bit));
collect.add(ctx.asValue((float) Math.PI)); collect.add(ctx.asValue((float) Math.PI));
collect.add(ctx.asValue((double) Math.E)); collect.add(ctx.asValue((double) Math.E));
collect.add(ctx.asValue(Double.NaN)); collect.add(ctx.asValue(Double.NaN));

View File

@ -12,6 +12,7 @@ import com.oracle.truffle.api.interop.TruffleObject;
import com.oracle.truffle.api.interop.UnsupportedMessageException; import com.oracle.truffle.api.interop.UnsupportedMessageException;
import com.oracle.truffle.api.library.CachedLibrary; import com.oracle.truffle.api.library.CachedLibrary;
import com.oracle.truffle.api.nodes.Node; import com.oracle.truffle.api.nodes.Node;
import java.math.BigDecimal;
import java.math.BigInteger; import java.math.BigInteger;
import org.enso.interpreter.node.expression.builtin.number.utils.BigIntegerOps; import org.enso.interpreter.node.expression.builtin.number.utils.BigIntegerOps;
import org.enso.interpreter.runtime.EnsoContext; import org.enso.interpreter.runtime.EnsoContext;
@ -153,21 +154,32 @@ public abstract class EqualsSimpleNode extends Node {
boolean equalsDoubleInterop( boolean equalsDoubleInterop(
double self, double self,
Object other, Object other,
@Shared("interop") @CachedLibrary(limit = "10") InteropLibrary interop) { @Shared("interop") @CachedLibrary(limit = "10") InteropLibrary iop) {
try { try {
if (interop.fitsInDouble(other)) { if (iop.fitsInDouble(other)) {
return self == interop.asDouble(other); return self == iop.asDouble(other);
}
if (iop.fitsInBigInteger(other)) {
if (Double.isNaN(self)) {
return false;
}
if (Double.isInfinite(self)) {
return false;
}
return equalsDoubleBigInteger(self, asBigInteger(iop, other));
} else {
return false;
} }
var otherBig = interop.asBigInteger(other);
return self == asDouble(otherBig);
} catch (UnsupportedMessageException ex) { } catch (UnsupportedMessageException ex) {
return false; return false;
} }
} }
@TruffleBoundary @TruffleBoundary
private static double asDouble(BigInteger big) { private static boolean equalsDoubleBigInteger(double self, BigInteger big) {
return big.doubleValue(); var selfDecimal = new BigDecimal(self);
var otherDecimal = new BigDecimal(big);
return selfDecimal.equals(otherDecimal);
} }
@Specialization(guards = {"isBigInteger(iop, self)", "isBigInteger(iop, other)"}) @Specialization(guards = {"isBigInteger(iop, self)", "isBigInteger(iop, other)"})
@ -180,12 +192,17 @@ public abstract class EqualsSimpleNode extends Node {
} }
@Specialization(guards = "isBigInteger(iop, self)") @Specialization(guards = "isBigInteger(iop, self)")
@TruffleBoundary
boolean equalsBigIntDouble( boolean equalsBigIntDouble(
Object self, Object self,
double other, double other,
@Shared("interop") @CachedLibrary(limit = "10") InteropLibrary iop) { @Shared("interop") @CachedLibrary(limit = "10") InteropLibrary iop) {
return asBigInteger(iop, self).doubleValue() == other; if (Double.isNaN(other)) {
return false;
}
if (Double.isInfinite(other)) {
return false;
}
return equalsDoubleBigInteger(other, asBigInteger(iop, self));
} }
@Specialization(guards = "isBigInteger(iop, self)") @Specialization(guards = "isBigInteger(iop, self)")

View File

@ -22,7 +22,6 @@ import com.oracle.truffle.api.nodes.ExplodeLoop;
import com.oracle.truffle.api.nodes.Node; import com.oracle.truffle.api.nodes.Node;
import com.oracle.truffle.api.profiles.ConditionProfile; import com.oracle.truffle.api.profiles.ConditionProfile;
import com.oracle.truffle.api.profiles.LoopConditionProfile; import com.oracle.truffle.api.profiles.LoopConditionProfile;
import java.math.BigDecimal;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.time.ZonedDateTime; import java.time.ZonedDateTime;
import java.util.Arrays; import java.util.Arrays;
@ -31,7 +30,6 @@ import org.enso.interpreter.dsl.BuiltinMethod;
import org.enso.interpreter.node.callable.InvokeCallableNode.ArgumentsExecutionMode; import org.enso.interpreter.node.callable.InvokeCallableNode.ArgumentsExecutionMode;
import org.enso.interpreter.node.callable.InvokeCallableNode.DefaultsExecutionMode; import org.enso.interpreter.node.callable.InvokeCallableNode.DefaultsExecutionMode;
import org.enso.interpreter.node.callable.dispatch.InvokeFunctionNode; import org.enso.interpreter.node.callable.dispatch.InvokeFunctionNode;
import org.enso.interpreter.node.expression.builtin.number.utils.BigIntegerOps;
import org.enso.interpreter.node.expression.builtin.ordering.CustomComparatorNode; import org.enso.interpreter.node.expression.builtin.ordering.CustomComparatorNode;
import org.enso.interpreter.node.expression.builtin.ordering.CustomComparatorNodeGen; import org.enso.interpreter.node.expression.builtin.ordering.CustomComparatorNodeGen;
import org.enso.interpreter.node.expression.builtin.ordering.HashCallbackNode; import org.enso.interpreter.node.expression.builtin.ordering.HashCallbackNode;
@ -89,70 +87,31 @@ public abstract class HashCodeNode extends Node {
public abstract long execute(@AcceptsError Object object); public abstract long execute(@AcceptsError Object object);
/** Specializations for primitive values * */
@Specialization
long hashCodeForShort(short s) {
return s;
}
@Specialization
long hashCodeForByte(byte b) {
return b;
}
@Specialization @Specialization
long hashCodeForLong(long l) { long hashCodeForLong(long l) {
// By casting long to double, we lose some precision on purpose // By casting long to double, we lose some precision on purpose
return hashCodeForDouble((double) l); return hashCodeForDouble((double) l);
} }
@Specialization
long hashCodeForInt(int i) {
return hashCodeForLong(i);
}
@Specialization
long hashCodeForFloat(float f) {
return Float.hashCode(f);
}
@Specialization @Specialization
long hashCodeForDouble(double d) { long hashCodeForDouble(double d) {
if (Double.isNaN(d)) { return Double.hashCode(d);
// NaN is Incomparable, just return a "random" constant
return 456879;
} else if (d % 1.0 != 0 || BigIntegerOps.fitsInLong(d)) {
return Double.hashCode(d);
} else {
return bigDoubleHash(d);
}
}
@TruffleBoundary
private static long bigDoubleHash(double d) {
try {
return BigDecimal.valueOf(d).toBigIntegerExact().hashCode();
} catch (ArithmeticException e) {
throw EnsoContext.get(null).raiseAssertionPanic(null, null, e);
}
} }
@Specialization @Specialization
@TruffleBoundary @TruffleBoundary
long hashCodeForBigInteger(EnsoBigInteger bigInteger) { long hashCodeForBigInteger(
return bigInteger.getValue().hashCode(); EnsoBigInteger bigInteger,
@Shared("interop") @CachedLibrary(limit = "10") InteropLibrary interop) {
return hashCodeForDouble(bigInteger.getValue().doubleValue());
} }
@Specialization(guards = {"interop.fitsInLong(v) || interop.fitsInBigInteger(v)"}) @Specialization(guards = {"interop.fitsInBigInteger(v)"})
@TruffleBoundary @TruffleBoundary
long hashCodeForBigInteger( long hashCodeForBigInteger(
Object v, @Shared("interop") @CachedLibrary(limit = "10") InteropLibrary interop) { Object v, @Shared("interop") @CachedLibrary(limit = "10") InteropLibrary interop) {
try { try {
if (interop.fitsInLong(v)) { return hashCodeForDouble(interop.asBigInteger(v).doubleValue());
return hashCodeForLong(interop.asLong(v));
} else {
return interop.asBigInteger(v).hashCode();
}
} catch (UnsupportedMessageException ex) { } catch (UnsupportedMessageException ex) {
throw EnsoContext.get(this).raiseAssertionPanic(this, "Expecting BigInteger", ex); throw EnsoContext.get(this).raiseAssertionPanic(this, "Expecting BigInteger", ex);
} }

View File

@ -238,6 +238,15 @@ public class BigIntegerOps {
return decimal <= Long.MAX_VALUE && decimal >= Long.MIN_VALUE; return decimal <= Long.MAX_VALUE && decimal >= Long.MIN_VALUE;
} }
private static boolean fitsInLongCompat(double decimal) {
var nulaMinus = Double.doubleToRawLongBits(-0d);
if (nulaMinus == Double.doubleToRawLongBits(decimal)) {
return false;
}
var converted = (long) decimal;
return converted != Long.MAX_VALUE && converted == decimal;
}
public static boolean fitsInInt(long number) { public static boolean fitsInInt(long number) {
return number >= Integer.MIN_VALUE && number <= Integer.MAX_VALUE; return number >= Integer.MIN_VALUE && number <= Integer.MAX_VALUE;
} }

View File

@ -26,7 +26,7 @@ public final class EnsoBigInteger implements EnsoObject {
* @param value the value to wrap. * @param value the value to wrap.
*/ */
public EnsoBigInteger(BigInteger value) { public EnsoBigInteger(BigInteger value) {
assert (value.bitLength() > 63); assert (value.bitLength() > 63) : "Too small BigInteger: " + value;
this.value = value; this.value = value;
} }

View File

@ -52,13 +52,17 @@ public class Decimal_Utils {
} }
} }
public static BigDecimal fromFloat(Double d) { public static BigDecimal fromFloat(double d) {
// According to the BigInteger Javadocs, valueOf is preferred because "the // According to the BigInteger Javadocs, valueOf is preferred because "the
// value returned is equal to that resulting from constructing a BigDecimal // value returned is equal to that resulting from constructing a BigDecimal
// from the result of using Double.toString(double)." // from the result of using Double.toString(double)."
return BigDecimal.valueOf(d); return BigDecimal.valueOf(d);
} }
public static BigDecimal fromFloatExact(double d) {
return new BigDecimal(d);
}
public static ConversionResult fromFloat(Double d, MathContext mc) { public static ConversionResult fromFloat(Double d, MathContext mc) {
// We do not check for precision loss here because we always attach a // We do not check for precision loss here because we always attach a
// warning when converting from float. // warning when converting from float.
@ -71,26 +75,7 @@ public class Decimal_Utils {
} }
public static int hashCodeOf(BigDecimal bd) { public static int hashCodeOf(BigDecimal bd) {
boolean isFractional = bd.remainder(BigDecimal.ONE).compareTo(BigDecimal.ZERO) != 0; return Double.hashCode(bd.doubleValue());
boolean fitsInLong = fitsInLong(bd);
if (isFractional || fitsInLong) {
double d = bd.doubleValue();
var fitsInDouble = Double.isFinite(d);
if (fitsInDouble) {
return Double.hashCode(d);
} else {
// Infinite values here just means finite values outside the double
// range. In this path, the values must be fractional, and so cannot
// coincide with a value of any other type (including BigInteger), so we
// can hash it however we want.
assert isFractional;
return bd.hashCode();
}
} else {
// Will not throw ArithmeticException since the value has a 0 fractional part.
assert !isFractional;
return bd.toBigIntegerExact().hashCode();
}
} }
public static BigDecimal round(BigDecimal bd, int decimalPlaces, boolean useBankers) { public static BigDecimal round(BigDecimal bd, int decimalPlaces, boolean useBankers) {

View File

@ -301,8 +301,8 @@ add_specs suite_builder =
values.map pr-> values.map pr->
v = pr.at 0 v = pr.at 0
d = Decimal.new v d = Decimal.new v . to_float
expected = pr.at 1 expected = pr.at 1 . to_float
d . should_equal expected d . should_equal expected
expected . should_equal d expected . should_equal d
@ -367,13 +367,23 @@ add_specs suite_builder =
suite_builder.group "edge cases" group_builder-> suite_builder.group "edge cases" group_builder->
group_builder.specify "can support values outside the double range" <| group_builder.specify "can support values outside the double range" <|
d = Decimal.new Float.max_value d = Decimal.new Float.max_value
(d == Float.max_value) . should_be_false
((d * d) == Float.max_value) . should_be_false
((-(d * d)) == -Float.max_value) . should_be_false
Comparable.hash_builtin (d * d) . should_equal 2146435072
Comparable.hash_builtin -(d * d) . should_equal -1048576
Comparable.hash_builtin ((d * d) + 0.1) . should_equal 2146435072
Comparable.hash_builtin ((-(d * d)) + 0.1) . should_equal -1048576
group_builder.specify "from conversion supports values outside the double range" <|
d = Float.max_value.to Decimal
(d == Float.max_value) . should_be_true (d == Float.max_value) . should_be_true
((d * d) == Float.max_value) . should_be_false ((d * d) == Float.max_value) . should_be_false
((-(d * d)) == -Float.max_value) . should_be_false ((-(d * d)) == -Float.max_value) . should_be_false
Comparable.hash_builtin (d * d) . should_equal -1851772048 Comparable.hash_builtin (d * d) . should_equal 2146435072
Comparable.hash_builtin -(d * d) . should_equal 1851772048 Comparable.hash_builtin -(d * d) . should_equal -1048576
Comparable.hash_builtin ((d * d) + 0.1) . should_equal -1176480326 Comparable.hash_builtin ((d * d) + 0.1) . should_equal 2146435072
Comparable.hash_builtin ((-(d * d)) + 0.1) . should_equal -534461530 Comparable.hash_builtin ((-(d * d)) + 0.1) . should_equal -1048576
suite_builder.group "arithmetic" group_builder-> suite_builder.group "arithmetic" group_builder->
group_builder.specify "should allow arithmetic with Decimals, without Math_Context" <| group_builder.specify "should allow arithmetic with Decimals, without Math_Context" <|
@ -577,12 +587,12 @@ add_specs suite_builder =
f.should_be_a Float f.should_be_a Float
f.should_equal 56.34 f.should_equal 56.34
group_builder.specify "Decimal.to_float should compare correctly with the original Decimal" <| group_builder.specify "Decimal.to_float cannot compare correctly with the original Decimal" <|
huge_a = Decimal.new "3.4E300" huge_a = Decimal.new "3.4E300"
huge_a_float = Float.from huge_a huge_a_float = Float.from huge_a
(huge_a_float == huge_a) . should_be_true (huge_a_float == huge_a) . should_be_false
(huge_a == huge_a_float) . should_be_true (huge_a == huge_a_float) . should_be_false
group_builder.specify "Decimal.to_integer should compare correctly with the original Decimal" <| group_builder.specify "Decimal.to_integer should compare correctly with the original Decimal" <|
huge_a = Decimal.new "3.4E320" huge_a = Decimal.new "3.4E320"
@ -604,14 +614,15 @@ add_specs suite_builder =
cases = [] cases = []
+ [[5, 3, 2], [5.0, 3.0, 2.0], [3.5, 2, 1.5], [10.5, 1.0, 0.5], [3, 1, 0], [3.0, 1.0, 0]] + [[5, 3, 2], [5.0, 3.0, 2.0], [3.5, 2, 1.5], [10.5, 1.0, 0.5], [3, 1, 0], [3.0, 1.0, 0]]
+ [[-5, 3, -2], [-5.0, 3.0, -2.0], [-3.5, 2, -1.5], [-10.5, 1.0, -0.5], [-3, 1, 0], [-3.0, 1.0, 0]] + [[-5, 3, -2], [-5.0, 3.0, -2.0], [-3.5, 2, -1.5], [-10.5, 1.0, -0.5], [-3, 1, 0], [-3.0, 1.0, 0]]
+ [[2.7, 0.5, 0.2], [2.7, 0.4, 0.3], [-2.7, 0.5, -0.2], [-2.7, 0.4, -0.3]] + [[2.7, 0.5, 0.2], [2.7, 0.4, 0.3, 0.001], [-2.7, 0.5, -0.2], [-2.7, 0.4, -0.3, 0.001]]
+ [["9223372036854775807", 10, 7], ["9223372036854775808", 10, 8], ["922337203685477580000000000008", 10, 8]] + [["9223372036854775807", 10, 7], ["9223372036854775808", 10, 8], ["922337203685477580000000000008", 10, 8]]
+ [["-9223372036854775808", 10, -8], ["-9223372036854775809", 10, -9], ["-922337203685477580000000000008", 10, -8]] + [["-9223372036854775808", 10, -8], ["-9223372036854775809", 10, -9], ["-922337203685477580000000000008", 10, -8]]
cases.map c-> cases.map c-> Test.with_clue ("["+c.to_text+"] ") <|
base = c.at 0 base = c.at 0
modulus = c.at 1 modulus = c.at 1
residue = c.at 2 residue = c.at 2
(Decimal.new base % modulus) . should_equal residue epsilon = if c.length > 3 then c.at 3 else 0.0
(Decimal.new base % modulus) . should_equal epsilon=epsilon residue
(Decimal.new base % Decimal.new modulus) . should_equal residue (Decimal.new base % Decimal.new modulus) . should_equal residue
group_builder.specify "should define remainder (example)" <| group_builder.specify "should define remainder (example)" <|
@ -734,7 +745,10 @@ add_specs suite_builder =
Decimal.new "0" . signum . should_equal 0 Decimal.new "0" . signum . should_equal 0
suite_builder.group "rounding" group_builder-> suite_builder.group "rounding" group_builder->
do_round n dp=0 use_bankers=False = Decimal.new n . round dp use_bankers do_round n dp=0 use_bankers=False =
d = Decimal.new n . round dp use_bankers
d.to_float
Round_Spec.add_specs group_builder do_round Round_Spec.add_specs group_builder do_round
group_builder.specify "Large values" <| group_builder.specify "Large values" <|
@ -916,7 +930,7 @@ add_specs suite_builder =
12 . min (Decimal.new "13") . should_equal (Decimal.new "12") 12 . min (Decimal.new "13") . should_equal (Decimal.new "12")
Decimal.new "12.1" . min 12.3 . should_equal (Decimal.new "12.1") Decimal.new "12.1" . min 12.3 . should_equal (Decimal.new "12.1")
12.1 . min (Decimal.new "12.3") . should_equal (Decimal.new "12.1") 12.1 . min (Decimal.new "12.3") . should_equal (Decimal.from 12.1)
Decimal.new "0" . min (Decimal.new "-1") . should_equal (Decimal.new "-1") Decimal.new "0" . min (Decimal.new "-1") . should_equal (Decimal.new "-1")
0 . min (Decimal.new "-1") . should_equal (Decimal.new "-1") 0 . min (Decimal.new "-1") . should_equal (Decimal.new "-1")

View File

@ -1072,7 +1072,7 @@ add_specs suite_builder setup =
decimal_col.value_type.is_decimal . should_be_true decimal_col.value_type.is_decimal . should_be_true
decimal_col2 = decimal_col + decimal_col*decimal_col decimal_col2 = decimal_col + decimal_col*decimal_col
[(.floor), (.ceil), (.truncate), (x-> x.round 0), (x-> x.round 2)].each op-> [(.floor), (.ceil), (.truncate), (x-> x.round 0), (x-> x.round 2)].each op->
op decimal_col2 . to_vector . should_equal [i1 + i1*i1] op decimal_col2 . to_vector . should_equal [i1 + i1*i1 . to_float]
group_builder.specify "should allow Nothing/NULL" <| group_builder.specify "should allow Nothing/NULL" <|
table = table_builder [["x", [Nothing, 0.51, 0.59, 3.51, Nothing, 3.59, -0.51, -0.59, -3.51, -3.59]]] table = table_builder [["x", [Nothing, 0.51, 0.59, 3.51, Nothing, 3.59, -0.51, -0.59, -3.51, -3.59]]]

View File

@ -384,8 +384,7 @@ run_union_tests group_builder setup call_union =
t4 = call_union [t2, t1, t3] allow_type_widening=True t4 = call_union [t2, t1, t3] allow_type_widening=True
t4.at "X" . value_type . should_equal Value_Type.Float t4.at "X" . value_type . should_equal Value_Type.Float
# Inexact float equality will make this pass: t4.at "X" . to_vector . should_equal [1.5, 2.5, 3.5, 1, (2^62)-1, 3, (2^100)+1 . to_float, 2^10, 2]
t4.at "X" . to_vector . should_equal [1.5, 2.5, 3.5, 1, (2^62)-1, 3, (2^100)+1, 2^10, 2]
w = Problems.expect_only_warning Loss_Of_Integer_Precision t4 w = Problems.expect_only_warning Loss_Of_Integer_Precision t4
# Losing precision on (2^62)-1 and 2^100+1. # Losing precision on (2^62)-1 and 2^100+1.

View File

@ -402,7 +402,7 @@ postgres_specific_spec suite_builder create_connection_fn db_name setup =
t2.at "Y" . value_type . scale . should_equal Nothing t2.at "Y" . value_type . scale . should_equal Nothing
t2.at "X" . to_vector . should_equal [10, x] t2.at "X" . to_vector . should_equal [10, x]
# Only works by approximation: # Only works by approximation:
t2.at "Y" . to_vector . should_equal [20, x+10] t2.at "Y" . to_vector . should_equal [20, x+10 . to_float]
t2.at "Y" . cast Value_Type.Char . to_vector . should_equal ["20", (x+10).to_text] t2.at "Y" . cast Value_Type.Char . to_vector . should_equal ["20", (x+10).to_text]
m2 = t2.remove_warnings.read m2 = t2.remove_warnings.read
@ -410,7 +410,7 @@ postgres_specific_spec suite_builder create_connection_fn db_name setup =
# As noted above - once operations are performed, the scale=0 may be lost and the column will be approximated as a float. # As noted above - once operations are performed, the scale=0 may be lost and the column will be approximated as a float.
m2.at "Y" . value_type . should_equal Value_Type.Float m2.at "Y" . value_type . should_equal Value_Type.Float
m2.at "X" . to_vector . should_equal [10, x] m2.at "X" . to_vector . should_equal [10, x]
m2.at "Y" . to_vector . should_equal [20, x+10] m2.at "Y" . to_vector . should_equal [20, x+10 . to_float]
w2 = Problems.expect_only_warning Inexact_Type_Coercion m2 w2 = Problems.expect_only_warning Inexact_Type_Coercion m2
w2.requested_type . should_equal (Value_Type.Decimal precision=Nothing scale=Nothing) w2.requested_type . should_equal (Value_Type.Decimal precision=Nothing scale=Nothing)
w2.actual_type . should_equal Value_Type.Float w2.actual_type . should_equal Value_Type.Float
@ -425,7 +425,7 @@ postgres_specific_spec suite_builder create_connection_fn db_name setup =
t3 . at "X" . value_type . precision . should_equal Nothing t3 . at "X" . value_type . precision . should_equal Nothing
t3 . at "X" . value_type . scale . should_equal Nothing t3 . at "X" . value_type . scale . should_equal Nothing
# Works but only relying on imprecise float equality: # Works but only relying on imprecise float equality:
t3 . at "X" . to_vector . should_equal [super_large] t3 . at "X" . to_vector . should_equal [super_large . to_float]
w3 = Problems.expect_only_warning Inexact_Type_Coercion t3 w3 = Problems.expect_only_warning Inexact_Type_Coercion t3
w3.requested_type . should_equal (Value_Type.Decimal precision=Nothing scale=0) w3.requested_type . should_equal (Value_Type.Decimal precision=Nothing scale=0)
w3.actual_type . should_equal (Value_Type.Decimal precision=Nothing scale=Nothing) w3.actual_type . should_equal (Value_Type.Decimal precision=Nothing scale=Nothing)
@ -433,7 +433,7 @@ postgres_specific_spec suite_builder create_connection_fn db_name setup =
m4 = t3.remove_warnings.read m4 = t3.remove_warnings.read
# Because we no longer have a set scale, we cannot get a BigInteger column back - we'd need BigDecimal, but that is not fully supported yet in Enso - so we get the closest approximation - the imprecise Float. # Because we no longer have a set scale, we cannot get a BigInteger column back - we'd need BigDecimal, but that is not fully supported yet in Enso - so we get the closest approximation - the imprecise Float.
m4 . at "X" . value_type . should_equal Value_Type.Float m4 . at "X" . value_type . should_equal Value_Type.Float
m4 . at "X" . to_vector . should_equal [super_large] m4 . at "X" . to_vector . should_equal [super_large . to_float]
w4 = Problems.expect_only_warning Inexact_Type_Coercion m4 w4 = Problems.expect_only_warning Inexact_Type_Coercion m4
w4.requested_type . should_equal (Value_Type.Decimal precision=Nothing scale=Nothing) w4.requested_type . should_equal (Value_Type.Decimal precision=Nothing scale=Nothing)
w4.actual_type . should_equal Value_Type.Float w4.actual_type . should_equal Value_Type.Float

View File

@ -261,7 +261,7 @@ add_specs suite_builder =
Problems.assume_no_problems c2 Problems.assume_no_problems c2
c2.value_type . should_equal Value_Type.Float c2.value_type . should_equal Value_Type.Float
# This will be an inexact equality, based on float rounding. # This will be an inexact equality, based on float rounding.
c2.to_vector . should_equal [x, 1.5] c2.to_vector . map .to_float . should_equal [x . to_float, 1.5]
c3 = Column.from_vector "X" [2^70, 1] c3 = Column.from_vector "X" [2^70, 1]
Problems.assume_no_problems c3 Problems.assume_no_problems c3
@ -346,8 +346,8 @@ add_specs suite_builder =
(x * y) . to_vector . should_equal [10^30, 2^71, Nothing, 3 * 4] (x * y) . to_vector . should_equal [10^30, 2^71, Nothing, 3 * 4]
(x % y) . to_vector . should_equal [0, 0, Nothing, 3] (x % y) . to_vector . should_equal [0, 0, Nothing, 3]
(x % z) . to_vector . should_equal [1.0, 1.5, Nothing, 3.0] (x % z) . to_vector . should_equal [1.0, 1.5, Nothing, 3.0]
(x / y) . to_vector . should_equal [10^30, 2^69, Nothing, 0.75] (x / y) . to_vector . should_equal [10^30 . to_float, 2^69 . to_float, Nothing, 0.75]
(x ^ y) . to_vector . should_equal [10^30, 2^140, Nothing, 3^4] (x ^ y) . to_vector . should_equal [10^30 . to_float, 2^140 . to_float, Nothing, 3^4]
(x == y) . to_vector . should_equal [False, False, Nothing, False] (x == y) . to_vector . should_equal [False, False, Nothing, False]
(x < y) . to_vector . should_equal [False, False, Nothing, True] (x < y) . to_vector . should_equal [False, False, Nothing, True]
(x >= y) . to_vector . should_equal [True, True, Nothing, False] (x >= y) . to_vector . should_equal [True, True, Nothing, False]
@ -357,7 +357,7 @@ add_specs suite_builder =
(x.min y) . to_vector . should_equal [1, 2, 3, 3] (x.min y) . to_vector . should_equal [1, 2, 3, 3]
(x.max y) . to_vector . should_equal [10^30, 2^70, 3, 4] (x.max y) . to_vector . should_equal [10^30, 2^70, 3, 4]
(x.min z) . to_vector . should_equal [1.5, 2.5, 3.5, 3] (x.min z) . to_vector . should_equal [1.5, 2.5, 3.5, 3]
(x.max [y, z]) . to_vector . should_equal [10^30, 2^70, 3.5, 4.5] (x.max [y, z]) . to_vector . should_equal [10^30 . to_float, 2^70 . to_float, 3.5, 4.5]
(y * x) . to_vector . should_equal [10^30, 2^71, Nothing, 3 * 4] (y * x) . to_vector . should_equal [10^30, 2^71, Nothing, 3 * 4]
(y % x) . to_vector . should_equal [1, 2, Nothing, 1] (y % x) . to_vector . should_equal [1, 2, Nothing, 1]
@ -372,22 +372,22 @@ add_specs suite_builder =
(y.min x) . to_vector . should_equal [1, 2, 3, 3] (y.min x) . to_vector . should_equal [1, 2, 3, 3]
(y.max x) . to_vector . should_equal [10^30, 2^70, 3, 4] (y.max x) . to_vector . should_equal [10^30, 2^70, 3, 4]
(z.min x) . to_vector . should_equal [1.5, 2.5, 3.5, 3] (z.min x) . to_vector . should_equal [1.5, 2.5, 3.5, 3]
(z.max x) . to_vector . should_equal [10^30, 2^70, 3.5, 4.5] (z.max x) . to_vector . should_equal [10^30 . to_float, 2^70 . to_float, 3.5, 4.5]
x.fill_nothing y . to_vector . should_equal [10^30, 2^70, 3, 3] x.fill_nothing y . to_vector . should_equal [10^30, 2^70, 3, 3]
x.fill_nothing z . to_vector . should_equal [10^30, 2^70, 3.5, 3] x.fill_nothing z . to_vector . should_equal [10^30 . to_float, 2^70 . to_float, 3.5, 3]
y.fill_nothing x . to_vector . should_equal [1, 2, 3, 4] y.fill_nothing x . to_vector . should_equal [1, 2, 3, 4]
y2 = (y < 2).iif Nothing y y2 = (y < 2).iif Nothing y
y2.to_vector . should_equal [Nothing, 2, 3, 4] y2.to_vector . should_equal [Nothing, 2, 3, 4]
y2.fill_nothing x . to_vector . should_equal [10^30, 2, 3, 4] y2.fill_nothing x . to_vector . should_equal [10^30, 2, 3, 4]
((z < 2).iif Nothing z).fill_nothing x . to_vector . should_equal [10^30, 2.5, 3.5, 4.5] ((z < 2).iif Nothing z).fill_nothing x . to_vector . should_equal [10^30 . to_float, 2.5, 3.5, 4.5]
r3 = x.fill_nothing 1.5 r3 = x.fill_nothing 1.5
r3.value_type . should_equal Value_Type.Float r3.value_type . should_equal Value_Type.Float
# 2^70 is not exactly representable as a Float. # 2^70 is not exactly representable as a Float.
(2^70 + 0.0).truncate . should_not_equal (2^70) (2^70 + 0.0).truncate . should_not_equal (2^70)
Problems.expect_only_warning Loss_Of_Integer_Precision r3 Problems.expect_only_warning Loss_Of_Integer_Precision r3
r3.to_vector . should_equal [10^30, 2^70, 1.5, 3] r3.to_vector . should_equal [10^30 . to_float, 2^70 . to_float, 1.5, 3]
r4 = x.fill_nothing 23 r4 = x.fill_nothing 23
r4.value_type . should_be_a (Value_Type.Decimal ...) r4.value_type . should_be_a (Value_Type.Decimal ...)