mirror of
https://github.com/enso-org/enso.git
synced 2024-12-23 13:02:07 +03:00
Handle some edge cases in rounding (inexact representations and overflows) (#6922)
This commit is contained in:
GregoryTravis
GitHub
parent
72b202b7d0
commit
3ffbe9cecf
@ -476,6 +476,7 @@
|
||||
- [Added `.round`, `.truncate`, `.ceil`, and `.floor` to `Column`.][6817]
|
||||
- [Added execution control to `Table.write` and various bug fixes.][6835]
|
||||
- [Implemented `Table.add_row_number`.][6890]
|
||||
- [Handling edge cases in rounding.][6922]
|
||||
|
||||
[debug-shortcuts]:
|
||||
https://github.com/enso-org/enso/blob/develop/app/gui/docs/product/shortcuts.md#debug
|
||||
@ -691,6 +692,7 @@
|
||||
[6817]: https://github.com/enso-org/enso/pull/6817
|
||||
[6835]: https://github.com/enso-org/enso/pull/6835
|
||||
[6890]: https://github.com/enso-org/enso/pull/6890
|
||||
[6922]: https://github.com/enso-org/enso/pull/6922
|
||||
|
||||
#### Enso Compiler
|
||||
|
||||
|
||||
@ -615,23 +615,16 @@ type Decimal
|
||||
msg = "round cannot accept " + self.to_text
|
||||
Error.throw (Arithmetic_Error.Error msg)
|
||||
False -> check_round_input self <|
|
||||
decimal_result = case use_bankers of
|
||||
False ->
|
||||
scale = 10 ^ decimal_places
|
||||
((self * scale) + 0.5).floor / scale
|
||||
True ->
|
||||
## If the largest integer <= self is odd, use normal
|
||||
round-towards-positive-infinity rounding; otherwise,
|
||||
use round-towards-negative-infinity rounding.
|
||||
scale = 10 ^ decimal_places
|
||||
scaled_self = self * scale
|
||||
self_scaled_floor = scaled_self.floor
|
||||
is_even = (self_scaled_floor % 2) == 0
|
||||
case is_even of
|
||||
False ->
|
||||
(scaled_self + 0.5).floor / scale
|
||||
True ->
|
||||
(scaled_self - 0.5).ceil / scale
|
||||
decimal_result =
|
||||
# Algorithm taken from https://stackoverflow.com/a/7211688
|
||||
scale = 10 ^ decimal_places
|
||||
scaled = self * scale
|
||||
round_base = scaled.floor
|
||||
round_midpoint = (round_base + 0.5) / scale
|
||||
even_is_up = if self >= 0 then (scaled.truncate % 2) != 0 else (scaled.truncate % 2) == 0
|
||||
half_goes_up = if use_bankers then even_is_up else True
|
||||
do_round_up = if half_goes_up then self >= round_midpoint else self > round_midpoint
|
||||
if do_round_up then ((round_base + 1.0) / scale) else (round_base / scale)
|
||||
# Convert to integer if it's really an integer anyway.
|
||||
if decimal_places > 0 then decimal_result else decimal_result.truncate
|
||||
|
||||
@ -940,11 +933,24 @@ type Integer
|
||||
12250 . round -2 use_bankers=True == 12200
|
||||
round : Integer -> Boolean -> Integer ! Illegal_Argument
|
||||
round self decimal_places=0 use_bankers=False =
|
||||
check_decimal_places decimal_places <| check_round_input self <|
|
||||
## It's already an integer so unless decimal_places is
|
||||
negative, the value is unchanged.
|
||||
if decimal_places >= 0 then self else
|
||||
self.to_decimal.round decimal_places use_bankers . truncate
|
||||
## It's already an integer so unless decimal_places is
|
||||
negative, the value is unchanged.
|
||||
if decimal_places >= 0 then self else
|
||||
check_decimal_places decimal_places <| check_round_input self <|
|
||||
scale = 10 ^ -decimal_places
|
||||
halfway = scale.div 2
|
||||
remainder = self % scale
|
||||
scaled_down = self.div scale
|
||||
result_unnudged = scaled_down * scale
|
||||
case self >= 0 of
|
||||
True ->
|
||||
half_goes_up = if use_bankers then (scaled_down % 2) != 0 else True
|
||||
round_up = if half_goes_up then remainder >= halfway else remainder > halfway
|
||||
if round_up then result_unnudged + scale else result_unnudged
|
||||
False ->
|
||||
half_goes_up = if use_bankers then (scaled_down % 2) == 0 else True
|
||||
round_up = if half_goes_up then remainder < -halfway else remainder <= -halfway
|
||||
if round_up then result_unnudged - scale else result_unnudged
|
||||
|
||||
## Compute the negation of this.
|
||||
|
||||
|
||||
@ -472,6 +472,7 @@ spec =
|
||||
Number.nan . equals 0 . should_fail_with Incomparable_Values
|
||||
|
||||
Test.group "Decimal.round" <|
|
||||
|
||||
Test.specify "Can round positive decimals correctly" <|
|
||||
3.0 . round . should_equal 3
|
||||
3.00001 . round . should_equal 3
|
||||
@ -585,6 +586,64 @@ spec =
|
||||
231.2 . round . should_be_a Integer
|
||||
231.2 . round -1 . should_be_a Integer
|
||||
|
||||
Test.specify "Can round correctly near the precision limit" <|
|
||||
1.22222222225 . round 10 . should_equal 1.2222222223
|
||||
1.222222222225 . round 11 . should_equal 1.22222222223
|
||||
1.2222222222225 . round 12 . should_equal 1.222222222223
|
||||
1.22222222222225 . round 13 . should_equal 1.2222222222223
|
||||
1.222222222222225 . round 14 . should_equal 1.22222222222223
|
||||
1.2222222222222225 . round 15 . should_equal 1.222222222222223
|
||||
|
||||
-1.22222222225 . round 10 . should_equal -1.2222222222
|
||||
-1.222222222225 . round 11 . should_equal -1.22222222222
|
||||
-1.2222222222225 . round 12 . should_equal -1.222222222222
|
||||
-1.22222222222225 . round 13 . should_equal -1.2222222222222
|
||||
-1.222222222222225 . round 14 . should_equal -1.22222222222222
|
||||
-1.2222222222222225 . round 15 . should_equal -1.222222222222222
|
||||
|
||||
1.22222222235 . round 10 . should_equal 1.2222222224
|
||||
1.222222222235 . round 11 . should_equal 1.22222222224
|
||||
1.2222222222235 . round 12 . should_equal 1.222222222224
|
||||
1.22222222222235 . round 13 . should_equal 1.2222222222224
|
||||
1.222222222222235 . round 14 . should_equal 1.22222222222224
|
||||
1.2222222222222235 . round 15 . should_equal 1.222222222222224
|
||||
|
||||
-1.22222222235 . round 10 . should_equal -1.2222222223
|
||||
-1.222222222235 . round 11 . should_equal -1.22222222223
|
||||
-1.2222222222235 . round 12 . should_equal -1.222222222223
|
||||
-1.22222222222235 . round 13 . should_equal -1.2222222222223
|
||||
-1.222222222222235 . round 14 . should_equal -1.22222222222223
|
||||
-1.2222222222222235 . round 15 . should_equal -1.222222222222223
|
||||
|
||||
Test.specify "Can round correctly near the precision limit, using banker's rounding" <|
|
||||
1.22222222225 . round 10 use_bankers=True . should_equal 1.2222222222
|
||||
1.222222222225 . round 11 use_bankers=True . should_equal 1.22222222222
|
||||
1.2222222222225 . round 12 use_bankers=True . should_equal 1.222222222222
|
||||
1.22222222222225 . round 13 use_bankers=True . should_equal 1.2222222222222
|
||||
1.222222222222225 . round 14 use_bankers=True . should_equal 1.22222222222222
|
||||
1.2222222222222225 . round 15 use_bankers=True . should_equal 1.222222222222222
|
||||
|
||||
-1.22222222225 . round 10 use_bankers=True . should_equal -1.2222222222
|
||||
-1.222222222225 . round 11 use_bankers=True . should_equal -1.22222222222
|
||||
-1.2222222222225 . round 12 use_bankers=True . should_equal -1.222222222222
|
||||
-1.22222222222225 . round 13 use_bankers=True . should_equal -1.2222222222222
|
||||
-1.222222222222225 . round 14 use_bankers=True . should_equal -1.22222222222222
|
||||
-1.2222222222222225 . round 15 use_bankers=True . should_equal -1.222222222222222
|
||||
|
||||
1.22222222235 . round 10 use_bankers=True . should_equal 1.2222222224
|
||||
1.222222222235 . round 11 use_bankers=True . should_equal 1.22222222224
|
||||
1.2222222222235 . round 12 use_bankers=True . should_equal 1.222222222224
|
||||
1.22222222222235 . round 13 use_bankers=True . should_equal 1.2222222222224
|
||||
1.222222222222235 . round 14 use_bankers=True . should_equal 1.22222222222224
|
||||
1.2222222222222235 . round 15 use_bankers=True . should_equal 1.222222222222224
|
||||
|
||||
-1.22222222235 . round 10 use_bankers=True . should_equal -1.2222222224
|
||||
-1.222222222235 . round 11 use_bankers=True . should_equal -1.22222222224
|
||||
-1.2222222222235 . round 12 use_bankers=True . should_equal -1.222222222224
|
||||
-1.22222222222235 . round 13 use_bankers=True . should_equal -1.2222222222224
|
||||
-1.222222222222235 . round 14 use_bankers=True . should_equal -1.22222222222224
|
||||
-1.2222222222222235 . round 15 use_bankers=True . should_equal -1.222222222222224
|
||||
|
||||
Test.specify "Input out of range" <|
|
||||
100000000000000.0 . round . should_fail_with Illegal_Argument
|
||||
-100000000000000.0 . round . should_fail_with Illegal_Argument
|
||||
@ -602,10 +661,12 @@ spec =
|
||||
Number.positive_infinity . round . should_fail_with Arithmetic_Error
|
||||
Number.negative_infinity . round . should_fail_with Arithmetic_Error
|
||||
|
||||
Test.specify "Banker's rounding failure" pending="Fails because of basic floating-point inaccuracy when adding 0.5" <|
|
||||
Test.specify "Floating point imperfect representation counter-examples" <|
|
||||
1.225 . round 2 use_bankers=True . should_equal 1.22 # Actual result 1.23
|
||||
37.785 . round 2 . should_equal 37.79
|
||||
|
||||
Test.group "Integer.round" <|
|
||||
|
||||
Test.specify "Can round small integers to a specified number of decimal places correctly (value is unchanged)"
|
||||
0 . round . should_equal 0
|
||||
3 . round . should_equal 3
|
||||
@ -615,76 +676,110 @@ spec =
|
||||
3 . round 1 . should_equal 3
|
||||
-3 . round 1 . should_equal -3
|
||||
|
||||
Test.specify "Can round small integers to a specified number of negative places correctly"
|
||||
Test.specify "Can round integers to a specified number of negative places correctly"
|
||||
0 . round -1 . should_equal 0
|
||||
4 . round -1 . should_equal 0
|
||||
5 . round -1 . should_equal 10
|
||||
6 . round -1 . should_equal 10
|
||||
9 . round -1 . should_equal 10
|
||||
10 . round -1 . should_equal 10
|
||||
11 . round -1 . should_equal 10
|
||||
24 . round -1 . should_equal 20
|
||||
25 . round -1 . should_equal 30
|
||||
29 . round -1 . should_equal 30
|
||||
30 . round -1 . should_equal 30
|
||||
31 . round -1 . should_equal 30
|
||||
|
||||
2000 . round -3 . should_equal 2000
|
||||
2001 . round -3 . should_equal 2000
|
||||
2412 . round -3 . should_equal 2000
|
||||
2499 . round -3 . should_equal 2000
|
||||
2500 . round -3 . should_equal 3000
|
||||
2501 . round -3 . should_equal 3000
|
||||
2511 . round -3 . should_equal 3000
|
||||
2907 . round -3 . should_equal 3000
|
||||
2999 . round -3 . should_equal 3000
|
||||
3000 . round -3 . should_equal 3000
|
||||
3001 . round -3 . should_equal 3000
|
||||
3098 . round -3 . should_equal 3000
|
||||
3101 . round -3 . should_equal 3000
|
||||
|
||||
Test.specify "Can round negative small integers to a specified number of negative places correctly"
|
||||
Test.specify "Can round negative integers to a specified number of negative places correctly"
|
||||
-4 . round -1 . should_equal 0
|
||||
-5 . round -1 . should_equal 0
|
||||
-6 . round -1 . should_equal -10
|
||||
-9 . round -1 . should_equal -10
|
||||
-10 . round -1 . should_equal -10
|
||||
-11 . round -1 . should_equal -10
|
||||
-24 . round -1 . should_equal -20
|
||||
-25 . round -1 . should_equal -20
|
||||
-29 . round -1 . should_equal -30
|
||||
-30 . round -1 . should_equal -30
|
||||
-31 . round -1 . should_equal -30
|
||||
|
||||
-2000 . round -3 . should_equal -2000
|
||||
-2001 . round -3 . should_equal -2000
|
||||
-2412 . round -3 . should_equal -2000
|
||||
-2499 . round -3 . should_equal -2000
|
||||
-2500 . round -3 . should_equal -2000
|
||||
-2501 . round -3 . should_equal -3000
|
||||
-2511 . round -3 . should_equal -3000
|
||||
-2907 . round -3 . should_equal -3000
|
||||
-2999 . round -3 . should_equal -3000
|
||||
-3000 . round -3 . should_equal -3000
|
||||
-3001 . round -3 . should_equal -3000
|
||||
-3098 . round -3 . should_equal -3000
|
||||
-3101 . round -3 . should_equal -3000
|
||||
|
||||
Test.specify "Banker's rounding handles half-way values correctly" <|
|
||||
12350 . round -2 use_bankers=True . should_equal 12400
|
||||
12250 . round -2 use_bankers=True . should_equal 12200
|
||||
-12350 . round -2 use_bankers=True . should_equal -12400
|
||||
-12250 . round -2 use_bankers=True . should_equal -12200
|
||||
|
||||
Test.specify "Banker's rounding handles non-half-way values just like normal rounding" <|
|
||||
Test.specify "Can round negative integers to a specified number of negative places with banker's rounding correctly" <|
|
||||
12300 . round -2 use_bankers=True . should_equal 12300
|
||||
12301 . round -2 use_bankers=True . should_equal 12300
|
||||
12330 . round -2 use_bankers=True . should_equal 12300
|
||||
12349 . round -2 use_bankers=True . should_equal 12300
|
||||
12350 . round -2 use_bankers=True . should_equal 12400
|
||||
12351 . round -2 use_bankers=True . should_equal 12400
|
||||
12370 . round -2 use_bankers=True . should_equal 12400
|
||||
12430 . round -2 use_bankers=True . should_equal 12400
|
||||
12470 . round -2 use_bankers=True . should_equal 12500
|
||||
|
||||
12249 . round -2 use_bankers=True . should_equal 12200
|
||||
12250 . round -2 use_bankers=True . should_equal 12200
|
||||
12251 . round -2 use_bankers=True . should_equal 12300
|
||||
|
||||
-12300 . round -2 use_bankers=True . should_equal -12300
|
||||
-12301 . round -2 use_bankers=True . should_equal -12300
|
||||
-12330 . round -2 use_bankers=True . should_equal -12300
|
||||
-12349 . round -2 use_bankers=True . should_equal -12300
|
||||
-12350 . round -2 use_bankers=True . should_equal -12400
|
||||
-12351 . round -2 use_bankers=True . should_equal -12400
|
||||
-12370 . round -2 use_bankers=True . should_equal -12400
|
||||
-12430 . round -2 use_bankers=True . should_equal -12400
|
||||
-12470 . round -2 use_bankers=True . should_equal -12500
|
||||
|
||||
-12249 . round -2 use_bankers=True . should_equal -12200
|
||||
-12250 . round -2 use_bankers=True . should_equal -12200
|
||||
-12251 . round -2 use_bankers=True . should_equal -12300
|
||||
|
||||
Test.specify "Returns the correct type" <|
|
||||
231 . round 1 . should_be_a Integer
|
||||
231 . round 0 . should_be_a Integer
|
||||
231 . round . should_be_a Integer
|
||||
231 . round -1 . should_be_a Integer
|
||||
|
||||
Test.specify "Decimal places out of range" <|
|
||||
3 . round 16 . should_fail_with Illegal_Argument
|
||||
3 . round -16 . should_fail_with Illegal_Argument
|
||||
|
||||
Test.specify "Input out of range" <|
|
||||
100000000000000 . round . should_fail_with Illegal_Argument
|
||||
-100000000000000 . round . should_fail_with Illegal_Argument
|
||||
100000000000000 . round -2 . should_fail_with Illegal_Argument
|
||||
-100000000000000 . round -2 . should_fail_with Illegal_Argument
|
||||
99999999999999 . round -2 . should_equal 100000000000000
|
||||
-99999999999999 . round -2 . should_equal -100000000000000
|
||||
|
||||
Test.specify "Input out of range is ignored when the implementation returns its argument immediately" <|
|
||||
100000000000000 . round . should_equal 100000000000000
|
||||
-100000000000000 . round . should_equal -100000000000000
|
||||
100000000000000 . round 1 . should_equal 100000000000000
|
||||
-100000000000000 . round 1 . should_equal -100000000000000
|
||||
|
||||
Test.group "Decimal.truncate"
|
||||
|
||||
Test.specify "Correctly converts to Integer" <|
|
||||
0.1.truncate . should_equal 0
|
||||
0.9.truncate . should_equal 0
|
||||
@ -696,6 +791,7 @@ spec =
|
||||
-3.9.truncate . should_equal -3
|
||||
|
||||
Test.group "Integer.truncate"
|
||||
|
||||
Test.specify "Returns its argument" <|
|
||||
0.truncate . should_equal 0
|
||||
3.truncate . should_equal 3
|
||||
|
||||
Loading…
Reference in New Issue
Block a user