AK: Refactor FixedPoint's formatter

The main change is the simplification of the expression
`(10^precision * fraction) / 2^precision` to `5^precision * fraction`.

Those expressions overflow or not depends on the value of `precision`
and `fraction`. For the maximum value of `fraction`, the following table
shows for which value of `precision` overflow will occur.

            Old   New
    u32      08    10
    u64      15    20
    u128     30    39

As of now `u64` type is used to calculate the result of the expression.
Meaning that before, only FixedPoints with `precision` less than 15
could be accurately rendered (for every value of fraction) in decimal.
Now, this limit gets increased to 20.

This refactor also fixes, broken decimal render for explicitly specified
precision width in format string, and broken hexadecimal render.
This commit is contained in:
ronak69 2023-08-13 15:34:00 +00:00 committed by Andrew Kaster
parent d40807681d
commit 352480338e
Notes: sideshowbarker 2024-07-17 05:06:13 +09:00
4 changed files with 147 additions and 53 deletions

View File

@ -420,12 +420,9 @@ struct Formatter<FixedPoint<precision, Underlying>> : StandardFormatter {
{
u8 base;
bool upper_case;
FormatBuilder::RealNumberDisplayMode real_number_display_mode = FormatBuilder::RealNumberDisplayMode::General;
if (m_mode == Mode::Default || m_mode == Mode::FixedPoint) {
base = 10;
upper_case = false;
if (m_mode == Mode::FixedPoint)
real_number_display_mode = FormatBuilder::RealNumberDisplayMode::FixedPoint;
} else if (m_mode == Mode::Hexfloat) {
base = 16;
upper_case = false;
@ -446,7 +443,7 @@ struct Formatter<FixedPoint<precision, Underlying>> : StandardFormatter {
i64 integer = value.ltrunc();
constexpr u64 one = static_cast<Underlying>(1) << precision;
u64 fraction_raw = value.raw() & (one - 1);
return builder.put_fixed_point(is_negative, integer, fraction_raw, one, base, upper_case, m_zero_pad, m_use_separator, m_align, m_width.value(), m_precision.value(), m_fill, m_sign_mode, real_number_display_mode);
return builder.put_fixed_point(is_negative, integer, fraction_raw, one, precision, base, upper_case, m_zero_pad, m_use_separator, m_align, m_width.value(), m_precision.value(), m_fill, m_sign_mode);
}
};

View File

@ -369,16 +369,16 @@ ErrorOr<void> FormatBuilder::put_fixed_point(
i64 integer_value,
u64 fraction_value,
u64 fraction_one,
size_t precision,
u8 base,
bool upper_case,
bool zero_pad,
bool use_separator,
Align align,
size_t min_width,
size_t precision,
size_t fraction_max_width,
char fill,
SignMode sign_mode,
RealNumberDisplayMode display_mode)
SignMode sign_mode)
{
StringBuilder string_builder;
FormatBuilder format_builder { string_builder };
@ -388,55 +388,50 @@ ErrorOr<void> FormatBuilder::put_fixed_point(
TRY(format_builder.put_u64(static_cast<u64>(integer_value), base, false, upper_case, false, use_separator, Align::Right, 0, ' ', sign_mode, is_negative));
if (precision > 0) {
if (fraction_max_width && (zero_pad || fraction_value)) {
// FIXME: This is a terrible approximation but doing it properly would be a lot of work. If someone is up for that, a good
// place to start would be the following video from CppCon 2019:
// https://youtu.be/4P_kbF0EbZM (Stephan T. Lavavej “Floating-Point <charconv>: Making Your Code 10x Faster With C++17's Final Boss”)
u64 scale = pow<u64>(10, precision);
if (is_negative && fraction_value)
fraction_value = fraction_one - fraction_value;
auto fraction = (scale * fraction_value) / fraction_one; // TODO: overflows
if (is_negative && fraction != 0)
fraction = scale - fraction;
TRY(string_builder.try_append('.'));
size_t leading_zeroes = 0;
{
auto scale_tmp = scale / 10;
for (; fraction < scale_tmp; ++leading_zeroes) {
scale_tmp /= 10;
}
if (base == 10) {
u64 scale = pow<u64>(5, precision);
// FIXME: overflows (not before: fraction_value = (2^precision - 1) and precision >= 20) (use wider integer type)
auto fraction = scale * fraction_value;
TRY(format_builder.put_u64(fraction, base, false, upper_case, true, use_separator, Align::Right, precision));
} else if (base == 16) {
auto fraction = fraction_value << ((4 - (precision % 4)) % 4);
TRY(format_builder.put_u64(fraction, base, false, upper_case, false, use_separator, Align::Right, precision/4 + (precision % 4 != 0), '0'));
} else {
VERIFY_NOT_REACHED();
}
while (fraction != 0 && fraction % 10 == 0)
fraction /= 10;
size_t visible_precision = 0;
{
auto fraction_tmp = fraction;
for (; visible_precision < precision; ++visible_precision) {
if (fraction_tmp == 0 && display_mode != RealNumberDisplayMode::FixedPoint)
break;
fraction_tmp /= 10;
}
}
if (visible_precision == 0)
leading_zeroes = 0;
if (zero_pad || visible_precision > 0)
TRY(string_builder.try_append('.'));
if (leading_zeroes > 0)
TRY(format_builder.put_u64(0, base, false, false, true, use_separator, Align::Right, leading_zeroes));
if (visible_precision > 0)
TRY(format_builder.put_u64(fraction, base, false, upper_case, true, use_separator, Align::Right, visible_precision));
if (zero_pad && (precision - leading_zeroes - visible_precision) > 0)
TRY(format_builder.put_u64(0, base, false, false, true, use_separator, Align::Right, precision - leading_zeroes - visible_precision));
}
TRY(put_string(string_builder.string_view(), align, min_width, NumericLimits<size_t>::max(), fill));
auto formatted_string = string_builder.string_view();
if (fraction_max_width && (zero_pad || fraction_value)) {
auto point_index = formatted_string.find('.').value_or(0);
if (!point_index)
VERIFY_NOT_REACHED();
if (auto formatted_length = (formatted_string.length() - point_index - 1); formatted_length > fraction_max_width) {
formatted_string = formatted_string.substring_view(0, 1 + point_index + fraction_max_width);
} else {
string_builder.append_repeated('0', fraction_max_width - formatted_length);
formatted_string = string_builder.string_view();
}
if (!zero_pad)
formatted_string = formatted_string.trim("0"sv, TrimMode::Right);
if (formatted_string.ends_with('.'))
formatted_string = formatted_string.trim("."sv, TrimMode::Right);
}
TRY(put_string(formatted_string, align, min_width, NumericLimits<size_t>::max(), fill));
return {};
}

View File

@ -214,16 +214,16 @@ public:
i64 integer_value,
u64 fraction_value,
u64 fraction_one,
size_t precision,
u8 base = 10,
bool upper_case = false,
bool zero_pad = false,
bool use_separator = false,
Align align = Align::Right,
size_t min_width = 0,
size_t precision = 6,
size_t fraction_max_width = 6,
char fill = ' ',
SignMode sign_mode = SignMode::OnlyIfNeeded,
RealNumberDisplayMode = RealNumberDisplayMode::Default);
SignMode sign_mode = SignMode::OnlyIfNeeded);
#ifndef KERNEL
ErrorOr<void> put_f80(

View File

@ -169,7 +169,7 @@ TEST_CASE(cast)
TEST_CASE(formatter)
{
EXPECT_EQ(DeprecatedString::formatted("{}", FixedPoint<16>(123.456)), "123.455993"sv);
EXPECT_EQ(DeprecatedString::formatted("{}", FixedPoint<16>(-123.456)), "-123.455994"sv);
EXPECT_EQ(DeprecatedString::formatted("{}", FixedPoint<16>(-123.456)), "-123.455993"sv);
EXPECT_EQ(DeprecatedString::formatted("{}", FixedPoint<4>(123.456)), "123.4375"sv);
EXPECT_EQ(DeprecatedString::formatted("{}", FixedPoint<4>(-123.456)), "-123.4375"sv);
EXPECT_EQ(DeprecatedString::formatted("{}", FixedPoint<16> {}), "0"sv);
@ -178,11 +178,113 @@ TEST_CASE(formatter)
EXPECT_EQ(DeprecatedString::formatted("{}", FixedPoint<16>(0.003)), "0.003005"sv);
EXPECT_EQ(DeprecatedString::formatted("{}", FixedPoint<16>(0.0004)), "0.000396"sv);
EXPECT_EQ(DeprecatedString::formatted("{}", FixedPoint<16>(0.0000000005)), "0"sv);
EXPECT_EQ(DeprecatedString::formatted("{}", FixedPoint<16>(-0.1)), "-0.100007"sv);
EXPECT_EQ(DeprecatedString::formatted("{}", FixedPoint<16>(-0.02)), "-0.020005"sv);
EXPECT_EQ(DeprecatedString::formatted("{}", FixedPoint<16>(-0.1)), "-0.100006"sv);
EXPECT_EQ(DeprecatedString::formatted("{}", FixedPoint<16>(-0.02)), "-0.020004"sv);
EXPECT_EQ(DeprecatedString::formatted("{}", FixedPoint<16>(-0.0000000005)), "0"sv);
EXPECT_EQ(DeprecatedString::formatted("{}", Type(-1)), "-1"sv);
EXPECT_EQ(DeprecatedString::formatted("{}", Type(-2)), "-2"sv);
EXPECT_EQ(DeprecatedString::formatted("{}", Type(-3)), "-3"sv);
// exact representation
EXPECT_EQ(DeprecatedString::formatted("{:.30}", FixedPoint<16>(123.456)), "123.45599365234375"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30}", FixedPoint<16>(-0.1)), "-0.100006103515625"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30}", FixedPoint<16>(-0.02)), "-0.0200042724609375"sv);
// maximum fraction per precision; 1 - 2^-precision
EXPECT_EQ(DeprecatedString::formatted("{:0.20}", AK::FixedPoint<7, u64>::create_raw((1ull << 7) - 1)), "0.99218750000000000000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.20}", AK::FixedPoint<8, u64>::create_raw((1ull << 8) - 1)), "0.99609375000000000000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.20}", AK::FixedPoint<9, u64>::create_raw((1ull << 9) - 1)), "0.99804687500000000000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.20}", AK::FixedPoint<10, u64>::create_raw((1ull << 10) - 1)), "0.99902343750000000000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.20}", AK::FixedPoint<11, u64>::create_raw((1ull << 11) - 1)), "0.99951171875000000000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.20}", AK::FixedPoint<12, u64>::create_raw((1ull << 12) - 1)), "0.99975585937500000000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.20}", AK::FixedPoint<13, u64>::create_raw((1ull << 13) - 1)), "0.99987792968750000000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.20}", AK::FixedPoint<14, u64>::create_raw((1ull << 14) - 1)), "0.99993896484375000000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.20}", AK::FixedPoint<15, u64>::create_raw((1ull << 15) - 1)), "0.99996948242187500000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.20}", AK::FixedPoint<16, u64>::create_raw((1ull << 16) - 1)), "0.99998474121093750000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.20}", AK::FixedPoint<17, u64>::create_raw((1ull << 17) - 1)), "0.99999237060546875000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.20}", AK::FixedPoint<18, u64>::create_raw((1ull << 18) - 1)), "0.99999618530273437500"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.20}", AK::FixedPoint<19, u64>::create_raw((1ull << 19) - 1)), "0.99999809265136718750"sv);
// maximum factor and precision >= 20 bits/digits will overflow u64: (5^20)*(2^20 - 1) > 2^64
// EXPECT_EQ(DeprecatedString::formatted("{:0.30}", AK::FixedPoint<20, u64>::create_raw((1ull << 20) - 1)), "0.99999904632568359375"sv);
// minimum fraction per precision; 2^-precision
EXPECT_EQ(DeprecatedString::formatted("{:0.30}", AK::FixedPoint<7, u64>::create_raw(1)), "0.007812500000000000000000000000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.30}", AK::FixedPoint<8, u64>::create_raw(1)), "0.003906250000000000000000000000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.30}", AK::FixedPoint<9, u64>::create_raw(1)), "0.001953125000000000000000000000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.30}", AK::FixedPoint<10, u64>::create_raw(1)), "0.000976562500000000000000000000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.30}", AK::FixedPoint<11, u64>::create_raw(1)), "0.000488281250000000000000000000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.30}", AK::FixedPoint<12, u64>::create_raw(1)), "0.000244140625000000000000000000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.30}", AK::FixedPoint<13, u64>::create_raw(1)), "0.000122070312500000000000000000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.30}", AK::FixedPoint<14, u64>::create_raw(1)), "0.000061035156250000000000000000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.30}", AK::FixedPoint<15, u64>::create_raw(1)), "0.000030517578125000000000000000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.30}", AK::FixedPoint<16, u64>::create_raw(1)), "0.000015258789062500000000000000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.30}", AK::FixedPoint<17, u64>::create_raw(1)), "0.000007629394531250000000000000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.30}", AK::FixedPoint<18, u64>::create_raw(1)), "0.000003814697265625000000000000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.30}", AK::FixedPoint<19, u64>::create_raw(1)), "0.000001907348632812500000000000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.30}", AK::FixedPoint<20, u64>::create_raw(1)), "0.000000953674316406250000000000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.30}", AK::FixedPoint<21, u64>::create_raw(1)), "0.000000476837158203125000000000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.30}", AK::FixedPoint<22, u64>::create_raw(1)), "0.000000238418579101562500000000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.30}", AK::FixedPoint<23, u64>::create_raw(1)), "0.000000119209289550781250000000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.30}", AK::FixedPoint<24, u64>::create_raw(1)), "0.000000059604644775390625000000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.30}", AK::FixedPoint<25, u64>::create_raw(1)), "0.000000029802322387695312500000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.30}", AK::FixedPoint<26, u64>::create_raw(1)), "0.000000014901161193847656250000"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.30}", AK::FixedPoint<27, u64>::create_raw(1)), "0.000000007450580596923828125000"sv);
// minimum factor and precision >= 28 bits/digits will overflow u64: (5^28)*(1) > 2^64
// EXPECT_EQ(DeprecatedString::formatted("{:0.30}", AK::FixedPoint<28, u64>::create_raw(1)), "0.000000003725290298461914062500"sv);
EXPECT_EQ(DeprecatedString::formatted("{:a}", FixedPoint<16>(42.42)), "2a.6b85"sv);
EXPECT_EQ(DeprecatedString::formatted("{:0.10a}", FixedPoint<16>(69.69)), "45.b0a4000000"sv);
// maximum fraction per precision rendered in hexadecimal; 1 - 2^-precision; no overflow
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<7, u64>::create_raw((1ull << 7) - 1)), "0.fe"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<8, u64>::create_raw((1ull << 8) - 1)), "0.ff"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<9, u64>::create_raw((1ull << 9) - 1)), "0.ff8"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<10, u64>::create_raw((1ull << 10) - 1)), "0.ffc"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<11, u64>::create_raw((1ull << 11) - 1)), "0.ffe"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<12, u64>::create_raw((1ull << 12) - 1)), "0.fff"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<13, u64>::create_raw((1ull << 13) - 1)), "0.fff8"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<14, u64>::create_raw((1ull << 14) - 1)), "0.fffc"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<15, u64>::create_raw((1ull << 15) - 1)), "0.fffe"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<16, u64>::create_raw((1ull << 16) - 1)), "0.ffff"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<17, u64>::create_raw((1ull << 17) - 1)), "0.ffff8"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<18, u64>::create_raw((1ull << 18) - 1)), "0.ffffc"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<19, u64>::create_raw((1ull << 19) - 1)), "0.ffffe"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<20, u64>::create_raw((1ull << 20) - 1)), "0.fffff"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<21, u64>::create_raw((1ull << 21) - 1)), "0.fffff8"sv);
// ...skip some precisions
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<56, u64>::create_raw((1ull << 56) - 1)), "0.ffffffffffffff"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<57, u64>::create_raw((1ull << 57) - 1)), "0.ffffffffffffff8"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<58, u64>::create_raw((1ull << 58) - 1)), "0.ffffffffffffffc"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<59, u64>::create_raw((1ull << 59) - 1)), "0.ffffffffffffffe"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<60, u64>::create_raw((1ull << 60) - 1)), "0.fffffffffffffff"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<61, u64>::create_raw((1ull << 61) - 1)), "0.fffffffffffffff8"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<62, u64>::create_raw((1ull << 62) - 1)), "0.fffffffffffffffc"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<63, u64>::create_raw((1ull << 63) - 1)), "0.fffffffffffffffe"sv);
// minimum fraction per precision rendered in hexadecimal; 2^-precision; no overflow
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<7, u64>::create_raw(1)), "0.02"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<8, u64>::create_raw(1)), "0.01"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<9, u64>::create_raw(1)), "0.008"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<10, u64>::create_raw(1)), "0.004"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<11, u64>::create_raw(1)), "0.002"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<12, u64>::create_raw(1)), "0.001"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<13, u64>::create_raw(1)), "0.0008"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<14, u64>::create_raw(1)), "0.0004"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<15, u64>::create_raw(1)), "0.0002"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<16, u64>::create_raw(1)), "0.0001"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<17, u64>::create_raw(1)), "0.00008"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<18, u64>::create_raw(1)), "0.00004"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<19, u64>::create_raw(1)), "0.00002"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<20, u64>::create_raw(1)), "0.00001"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<21, u64>::create_raw(1)), "0.000008"sv);
// ..skip some precisions
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<56, u64>::create_raw(1)), "0.00000000000001"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<57, u64>::create_raw(1)), "0.000000000000008"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<58, u64>::create_raw(1)), "0.000000000000004"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<59, u64>::create_raw(1)), "0.000000000000002"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<60, u64>::create_raw(1)), "0.000000000000001"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<61, u64>::create_raw(1)), "0.0000000000000008"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<62, u64>::create_raw(1)), "0.0000000000000004"sv);
EXPECT_EQ(DeprecatedString::formatted("{:.30a}", AK::FixedPoint<63, u64>::create_raw(1)), "0.0000000000000002"sv);
}