LadybirdBrowser/ladybird
1
1
Fork 0
mirror of https://github.com/LadybirdBrowser/ladybird.git synced 2024-08-16 08:50:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

LibLocale: Remove the number formatting data generator

Browse Source 
The last user of the remaining data (Intl.DurationFormat) no longer
consumes these APIs.
This commit is contained in:
Timothy Flynn 2024-06-13 16:11:32 -04:00 committed by Andreas Kling
parent d634039c10
commit f9a7fc6263
Notes: sideshowbarker 2024-07-17 07:11:12 +09:00 
Author: https://github.com/trflynn89
Commit: https://github.com/LadybirdBrowser/ladybird/commit/f9a7fc6263
Pull-request: https://github.com/LadybirdBrowser/ladybird/pull/157
6 changed files with 0 additions and 500 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
Meta/CMake/locale_data.cmake
View File

@ -41,9 +41,6 @@ if (ENABLE_UNICODE_DATABASE_DOWNLOAD)
set(LOCALE_DATA_HEADER LocaleData.h)
set(LOCALE_DATA_IMPLEMENTATION LocaleData.cpp)
set(NUMBER_FORMAT_DATA_HEADER NumberFormatData.h)
set(NUMBER_FORMAT_DATA_IMPLEMENTATION NumberFormatData.cpp)
set(PLURAL_RULES_DATA_HEADER PluralRulesData.h)
set(PLURAL_RULES_DATA_IMPLEMENTATION PluralRulesData.cpp)
@ -58,14 +55,6 @@ if (ENABLE_UNICODE_DATABASE_DOWNLOAD)
"${LOCALE_DATA_IMPLEMENTATION}"
arguments -b "${CLDR_BCP47_PATH}" -r "${CLDR_CORE_PATH}" -n "${CLDR_NUMBERS_PATH}" -d "${CLDR_DATES_PATH}"
)
invoke_generator(
"NumberFormatData"
Lagom::GenerateNumberFormatData
"${CLDR_VERSION_FILE}"
"${NUMBER_FORMAT_DATA_HEADER}"
"${NUMBER_FORMAT_DATA_IMPLEMENTATION}"
arguments -r "${CLDR_CORE_PATH}" -n "${CLDR_NUMBERS_PATH}"
)
invoke_generator(
"PluralRulesData"
Lagom::GeneratePluralRulesData
@ -86,8 +75,6 @@ if (ENABLE_UNICODE_DATABASE_DOWNLOAD)
set(LOCALE_DATA_SOURCES
${LOCALE_DATA_HEADER}
${LOCALE_DATA_IMPLEMENTATION}
${NUMBER_FORMAT_DATA_HEADER}
${NUMBER_FORMAT_DATA_IMPLEMENTATION}
${PLURAL_RULES_DATA_HEADER}
${PLURAL_RULES_DATA_IMPLEMENTATION}
${RELATIVE_TIME_FORMAT_DATA_HEADER}

1
Meta/Lagom/Tools/CodeGenerators/LibLocale/CMakeLists.txt
View File

@ -1,4 +1,3 @@
lagom_tool(GenerateLocaleData SOURCES GenerateLocaleData.cpp LIBS LibMain)
lagom_tool(GenerateNumberFormatData SOURCES GenerateNumberFormatData.cpp LIBS LibMain)
lagom_tool(GeneratePluralRulesData SOURCES GeneratePluralRulesData.cpp LIBS LibMain)
lagom_tool(GenerateRelativeTimeFormatData SOURCES GenerateRelativeTimeFormatData.cpp LIBS LibMain)

436
Meta/Lagom/Tools/CodeGenerators/LibLocale/GenerateNumberFormatData.cpp
View File

@ -1,436 +0,0 @@
/*
* Copyright (c) 2021-2023, Tim Flynn <trflynn89@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include "../LibUnicode/GeneratorUtil.h" // FIXME: Move this somewhere common.
#include <AK/Array.h>
#include <AK/ByteString.h>
#include <AK/Format.h>
#include <AK/HashFunctions.h>
#include <AK/HashMap.h>
#include <AK/JsonObject.h>
#include <AK/JsonParser.h>
#include <AK/JsonValue.h>
#include <AK/LexicalPath.h>
#include <AK/QuickSort.h>
#include <AK/SourceGenerator.h>
#include <AK/StringBuilder.h>
#include <AK/Traits.h>
#include <AK/Utf8View.h>
#include <LibCore/ArgsParser.h>
#include <LibCore/Directory.h>
#include <LibFileSystem/FileSystem.h>
#include <LibLocale/NumberFormat.h>
#include <math.h>
using NumericSymbolList = Vector<size_t>;
struct NumberSystem {
unsigned hash() const { return int_hash(symbols); }
bool operator==(NumberSystem const& other) const { return (symbols == other.symbols); }
size_t symbols { 0 };
};
template<>
struct AK::Formatter<NumberSystem> : Formatter<FormatString> {
ErrorOr<void> format(FormatBuilder& builder, NumberSystem const& system)
{
return Formatter<FormatString>::format(builder, "{{ {} }}"sv, system.symbols);
}
};
template<>
struct AK::Traits<NumberSystem> : public DefaultTraits<NumberSystem> {
static unsigned hash(NumberSystem const& s) { return s.hash(); }
};
struct LocaleData {
Vector<size_t> number_systems;
};
struct CLDR {
UniqueStringStorage unique_strings;
UniqueStorage<NumericSymbolList> unique_symbols;
UniqueStorage<NumberSystem> unique_systems;
HashMap<ByteString, Array<u32, 10>> number_system_digits;
Vector<ByteString> number_systems;
HashMap<ByteString, LocaleData> locales;
};
static ErrorOr<void> parse_number_system_digits(ByteString core_supplemental_path, CLDR& cldr)
{
LexicalPath number_systems_path(move(core_supplemental_path));
number_systems_path = number_systems_path.append("numberingSystems.json"sv);
auto number_systems = TRY(read_json_file(number_systems_path.string()));
auto const& supplemental_object = number_systems.as_object().get_object("supplemental"sv).value();
auto const& number_systems_object = supplemental_object.get_object("numberingSystems"sv).value();
number_systems_object.for_each_member([&](auto const& number_system, auto const& digits_object) {
auto type = digits_object.as_object().get_byte_string("_type"sv).value();
if (type != "numeric"sv)
return;
auto digits = digits_object.as_object().get_byte_string("_digits"sv).value();
Utf8View utf8_digits { digits };
VERIFY(utf8_digits.length() == 10);
auto& number_system_digits = cldr.number_system_digits.ensure(number_system);
size_t index = 0;
for (u32 digit : utf8_digits)
number_system_digits[index++] = digit;
if (!cldr.number_systems.contains_slow(number_system))
cldr.number_systems.append(number_system);
});
return {};
}
static ErrorOr<void> parse_number_systems(ByteString locale_numbers_path, CLDR& cldr, LocaleData& locale)
{
LexicalPath numbers_path(move(locale_numbers_path));
numbers_path = numbers_path.append("numbers.json"sv);
auto numbers = TRY(read_json_file(numbers_path.string()));
auto const& main_object = numbers.as_object().get_object("main"sv).value();
auto const& locale_object = main_object.get_object(numbers_path.parent().basename()).value();
auto const& locale_numbers_object = locale_object.get_object("numbers"sv).value();
auto const& minimum_grouping_digits = locale_numbers_object.get_byte_string("minimumGroupingDigits"sv).value();
Vector<Optional<NumberSystem>> number_systems;
number_systems.resize(cldr.number_systems.size());
auto ensure_number_system = [&](auto const& system) -> NumberSystem& {
auto system_index = cldr.number_systems.find_first_index(system).value();
VERIFY(system_index < number_systems.size());
auto& number_system = number_systems.at(system_index);
if (!number_system.has_value())
number_system = NumberSystem {};
return number_system.value();
};
auto numeric_symbol_from_string = [&](StringView numeric_symbol) -> Optional<Locale::NumericSymbol> {
if (numeric_symbol == "approximatelySign"sv)
return Locale::NumericSymbol::ApproximatelySign;
if (numeric_symbol == "decimal"sv)
return Locale::NumericSymbol::Decimal;
if (numeric_symbol == "exponential"sv)
return Locale::NumericSymbol::Exponential;
if (numeric_symbol == "group"sv)
return Locale::NumericSymbol::Group;
if (numeric_symbol == "infinity"sv)
return Locale::NumericSymbol::Infinity;
if (numeric_symbol == "minusSign"sv)
return Locale::NumericSymbol::MinusSign;
if (numeric_symbol == "nan"sv)
return Locale::NumericSymbol::NaN;
if (numeric_symbol == "percentSign"sv)
return Locale::NumericSymbol::PercentSign;
if (numeric_symbol == "plusSign"sv)
return Locale::NumericSymbol::PlusSign;
if (numeric_symbol == "timeSeparator"sv)
return Locale::NumericSymbol::TimeSeparator;
return {};
};
locale_numbers_object.for_each_member([&](auto const& key, JsonValue const& value) {
constexpr auto symbols_prefix = "symbols-numberSystem-"sv;
constexpr auto misc_patterns_prefix = "miscPatterns-numberSystem-"sv;
if (key.starts_with(symbols_prefix)) {
auto system = key.substring(symbols_prefix.length());
auto& number_system = ensure_number_system(system);
NumericSymbolList symbols;
value.as_object().for_each_member([&](auto const& symbol, JsonValue const& localization) {
auto numeric_symbol = numeric_symbol_from_string(symbol);
if (!numeric_symbol.has_value())
return;
if (to_underlying(*numeric_symbol) >= symbols.size())
symbols.resize(to_underlying(*numeric_symbol) + 1);
auto symbol_index = cldr.unique_strings.ensure(localization.as_string());
symbols[to_underlying(*numeric_symbol)] = symbol_index;
});
// The range separator does not appear in the symbols list, we have to extract it from
// the range pattern.
auto misc_patterns_key = ByteString::formatted("{}{}", misc_patterns_prefix, system);
auto misc_patterns = locale_numbers_object.get_object(misc_patterns_key).value();
auto range_separator = misc_patterns.get_byte_string("range"sv).value();
auto begin_index = range_separator.find("{0}"sv).value() + "{0}"sv.length();
auto end_index = range_separator.find("{1}"sv).value();
range_separator = range_separator.substring(begin_index, end_index - begin_index);
if (to_underlying(Locale::NumericSymbol::RangeSeparator) >= symbols.size())
symbols.resize(to_underlying(Locale::NumericSymbol::RangeSeparator) + 1);
auto symbol_index = cldr.unique_strings.ensure(move(range_separator));
symbols[to_underlying(Locale::NumericSymbol::RangeSeparator)] = symbol_index;
number_system.symbols = cldr.unique_symbols.ensure(move(symbols));
}
});
locale.number_systems.ensure_capacity(number_systems.size());
for (auto& number_system : number_systems) {
size_t system_index = 0;
if (number_system.has_value())
system_index = cldr.unique_systems.ensure(number_system.release_value());
locale.number_systems.append(system_index);
}
// locale.minimum_grouping_digits = minimum_grouping_digits.template to_number<u8>().value();
return {};
}
static ErrorOr<void> parse_all_locales(ByteString core_path, ByteString numbers_path, CLDR& cldr)
{
LexicalPath core_supplemental_path(move(core_path));
core_supplemental_path = core_supplemental_path.append("supplemental"sv);
VERIFY(FileSystem::is_directory(core_supplemental_path.string()));
TRY(parse_number_system_digits(core_supplemental_path.string(), cldr));
auto remove_variants_from_path = [&](ByteString path) -> ErrorOr<ByteString> {
auto parsed_locale = TRY(CanonicalLanguageID::parse(cldr.unique_strings, LexicalPath::basename(path)));
StringBuilder builder;
builder.append(cldr.unique_strings.get(parsed_locale.language));
if (auto script = cldr.unique_strings.get(parsed_locale.script); !script.is_empty())
builder.appendff("-{}", script);
if (auto region = cldr.unique_strings.get(parsed_locale.region); !region.is_empty())
builder.appendff("-{}", region);
return builder.to_byte_string();
};
TRY(Core::Directory::for_each_entry(TRY(String::formatted("{}/main", numbers_path)), Core::DirIterator::SkipParentAndBaseDir, [&](auto& entry, auto& directory) -> ErrorOr<IterationDecision> {
auto numbers_path = LexicalPath::join(directory.path().string(), entry.name).string();
auto language = TRY(remove_variants_from_path(numbers_path));
auto& locale = cldr.locales.ensure(language);
TRY(parse_number_systems(numbers_path, cldr, locale));
return IterationDecision::Continue;
}));
return {};
}
static ByteString format_identifier(StringView, ByteString const& identifier)
{
return identifier.to_titlecase();
}
static ErrorOr<void> generate_unicode_locale_header(Core::InputBufferedFile& file, CLDR& cldr)
{
StringBuilder builder;
SourceGenerator generator { builder };
generator.append(R"~~~(
#pragma once
#include <AK/Types.h>
namespace Locale {
)~~~");
generate_enum(generator, format_identifier, "NumberSystem"sv, {}, cldr.number_systems);
generator.append(R"~~~(
}
)~~~");
TRY(file.write_until_depleted(generator.as_string_view().bytes()));
return {};
}
static ErrorOr<void> generate_unicode_locale_implementation(Core::InputBufferedFile& file, CLDR& cldr)
{
StringBuilder builder;
SourceGenerator generator { builder };
generator.set("string_index_type"sv, cldr.unique_strings.type_that_fits());
generator.set("numeric_symbol_list_index_type"sv, cldr.unique_symbols.type_that_fits());
generator.append(R"~~~(
#include <AK/Array.h>
#include <AK/Optional.h>
#include <AK/Span.h>
#include <AK/StringView.h>
#include <LibLocale/Locale.h>
#include <LibLocale/LocaleData.h>
#include <LibLocale/NumberFormat.h>
#include <LibLocale/NumberFormatData.h>
namespace Locale {
)~~~");
cldr.unique_strings.generate(generator);
generator.append(R"~~~(
struct NumberSystemData {
@numeric_symbol_list_index_type@ symbols { 0 };
};
)~~~");
cldr.unique_symbols.generate(generator, cldr.unique_strings.type_that_fits(), "s_numeric_symbol_lists"sv);
cldr.unique_systems.generate(generator, "NumberSystemData"sv, "s_number_systems"sv, 10);
auto locales = cldr.locales.keys();
quick_sort(locales);
auto append_map = [&](ByteString name, auto type, auto const& map) {
generator.set("name", move(name));
generator.set("type", type);
generator.set("size", ByteString::number(map.size()));
generator.append(R"~~~(
static constexpr Array<@type@, @size@> @name@ { {)~~~");
bool first = true;
for (auto const& item : map) {
generator.append(first ? " "sv : ", "sv);
if constexpr (requires { item.value; })
generator.append(ByteString::number(item.value));
else
generator.append(ByteString::number(item));
first = false;
}
generator.append(" } };");
};
generate_mapping(generator, cldr.number_system_digits, "u32"sv, "s_number_systems_digits"sv, "s_number_systems_digits_{}"sv, nullptr, [&](auto const& name, auto const& value) { append_map(name, "u32"sv, value); });
generate_mapping(generator, cldr.locales, cldr.unique_systems.type_that_fits(), "s_locale_number_systems"sv, "s_number_systems_{}"sv, nullptr, [&](auto const& name, auto const& value) { append_map(name, cldr.unique_systems.type_that_fits(), value.number_systems); });
generator.append(R"~~~(
static Optional<NumberSystem> keyword_to_number_system(KeywordNumbers keyword)
{
switch (keyword) {)~~~");
for (auto const& number_system : cldr.number_systems) {
generator.set("name"sv, format_identifier({}, number_system));
generator.append(R"~~~(
case KeywordNumbers::@name@:
return NumberSystem::@name@;)~~~");
}
generator.append(R"~~~(
default:
return {};
}
}
Optional<ReadonlySpan<u32>> get_digits_for_number_system(StringView system)
{
auto number_system_keyword = keyword_nu_from_string(system);
if (!number_system_keyword.has_value())
return {};
auto number_system_value = keyword_to_number_system(*number_system_keyword);
if (!number_system_value.has_value())
return {};
auto number_system_index = to_underlying(*number_system_value);
return s_number_systems_digits[number_system_index];
}
static NumberSystemData const* find_number_system(StringView locale, StringView system)
{
auto locale_value = locale_from_string(locale);
if (!locale_value.has_value())
return nullptr;
auto locale_index = to_underlying(*locale_value) - 1; // Subtract 1 because 0 == Locale::None.
auto const& number_systems = s_locale_number_systems.at(locale_index);
auto lookup_number_system = [&](auto number_system) -> NumberSystemData const* {
auto number_system_keyword = keyword_nu_from_string(number_system);
if (!number_system_keyword.has_value())
return nullptr;
auto number_system_value = keyword_to_number_system(*number_system_keyword);
if (!number_system_value.has_value())
return nullptr;
auto number_system_index = to_underlying(*number_system_value);
number_system_index = number_systems.at(number_system_index);
if (number_system_index == 0)
return nullptr;
return &s_number_systems.at(number_system_index);
};
if (auto const* number_system = lookup_number_system(system))
return number_system;
auto default_number_system = get_preferred_keyword_value_for_locale(locale, "nu"sv);
if (!default_number_system.has_value())
return nullptr;
return lookup_number_system(*default_number_system);
}
Optional<StringView> get_number_system_symbol(StringView locale, StringView system, NumericSymbol symbol)
{
if (auto const* number_system = find_number_system(locale, system); number_system != nullptr) {
auto symbols = s_numeric_symbol_lists.at(number_system->symbols);
auto symbol_index = to_underlying(symbol);
if (symbol_index >= symbols.size())
return {};
return decode_string(symbols[symbol_index]);
}
return {};
}
}
)~~~");
TRY(file.write_until_depleted(generator.as_string_view().bytes()));
return {};
}
ErrorOr<int> serenity_main(Main::Arguments arguments)
{
StringView generated_header_path;
StringView generated_implementation_path;
StringView core_path;
StringView numbers_path;
Core::ArgsParser args_parser;
args_parser.add_option(generated_header_path, "Path to the Unicode locale header file to generate", "generated-header-path", 'h', "generated-header-path");
args_parser.add_option(generated_implementation_path, "Path to the Unicode locale implementation file to generate", "generated-implementation-path", 'c', "generated-implementation-path");
args_parser.add_option(core_path, "Path to cldr-core directory", "core-path", 'r', "core-path");
args_parser.add_option(numbers_path, "Path to cldr-numbers directory", "numbers-path", 'n', "numbers-path");
args_parser.parse(arguments);
auto generated_header_file = TRY(open_file(generated_header_path, Core::File::OpenMode::Write));
auto generated_implementation_file = TRY(open_file(generated_implementation_path, Core::File::OpenMode::Write));
CLDR cldr;
TRY(parse_all_locales(core_path, numbers_path, cldr));
TRY(generate_unicode_locale_header(*generated_header_file, cldr));
TRY(generate_unicode_locale_implementation(*generated_implementation_file, cldr));
return 0;
}

1
Userland/Libraries/LibLocale/Forward.h
View File

@ -20,7 +20,6 @@ enum class KeywordColNumeric : u8;
enum class KeywordHours : u8;
enum class KeywordNumbers : u8;
enum class Locale : u16;
enum class NumericSymbol : u8;
enum class PluralCategory : u8;
enum class Style : u8;
enum class Weekday : u8;

30
Userland/Libraries/LibLocale/NumberFormat.cpp
View File

@ -825,34 +825,4 @@ NonnullOwnPtr<NumberFormat> NumberFormat::create(
return adopt_own(*new NumberFormatImpl(locale_data->locale(), move(formatter), is_unit));
}
Optional<StringView> __attribute__((weak)) get_number_system_symbol(StringView, StringView, NumericSymbol) { return {}; }
Optional<ReadonlySpan<u32>> __attribute__((weak)) get_digits_for_number_system(StringView)
{
// Fall back to "latn" digits when Unicode data generation is disabled.
constexpr Array<u32, 10> digits { { 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39 } };
return digits.span();
}
String replace_digits_for_number_system(StringView system, StringView number)
{
auto digits = get_digits_for_number_system(system);
if (!digits.has_value())
digits = get_digits_for_number_system("latn"sv);
VERIFY(digits.has_value());
StringBuilder builder;
for (auto ch : number) {
if (is_ascii_digit(ch)) {
u32 digit = digits->at(parse_ascii_digit(ch));
builder.append_code_point(digit);
} else {
builder.append(ch);
}
}
return MUST(builder.to_string());
}
}

19
Userland/Libraries/LibLocale/NumberFormat.h
View File

@ -12,7 +12,6 @@
#include <AK/Variant.h>
#include <AK/Vector.h>
#include <LibLocale/Forward.h>
#include <LibLocale/PluralRules.h>
namespace Locale {
@ -167,22 +166,4 @@ protected:
NumberFormat() = default;
};
enum class NumericSymbol : u8 {
ApproximatelySign,
Decimal,
Exponential,
Group,
Infinity,
MinusSign,
NaN,
PercentSign,
PlusSign,
RangeSeparator,
TimeSeparator,
};
Optional<StringView> get_number_system_symbol(StringView locale, StringView system, NumericSymbol symbol);
Optional<ReadonlySpan<u32>> get_digits_for_number_system(StringView system);
String replace_digits_for_number_system(StringView system, StringView number);
}