mirror of
https://github.com/LadybirdBrowser/ladybird.git
synced 2024-09-11 05:25:32 +03:00
LibUnicode: Add Punycode::decode
This commit is contained in:
parent
37b5c05ec5
commit
299d35aadc
Notes:
sideshowbarker
2024-07-17 09:37:30 +09:00
Author: https://github.com/skyrising Commit: https://github.com/SerenityOS/serenity/commit/299d35aadc Pull-request: https://github.com/SerenityOS/serenity/pull/19414 Reviewed-by: https://github.com/AtkinsSJ Reviewed-by: https://github.com/nico Reviewed-by: https://github.com/shannonbooth Reviewed-by: https://github.com/trflynn89
@ -1,5 +1,6 @@
|
||||
set(TEST_SOURCES
|
||||
TestEmoji.cpp
|
||||
TestPunycode.cpp
|
||||
TestSegmentation.cpp
|
||||
TestUnicodeCharacterTypes.cpp
|
||||
TestUnicodeNormalization.cpp
|
||||
|
50
Tests/LibUnicode/TestPunycode.cpp
Normal file
50
Tests/LibUnicode/TestPunycode.cpp
Normal file
@ -0,0 +1,50 @@
|
||||
/*
|
||||
* Copyright (c) 2023, Simon Wanner <simon@skyrising.xyz>
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#include <LibTest/TestCase.h>
|
||||
|
||||
#include <LibUnicode/Punycode.h>
|
||||
|
||||
namespace Unicode::Punycode {
|
||||
|
||||
#define ENUMERATE_TEST_CASES \
|
||||
CASE(""sv, ""sv) \
|
||||
CASE("Well hello friends!"sv, "Well hello friends!-"sv) \
|
||||
CASE("Well-hello-friends"sv, "Well-hello-friends-"sv) \
|
||||
CASE("Wгellд-бhellбвo"sv, "Well-hello-friends"sv) \
|
||||
CASE("Hallöchen Freunde!"sv, "Hallchen Freunde!-2zb"sv) \
|
||||
CASE("Nåväl hej vänner"sv, "Nvl hej vnner-cfbhg"sv) \
|
||||
CASE("Ну привіт друзі"sv, " -kjc9flsd9cjetgj5xg"sv) \
|
||||
CASE("ليهمابتكلموشعربي؟"sv, "egbpdaj6bu4bxfgehfvwxn"sv) \
|
||||
CASE("他们为什么不说中文"sv, "ihqwcrb4cv8a8dqg056pqjye"sv) \
|
||||
CASE("他們爲什麽不說中文"sv, "ihqwctvzc91f659drss3x8bo0yb"sv) \
|
||||
CASE("Pročprostěnemluvíčesky"sv, "Proprostnemluvesky-uyb24dma41a"sv) \
|
||||
CASE("למההםפשוטלאמדבריםעברית"sv, "4dbcagdahymbxekheh6e0a7fei0b"sv) \
|
||||
CASE("यहलोगहिन्दीक्योंनहींबोलसकतेहैं"sv, "i1baa7eci9glrd9b2ae1bj0hfcgg6iyaf8o0a1dig0cd"sv) \
|
||||
CASE("なぜみんな日本語を話してくれないのか"sv, "n8jok5ay5dzabd5bym9f0cm5685rrjetr6pdxa"sv) \
|
||||
CASE("세계의모든사람들이한국어를이해한다면얼마나좋을까"sv, "989aomsvi5e83db1d2a355cv1e0vak1dwrv93d5xbh15a0dt30a5jpsd879ccm6fea98c"sv) \
|
||||
CASE("почемужеонинеговорятпорусски"sv, "b1abfaaepdrnnbgefbadotcwatmq2g4l"sv) \
|
||||
CASE("PorquénopuedensimplementehablarenEspañol"sv, "PorqunopuedensimplementehablarenEspaol-fmd56a"sv) \
|
||||
CASE("TạisaohọkhôngthểchỉnóitiếngViệt"sv, "TisaohkhngthchnitingVit-kjcr8268qyxafd2f1b9g"sv) \
|
||||
CASE("3年B組金八先生"sv, "3B-ww4c5e180e575a65lsy2b"sv) \
|
||||
CASE("安室奈美恵-with-SUPER-MONKEYS"sv, "-with-SUPER-MONKEYS-pc58ag80a8qai00g7n9n"sv) \
|
||||
CASE("Hello-Another-Way-それぞれの場所"sv, "Hello-Another-Way--fc4qua05auwb3674vfr0b"sv) \
|
||||
CASE("ひとつ屋根の下2"sv, "2-u9tlzr9756bt3uc0v"sv) \
|
||||
CASE("MajiでKoiする5秒前"sv, "MajiKoi5-783gue6qz075azm5e"sv) \
|
||||
CASE("パフィーdeルンバ"sv, "de-jg4avhby1noc0d"sv) \
|
||||
CASE("そのスピードで"sv, "d9juau41awczczp"sv) \
|
||||
CASE("-> $1.00 <-"sv, "-> $1.00 <--"sv)
|
||||
|
||||
TEST_CASE(decode)
|
||||
{
|
||||
#define CASE(a, b) EXPECT_EQ(TRY_OR_FAIL(decode(b)), a);
|
||||
ENUMERATE_TEST_CASES
|
||||
#undef CASE
|
||||
EXPECT(decode("Well hello friends!"sv).is_error());
|
||||
EXPECT(decode("Nåväl hej vänner"sv).is_error());
|
||||
}
|
||||
|
||||
}
|
@ -5,6 +5,7 @@ set(SOURCES
|
||||
CurrencyCode.cpp
|
||||
Emoji.cpp
|
||||
Normalize.cpp
|
||||
Punycode.cpp
|
||||
Segmentation.cpp
|
||||
String.cpp
|
||||
UnicodeUtils.cpp
|
||||
|
163
Userland/Libraries/LibUnicode/Punycode.cpp
Normal file
163
Userland/Libraries/LibUnicode/Punycode.cpp
Normal file
@ -0,0 +1,163 @@
|
||||
/*
|
||||
* Copyright (c) 2023, Simon Wanner <simon@skyrising.xyz>
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#include <AK/Utf32View.h>
|
||||
#include <LibUnicode/Punycode.h>
|
||||
|
||||
namespace Unicode::Punycode {
|
||||
|
||||
// https://www.rfc-editor.org/rfc/rfc3492.html#section-5
|
||||
static constexpr u32 BASE = 36;
|
||||
static constexpr u32 TMIN = 1;
|
||||
static constexpr u32 TMAX = 26;
|
||||
static constexpr u32 SKEW = 38;
|
||||
static constexpr u32 DAMP = 700;
|
||||
static constexpr u32 INITIAL_BIAS = 72;
|
||||
static constexpr u32 INITIAL_N = 0x80;
|
||||
static constexpr u32 DELIMITER = '-';
|
||||
|
||||
static Optional<u32> digit_value_of_code_point(u32 code_point)
|
||||
{
|
||||
if (code_point >= 'A' && code_point <= 'Z')
|
||||
return code_point - 'A';
|
||||
if (code_point >= 'a' && code_point <= 'z')
|
||||
return code_point - 'a';
|
||||
if (code_point >= '0' && code_point <= '9')
|
||||
return code_point - '0' + 26;
|
||||
return {};
|
||||
}
|
||||
|
||||
// https://www.rfc-editor.org/rfc/rfc3492.html#section-6.1
|
||||
static u32 adapt(u32 delta, u32 num_points, bool first_time)
|
||||
{
|
||||
// if firsttime then let delta = delta div damp
|
||||
if (first_time)
|
||||
delta = delta / DAMP;
|
||||
// else let delta = delta div 2
|
||||
else
|
||||
delta = delta / 2;
|
||||
|
||||
// let delta = delta + (delta div numpoints)
|
||||
delta = delta + (delta / num_points);
|
||||
|
||||
// let k = 0
|
||||
u32 k = 0;
|
||||
|
||||
// while delta > ((base - tmin) * tmax) div 2 do begin
|
||||
while (delta > ((BASE - TMIN) * TMAX) / 2) {
|
||||
// let delta = delta div (base - tmin)
|
||||
delta = delta / (BASE - TMIN);
|
||||
|
||||
// let k = k + base
|
||||
k = k + BASE;
|
||||
}
|
||||
|
||||
// return k + (((base - tmin + 1) * delta) div (delta + skew))
|
||||
return k + (((BASE - TMIN + 1) * delta) / (delta + SKEW));
|
||||
}
|
||||
|
||||
// https://www.rfc-editor.org/rfc/rfc3492.html#section-6.2
|
||||
ErrorOr<String> decode(StringView input)
|
||||
{
|
||||
size_t consumed = 0;
|
||||
|
||||
// let n = initial_n
|
||||
Checked<size_t> n = INITIAL_N;
|
||||
|
||||
// let i = 0
|
||||
Checked<u32> i = 0;
|
||||
|
||||
// let bias = initial_bias
|
||||
u32 bias = INITIAL_BIAS;
|
||||
|
||||
// let output = an empty string indexed from 0
|
||||
Vector<u32> output;
|
||||
|
||||
// consume all code points before the last delimiter (if there is one)
|
||||
// and copy them to output, fail on any non-basic code point
|
||||
Optional<size_t> last_delimiter_index = input.find_last(DELIMITER);
|
||||
if (last_delimiter_index.has_value()) {
|
||||
for (; consumed < last_delimiter_index.value(); consumed++) {
|
||||
if (!is_ascii(input[consumed]))
|
||||
return Error::from_string_literal("Unexpected non-basic code point");
|
||||
TRY(output.try_append(input[consumed]));
|
||||
}
|
||||
|
||||
// if more than zero code points were consumed then consume one more
|
||||
// (which will be the last delimiter)
|
||||
if (last_delimiter_index.value() > 0) {
|
||||
auto next = input[consumed++];
|
||||
VERIFY(next == DELIMITER);
|
||||
}
|
||||
}
|
||||
|
||||
// while the input is not exhausted do begin
|
||||
while (consumed < input.length()) {
|
||||
// let oldi = i
|
||||
Checked<u32> old_i = i;
|
||||
|
||||
// let w = 1
|
||||
Checked<u32> w = 1;
|
||||
|
||||
// for k = base to infinity in steps of base do begin
|
||||
for (size_t k = BASE;; k += BASE) {
|
||||
// consume a code point, or fail if there was none to consume
|
||||
if (consumed >= input.length())
|
||||
return Error::from_string_literal("No more code points to consume");
|
||||
auto code_point = input[consumed++];
|
||||
|
||||
// let digit = the code point's digit-value, fail if it has none
|
||||
auto digit = digit_value_of_code_point(code_point);
|
||||
if (!digit.has_value())
|
||||
return Error::from_string_literal("Invalid base-36 digit");
|
||||
|
||||
// let i = i + digit * w, fail on overflow
|
||||
i = i + Checked(digit.value()) * w;
|
||||
if (i.has_overflow())
|
||||
return Error::from_string_literal("Numeric overflow");
|
||||
|
||||
// let t = tmin if k <= bias {+ tmin}, or
|
||||
// tmax if k >= bias + tmax, or k - bias otherwise
|
||||
u32 t = k <= bias ? TMIN : (k >= bias + TMAX ? TMAX : k - bias);
|
||||
|
||||
// if digit < t then break
|
||||
if (digit.value() < t)
|
||||
break;
|
||||
|
||||
// let w = w * (base - t), fail on overflow
|
||||
w = w * Checked(BASE - t);
|
||||
if (w.has_overflow())
|
||||
return Error::from_string_literal("Numeric overflow");
|
||||
}
|
||||
// let bias = adapt(i - oldi, length(output) + 1, test oldi is 0?)
|
||||
bias = adapt((i - old_i).value(), output.size() + 1, !old_i);
|
||||
|
||||
// let n = n + i div (length(output) + 1), fail on overflow
|
||||
n = n + Checked(static_cast<size_t>(i.value() / static_cast<u32>(output.size() + 1)));
|
||||
if (n.has_overflow())
|
||||
return Error::from_string_literal("Numeric overflow");
|
||||
|
||||
// let i = i mod (length(output) + 1)
|
||||
i = i % Checked(static_cast<u32>(output.size() + 1));
|
||||
|
||||
// {if n is a basic code point then fail}
|
||||
// NOTE: The full statement enclosed in braces (checking whether n is a basic code point) can be omitted if initial_n exceeds all basic code points
|
||||
// (which is true for Punycode), because n is never less than initial_n.
|
||||
VERIFY(!is_ascii(n.value()));
|
||||
|
||||
// insert n into output at position i
|
||||
TRY(output.try_insert(i.value(), n.value()));
|
||||
|
||||
// increment i
|
||||
i++;
|
||||
}
|
||||
|
||||
StringBuilder builder;
|
||||
TRY(builder.try_append(Utf32View(output.data(), output.size())));
|
||||
return builder.to_string();
|
||||
}
|
||||
|
||||
}
|
15
Userland/Libraries/LibUnicode/Punycode.h
Normal file
15
Userland/Libraries/LibUnicode/Punycode.h
Normal file
@ -0,0 +1,15 @@
|
||||
/*
|
||||
* Copyright (c) 2023, Simon Wanner <simon@skyrising.xyz>
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <AK/String.h>
|
||||
|
||||
namespace Unicode::Punycode {
|
||||
|
||||
ErrorOr<String> decode(StringView);
|
||||
|
||||
}
|
Loading…
Reference in New Issue
Block a user