mirror of
https://github.com/LadybirdBrowser/ladybird.git
synced 2024-09-11 05:25:32 +03:00
LibUnicode: Add Punycode::decode
This commit is contained in:
parent
37b5c05ec5
commit
299d35aadc
Notes:
sideshowbarker
2024-07-17 09:37:30 +09:00
Author: https://github.com/skyrising Commit: https://github.com/SerenityOS/serenity/commit/299d35aadc Pull-request: https://github.com/SerenityOS/serenity/pull/19414 Reviewed-by: https://github.com/AtkinsSJ Reviewed-by: https://github.com/nico Reviewed-by: https://github.com/shannonbooth Reviewed-by: https://github.com/trflynn89
@ -1,5 +1,6 @@
|
|||||||
set(TEST_SOURCES
|
set(TEST_SOURCES
|
||||||
TestEmoji.cpp
|
TestEmoji.cpp
|
||||||
|
TestPunycode.cpp
|
||||||
TestSegmentation.cpp
|
TestSegmentation.cpp
|
||||||
TestUnicodeCharacterTypes.cpp
|
TestUnicodeCharacterTypes.cpp
|
||||||
TestUnicodeNormalization.cpp
|
TestUnicodeNormalization.cpp
|
||||||
|
50
Tests/LibUnicode/TestPunycode.cpp
Normal file
50
Tests/LibUnicode/TestPunycode.cpp
Normal file
@ -0,0 +1,50 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (c) 2023, Simon Wanner <simon@skyrising.xyz>
|
||||||
|
*
|
||||||
|
* SPDX-License-Identifier: BSD-2-Clause
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include <LibTest/TestCase.h>
|
||||||
|
|
||||||
|
#include <LibUnicode/Punycode.h>
|
||||||
|
|
||||||
|
namespace Unicode::Punycode {
|
||||||
|
|
||||||
|
#define ENUMERATE_TEST_CASES \
|
||||||
|
CASE(""sv, ""sv) \
|
||||||
|
CASE("Well hello friends!"sv, "Well hello friends!-"sv) \
|
||||||
|
CASE("Well-hello-friends"sv, "Well-hello-friends-"sv) \
|
||||||
|
CASE("Wгellд-бhellбвo"sv, "Well-hello-friends"sv) \
|
||||||
|
CASE("Hallöchen Freunde!"sv, "Hallchen Freunde!-2zb"sv) \
|
||||||
|
CASE("Nåväl hej vänner"sv, "Nvl hej vnner-cfbhg"sv) \
|
||||||
|
CASE("Ну привіт друзі"sv, " -kjc9flsd9cjetgj5xg"sv) \
|
||||||
|
CASE("ليهمابتكلموشعربي؟"sv, "egbpdaj6bu4bxfgehfvwxn"sv) \
|
||||||
|
CASE("他们为什么不说中文"sv, "ihqwcrb4cv8a8dqg056pqjye"sv) \
|
||||||
|
CASE("他們爲什麽不說中文"sv, "ihqwctvzc91f659drss3x8bo0yb"sv) \
|
||||||
|
CASE("Pročprostěnemluvíčesky"sv, "Proprostnemluvesky-uyb24dma41a"sv) \
|
||||||
|
CASE("למההםפשוטלאמדבריםעברית"sv, "4dbcagdahymbxekheh6e0a7fei0b"sv) \
|
||||||
|
CASE("यहलोगहिन्दीक्योंनहींबोलसकतेहैं"sv, "i1baa7eci9glrd9b2ae1bj0hfcgg6iyaf8o0a1dig0cd"sv) \
|
||||||
|
CASE("なぜみんな日本語を話してくれないのか"sv, "n8jok5ay5dzabd5bym9f0cm5685rrjetr6pdxa"sv) \
|
||||||
|
CASE("세계의모든사람들이한국어를이해한다면얼마나좋을까"sv, "989aomsvi5e83db1d2a355cv1e0vak1dwrv93d5xbh15a0dt30a5jpsd879ccm6fea98c"sv) \
|
||||||
|
CASE("почемужеонинеговорятпорусски"sv, "b1abfaaepdrnnbgefbadotcwatmq2g4l"sv) \
|
||||||
|
CASE("PorquénopuedensimplementehablarenEspañol"sv, "PorqunopuedensimplementehablarenEspaol-fmd56a"sv) \
|
||||||
|
CASE("TạisaohọkhôngthểchỉnóitiếngViệt"sv, "TisaohkhngthchnitingVit-kjcr8268qyxafd2f1b9g"sv) \
|
||||||
|
CASE("3年B組金八先生"sv, "3B-ww4c5e180e575a65lsy2b"sv) \
|
||||||
|
CASE("安室奈美恵-with-SUPER-MONKEYS"sv, "-with-SUPER-MONKEYS-pc58ag80a8qai00g7n9n"sv) \
|
||||||
|
CASE("Hello-Another-Way-それぞれの場所"sv, "Hello-Another-Way--fc4qua05auwb3674vfr0b"sv) \
|
||||||
|
CASE("ひとつ屋根の下2"sv, "2-u9tlzr9756bt3uc0v"sv) \
|
||||||
|
CASE("MajiでKoiする5秒前"sv, "MajiKoi5-783gue6qz075azm5e"sv) \
|
||||||
|
CASE("パフィーdeルンバ"sv, "de-jg4avhby1noc0d"sv) \
|
||||||
|
CASE("そのスピードで"sv, "d9juau41awczczp"sv) \
|
||||||
|
CASE("-> $1.00 <-"sv, "-> $1.00 <--"sv)
|
||||||
|
|
||||||
|
TEST_CASE(decode)
|
||||||
|
{
|
||||||
|
#define CASE(a, b) EXPECT_EQ(TRY_OR_FAIL(decode(b)), a);
|
||||||
|
ENUMERATE_TEST_CASES
|
||||||
|
#undef CASE
|
||||||
|
EXPECT(decode("Well hello friends!"sv).is_error());
|
||||||
|
EXPECT(decode("Nåväl hej vänner"sv).is_error());
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
@ -5,6 +5,7 @@ set(SOURCES
|
|||||||
CurrencyCode.cpp
|
CurrencyCode.cpp
|
||||||
Emoji.cpp
|
Emoji.cpp
|
||||||
Normalize.cpp
|
Normalize.cpp
|
||||||
|
Punycode.cpp
|
||||||
Segmentation.cpp
|
Segmentation.cpp
|
||||||
String.cpp
|
String.cpp
|
||||||
UnicodeUtils.cpp
|
UnicodeUtils.cpp
|
||||||
|
163
Userland/Libraries/LibUnicode/Punycode.cpp
Normal file
163
Userland/Libraries/LibUnicode/Punycode.cpp
Normal file
@ -0,0 +1,163 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (c) 2023, Simon Wanner <simon@skyrising.xyz>
|
||||||
|
*
|
||||||
|
* SPDX-License-Identifier: BSD-2-Clause
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include <AK/Utf32View.h>
|
||||||
|
#include <LibUnicode/Punycode.h>
|
||||||
|
|
||||||
|
namespace Unicode::Punycode {
|
||||||
|
|
||||||
|
// https://www.rfc-editor.org/rfc/rfc3492.html#section-5
|
||||||
|
static constexpr u32 BASE = 36;
|
||||||
|
static constexpr u32 TMIN = 1;
|
||||||
|
static constexpr u32 TMAX = 26;
|
||||||
|
static constexpr u32 SKEW = 38;
|
||||||
|
static constexpr u32 DAMP = 700;
|
||||||
|
static constexpr u32 INITIAL_BIAS = 72;
|
||||||
|
static constexpr u32 INITIAL_N = 0x80;
|
||||||
|
static constexpr u32 DELIMITER = '-';
|
||||||
|
|
||||||
|
static Optional<u32> digit_value_of_code_point(u32 code_point)
|
||||||
|
{
|
||||||
|
if (code_point >= 'A' && code_point <= 'Z')
|
||||||
|
return code_point - 'A';
|
||||||
|
if (code_point >= 'a' && code_point <= 'z')
|
||||||
|
return code_point - 'a';
|
||||||
|
if (code_point >= '0' && code_point <= '9')
|
||||||
|
return code_point - '0' + 26;
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
// https://www.rfc-editor.org/rfc/rfc3492.html#section-6.1
|
||||||
|
static u32 adapt(u32 delta, u32 num_points, bool first_time)
|
||||||
|
{
|
||||||
|
// if firsttime then let delta = delta div damp
|
||||||
|
if (first_time)
|
||||||
|
delta = delta / DAMP;
|
||||||
|
// else let delta = delta div 2
|
||||||
|
else
|
||||||
|
delta = delta / 2;
|
||||||
|
|
||||||
|
// let delta = delta + (delta div numpoints)
|
||||||
|
delta = delta + (delta / num_points);
|
||||||
|
|
||||||
|
// let k = 0
|
||||||
|
u32 k = 0;
|
||||||
|
|
||||||
|
// while delta > ((base - tmin) * tmax) div 2 do begin
|
||||||
|
while (delta > ((BASE - TMIN) * TMAX) / 2) {
|
||||||
|
// let delta = delta div (base - tmin)
|
||||||
|
delta = delta / (BASE - TMIN);
|
||||||
|
|
||||||
|
// let k = k + base
|
||||||
|
k = k + BASE;
|
||||||
|
}
|
||||||
|
|
||||||
|
// return k + (((base - tmin + 1) * delta) div (delta + skew))
|
||||||
|
return k + (((BASE - TMIN + 1) * delta) / (delta + SKEW));
|
||||||
|
}
|
||||||
|
|
||||||
|
// https://www.rfc-editor.org/rfc/rfc3492.html#section-6.2
|
||||||
|
ErrorOr<String> decode(StringView input)
|
||||||
|
{
|
||||||
|
size_t consumed = 0;
|
||||||
|
|
||||||
|
// let n = initial_n
|
||||||
|
Checked<size_t> n = INITIAL_N;
|
||||||
|
|
||||||
|
// let i = 0
|
||||||
|
Checked<u32> i = 0;
|
||||||
|
|
||||||
|
// let bias = initial_bias
|
||||||
|
u32 bias = INITIAL_BIAS;
|
||||||
|
|
||||||
|
// let output = an empty string indexed from 0
|
||||||
|
Vector<u32> output;
|
||||||
|
|
||||||
|
// consume all code points before the last delimiter (if there is one)
|
||||||
|
// and copy them to output, fail on any non-basic code point
|
||||||
|
Optional<size_t> last_delimiter_index = input.find_last(DELIMITER);
|
||||||
|
if (last_delimiter_index.has_value()) {
|
||||||
|
for (; consumed < last_delimiter_index.value(); consumed++) {
|
||||||
|
if (!is_ascii(input[consumed]))
|
||||||
|
return Error::from_string_literal("Unexpected non-basic code point");
|
||||||
|
TRY(output.try_append(input[consumed]));
|
||||||
|
}
|
||||||
|
|
||||||
|
// if more than zero code points were consumed then consume one more
|
||||||
|
// (which will be the last delimiter)
|
||||||
|
if (last_delimiter_index.value() > 0) {
|
||||||
|
auto next = input[consumed++];
|
||||||
|
VERIFY(next == DELIMITER);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// while the input is not exhausted do begin
|
||||||
|
while (consumed < input.length()) {
|
||||||
|
// let oldi = i
|
||||||
|
Checked<u32> old_i = i;
|
||||||
|
|
||||||
|
// let w = 1
|
||||||
|
Checked<u32> w = 1;
|
||||||
|
|
||||||
|
// for k = base to infinity in steps of base do begin
|
||||||
|
for (size_t k = BASE;; k += BASE) {
|
||||||
|
// consume a code point, or fail if there was none to consume
|
||||||
|
if (consumed >= input.length())
|
||||||
|
return Error::from_string_literal("No more code points to consume");
|
||||||
|
auto code_point = input[consumed++];
|
||||||
|
|
||||||
|
// let digit = the code point's digit-value, fail if it has none
|
||||||
|
auto digit = digit_value_of_code_point(code_point);
|
||||||
|
if (!digit.has_value())
|
||||||
|
return Error::from_string_literal("Invalid base-36 digit");
|
||||||
|
|
||||||
|
// let i = i + digit * w, fail on overflow
|
||||||
|
i = i + Checked(digit.value()) * w;
|
||||||
|
if (i.has_overflow())
|
||||||
|
return Error::from_string_literal("Numeric overflow");
|
||||||
|
|
||||||
|
// let t = tmin if k <= bias {+ tmin}, or
|
||||||
|
// tmax if k >= bias + tmax, or k - bias otherwise
|
||||||
|
u32 t = k <= bias ? TMIN : (k >= bias + TMAX ? TMAX : k - bias);
|
||||||
|
|
||||||
|
// if digit < t then break
|
||||||
|
if (digit.value() < t)
|
||||||
|
break;
|
||||||
|
|
||||||
|
// let w = w * (base - t), fail on overflow
|
||||||
|
w = w * Checked(BASE - t);
|
||||||
|
if (w.has_overflow())
|
||||||
|
return Error::from_string_literal("Numeric overflow");
|
||||||
|
}
|
||||||
|
// let bias = adapt(i - oldi, length(output) + 1, test oldi is 0?)
|
||||||
|
bias = adapt((i - old_i).value(), output.size() + 1, !old_i);
|
||||||
|
|
||||||
|
// let n = n + i div (length(output) + 1), fail on overflow
|
||||||
|
n = n + Checked(static_cast<size_t>(i.value() / static_cast<u32>(output.size() + 1)));
|
||||||
|
if (n.has_overflow())
|
||||||
|
return Error::from_string_literal("Numeric overflow");
|
||||||
|
|
||||||
|
// let i = i mod (length(output) + 1)
|
||||||
|
i = i % Checked(static_cast<u32>(output.size() + 1));
|
||||||
|
|
||||||
|
// {if n is a basic code point then fail}
|
||||||
|
// NOTE: The full statement enclosed in braces (checking whether n is a basic code point) can be omitted if initial_n exceeds all basic code points
|
||||||
|
// (which is true for Punycode), because n is never less than initial_n.
|
||||||
|
VERIFY(!is_ascii(n.value()));
|
||||||
|
|
||||||
|
// insert n into output at position i
|
||||||
|
TRY(output.try_insert(i.value(), n.value()));
|
||||||
|
|
||||||
|
// increment i
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
|
||||||
|
StringBuilder builder;
|
||||||
|
TRY(builder.try_append(Utf32View(output.data(), output.size())));
|
||||||
|
return builder.to_string();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
15
Userland/Libraries/LibUnicode/Punycode.h
Normal file
15
Userland/Libraries/LibUnicode/Punycode.h
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (c) 2023, Simon Wanner <simon@skyrising.xyz>
|
||||||
|
*
|
||||||
|
* SPDX-License-Identifier: BSD-2-Clause
|
||||||
|
*/
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <AK/String.h>
|
||||||
|
|
||||||
|
namespace Unicode::Punycode {
|
||||||
|
|
||||||
|
ErrorOr<String> decode(StringView);
|
||||||
|
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user