mirror of
https://github.com/LadybirdBrowser/ladybird.git
synced 2024-09-20 09:49:15 +03:00
LibCpp: Add lex_iterable() method to the Lexer
This allows us to collect the tokens iteratively instead of having to lex the whole program and then get a tokens vector.
This commit is contained in:
parent
7a4a32b112
commit
606e05852f
Notes:
sideshowbarker
2024-07-18 05:25:50 +09:00
Author: https://github.com/itamar8910 Commit: https://github.com/SerenityOS/serenity/commit/606e05852f6 Pull-request: https://github.com/SerenityOS/serenity/pull/9528 Reviewed-by: https://github.com/alimpfard
@ -6,6 +6,7 @@
|
|||||||
|
|
||||||
#include "Lexer.h"
|
#include "Lexer.h"
|
||||||
#include <AK/CharacterTypes.h>
|
#include <AK/CharacterTypes.h>
|
||||||
|
#include <AK/Function.h>
|
||||||
#include <AK/HashTable.h>
|
#include <AK/HashTable.h>
|
||||||
#include <AK/StdLibExtras.h>
|
#include <AK/StdLibExtras.h>
|
||||||
#include <AK/String.h>
|
#include <AK/String.h>
|
||||||
@ -205,15 +206,13 @@ static bool is_known_type(StringView const& string)
|
|||||||
return types.contains(string);
|
return types.contains(string);
|
||||||
}
|
}
|
||||||
|
|
||||||
Vector<Token> Lexer::lex()
|
void Lexer::lex_impl(Function<void(Token)> callback)
|
||||||
{
|
{
|
||||||
Vector<Token> tokens;
|
|
||||||
|
|
||||||
size_t token_start_index = 0;
|
size_t token_start_index = 0;
|
||||||
Position token_start_position;
|
Position token_start_position;
|
||||||
|
|
||||||
auto emit_single_char_token = [&](auto type) {
|
auto emit_single_char_token = [&](auto type) {
|
||||||
tokens.empend(type, m_position, m_position, m_input.substring_view(m_index, 1));
|
callback(Token(type, m_position, m_position, m_input.substring_view(m_index, 1)));
|
||||||
consume();
|
consume();
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -224,7 +223,7 @@ Vector<Token> Lexer::lex()
|
|||||||
auto commit_token = [&](auto type) {
|
auto commit_token = [&](auto type) {
|
||||||
if (m_options.ignore_whitespace && type == Token::Type::Whitespace)
|
if (m_options.ignore_whitespace && type == Token::Type::Whitespace)
|
||||||
return;
|
return;
|
||||||
tokens.empend(type, token_start_position, m_previous_position, m_input.substring_view(token_start_index, m_index - token_start_index));
|
callback(Token(type, token_start_position, m_previous_position, m_input.substring_view(token_start_index, m_index - token_start_index)));
|
||||||
};
|
};
|
||||||
|
|
||||||
auto emit_token_equals = [&](auto type, auto equals_type) {
|
auto emit_token_equals = [&](auto type, auto equals_type) {
|
||||||
@ -784,6 +783,14 @@ Vector<Token> Lexer::lex()
|
|||||||
dbgln("Unimplemented token character: {}", ch);
|
dbgln("Unimplemented token character: {}", ch);
|
||||||
emit_single_char_token(Token::Type::Unknown);
|
emit_single_char_token(Token::Type::Unknown);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Vector<Token> Lexer::lex()
|
||||||
|
{
|
||||||
|
Vector<Token> tokens;
|
||||||
|
lex_impl([&](auto token) {
|
||||||
|
tokens.append(move(token));
|
||||||
|
});
|
||||||
return tokens;
|
return tokens;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -17,12 +17,15 @@ public:
|
|||||||
explicit Lexer(StringView const&, size_t start_line = 0);
|
explicit Lexer(StringView const&, size_t start_line = 0);
|
||||||
|
|
||||||
Vector<Token> lex();
|
Vector<Token> lex();
|
||||||
|
template<typename Callback>
|
||||||
|
void lex_iterable(Callback);
|
||||||
|
|
||||||
void set_ignore_whitespace(bool value) { m_options.ignore_whitespace = value; }
|
void set_ignore_whitespace(bool value) { m_options.ignore_whitespace = value; }
|
||||||
|
|
||||||
private:
|
private:
|
||||||
char peek(size_t offset = 0) const;
|
char peek(size_t offset = 0) const;
|
||||||
char consume();
|
char consume();
|
||||||
|
void lex_impl(Function<void(Token)>);
|
||||||
|
|
||||||
StringView m_input;
|
StringView m_input;
|
||||||
size_t m_index { 0 };
|
size_t m_index { 0 };
|
||||||
@ -34,4 +37,10 @@ private:
|
|||||||
} m_options;
|
} m_options;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
template<typename Callback>
|
||||||
|
void Lexer::lex_iterable(Callback callback)
|
||||||
|
{
|
||||||
|
return lex_impl(move(callback));
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user