LibCpp: Replace defined preprocessor values when parsing

Author: https://github.com/itamar8910
Commit: https://github.com/SerenityOS/serenity/commit/3658c4c567d
Pull-request: https://github.com/SerenityOS/serenity/pull/5755

Base/home/anon/Source/little/other.cpp

@@ -5,6 +5,7 @@ int func()
 {
     int x = 1;
     int y = 2;
+    INT_Z = 3;
     StructInHeader mystruct;
     printf("x: %d\n", x);
     printf("y: %d\n", y);

Base/home/anon/Source/little/other.h

@@ -2,6 +2,8 @@ int func();
 
 #define USE_VAR2
 
+#define INT_Z int z
+
 struct StructInHeader {
     int var1;
 #ifdef USE_VAR2

Userland/DevTools/HackStudio/LanguageServers/Cpp/ParserAutoComplete.cpp

@@ -64,9 +64,9 @@ OwnPtr<ParserAutoComplete::DocumentData> ParserAutoComplete::create_document_dat
     if (!document)
         return {};
     auto content = document->text();
-    auto document_data = make<DocumentData>(document->text(), file);
+    auto document_data = create_document_data(document->text(), file);
-    auto root = document_data->parser.parse();
+    auto root = document_data->parser().parse();
-    for (auto& path : document_data->preprocessor.included_paths()) {
+    for (auto& path : document_data->preprocessor().included_paths()) {
         get_or_create_document_data(document_path_from_include_path(path));
     }
 #ifdef CPP_LANGUAGE_SERVER_DEBUG
@@ -83,14 +83,6 @@ void ParserAutoComplete::set_document_data(const String& file, OwnPtr<DocumentDa
     m_documents.set(filedb().to_absolute_path(file), move(data));
 }
 
-ParserAutoComplete::DocumentData::DocumentData(String&& _text, const String& _filename)
-    : filename(_filename)
-    , text(move(_text))
-    , preprocessor(text.view())
-    , parser(preprocessor.process().view(), filename)
-{
-}
-
 Vector<GUI::AutocompleteProvider::Entry> ParserAutoComplete::get_suggestions(const String& file, const GUI::TextPosition& autocomplete_position)
 {
     Cpp::Position position { autocomplete_position.line(), autocomplete_position.column() > 0 ? autocomplete_position.column() - 1 : 0 };
@@ -102,7 +94,7 @@ Vector<GUI::AutocompleteProvider::Entry> ParserAutoComplete::get_suggestions(con
         return {};
 
     const auto& document = *document_ptr;
-    auto node = document.parser.node_at(position);
+    auto node = document.parser().node_at(position);
     if (!node) {
         dbgln_if(CPP_LANGUAGE_SERVER_DEBUG, "no node at position {}:{}", position.line, position.column);
         return {};
@@ -110,10 +102,10 @@ Vector<GUI::AutocompleteProvider::Entry> ParserAutoComplete::get_suggestions(con
 
     if (node->is_identifier()) {
         if (is_property(*node)) {
-            return autocomplete_property(document, (MemberExpression&)(*node->parent()), document.parser.text_of_node(*node));
+            return autocomplete_property(document, (MemberExpression&)(*node->parent()), document.parser().text_of_node(*node));
         }
 
-        return autocomplete_name(document, *node, document.parser.text_of_node(*node));
+        return autocomplete_name(document, *node, document.parser().text_of_node(*node));
     }
 
     if (is_empty_property(document, *node, position)) {
@@ -122,9 +114,9 @@ Vector<GUI::AutocompleteProvider::Entry> ParserAutoComplete::get_suggestions(con
     }
 
     String partial_text = String::empty();
-    auto containing_token = document.parser.token_at(position);
+    auto containing_token = document.parser().token_at(position);
     if (containing_token.has_value()) {
-        partial_text = document.parser.text_of_token(containing_token.value());
+        partial_text = document.parser().text_of_token(containing_token.value());
     }
 
     return autocomplete_name(document, *node, partial_text.view());
@@ -204,7 +196,7 @@ bool ParserAutoComplete::is_empty_property(const DocumentData& document, const A
 {
     if (!node.is_member_expression())
         return false;
-    auto previous_token = document.parser.token_at(autocomplete_position);
+    auto previous_token = document.parser().token_at(autocomplete_position);
     if (!previous_token.has_value())
         return false;
     return previous_token.value().type() == Token::Type::Dot;
@@ -276,14 +268,14 @@ Vector<ParserAutoComplete::PropertyInfo> ParserAutoComplete::properties_of_type(
 NonnullRefPtrVector<Declaration> ParserAutoComplete::get_declarations_in_outer_scope_including_headers(const DocumentData& document) const
 {
     NonnullRefPtrVector<Declaration> declarations;
-    for (auto& include : document.preprocessor.included_paths()) {
+    for (auto& include : document.preprocessor().included_paths()) {
         document_path_from_include_path(include);
         auto included_document = get_document_data(document_path_from_include_path(include));
         if (!included_document)
             continue;
         declarations.append(get_declarations_in_outer_scope_including_headers(*included_document));
     }
-    for (auto& decl : document.parser.root_node()->declarations()) {
+    for (auto& decl : document.parser().root_node()->declarations()) {
         declarations.append(decl);
     }
     return declarations;
@@ -336,7 +328,7 @@ Optional<GUI::AutocompleteProvider::ProjectLocation> ParserAutoComplete::find_de
         return {};
 
     const auto& document = *document_ptr;
-    auto node = document.parser.node_at(Cpp::Position { identifier_position.line(), identifier_position.column() });
+    auto node = document.parser().node_at(Cpp::Position { identifier_position.line(), identifier_position.column() });
     if (!node) {
         dbgln_if(CPP_LANGUAGE_SERVER_DEBUG, "no node at position {}:{}", identifier_position.line(), identifier_position.column());
         return {};
@@ -350,7 +342,7 @@ Optional<GUI::AutocompleteProvider::ProjectLocation> ParserAutoComplete::find_de
 
 RefPtr<Declaration> ParserAutoComplete::find_declaration_of(const DocumentData& document_data, const ASTNode& node) const
 {
-    dbgln_if(CPP_LANGUAGE_SERVER_DEBUG, "find_declaration_of: {}", document_data.parser.text_of_node(node));
+    dbgln_if(CPP_LANGUAGE_SERVER_DEBUG, "find_declaration_of: {}", document_data.parser().text_of_node(node));
     auto declarations = get_available_declarations(document_data, node);
     for (auto& decl : declarations) {
         if (node.is_identifier() && decl.is_variable_or_parameter_declaration()) {
@@ -379,10 +371,10 @@ RefPtr<Declaration> ParserAutoComplete::find_declaration_of(const DocumentData&
 void ParserAutoComplete::update_declared_symbols(const DocumentData& document)
 {
     Vector<GUI::AutocompleteProvider::Declaration> declarations;
-    for (auto& decl : document.parser.root_node()->declarations()) {
+    for (auto& decl : document.parser().root_node()->declarations()) {
-        declarations.append({ decl.name(), { document.filename, decl.start().line, decl.start().column }, type_of_declaration(decl) });
+        declarations.append({ decl.name(), { document.filename(), decl.start().line, decl.start().column }, type_of_declaration(decl) });
     }
-    set_declarations_of_document(document.filename, move(declarations));
+    set_declarations_of_document(document.filename(), move(declarations));
 }
 
 GUI::AutocompleteProvider::DeclarationType ParserAutoComplete::type_of_declaration(const Declaration& decl)
@@ -398,4 +390,29 @@ GUI::AutocompleteProvider::DeclarationType ParserAutoComplete::type_of_declarati
     return GUI::AutocompleteProvider::DeclarationType::Variable;
 }
 
+OwnPtr<ParserAutoComplete::DocumentData> ParserAutoComplete::create_document_data(String&& text, const String& filename)
+{
+    auto document_data = make<DocumentData>();
+    document_data->m_filename = move(filename);
+    document_data->m_text = move(text);
+    document_data->m_preprocessor = make<Preprocessor>(document_data->text());
+    document_data->preprocessor().process();
+
+    Preprocessor::Definitions all_definitions;
+    for (auto item : document_data->preprocessor().definitions())
+        all_definitions.set(move(item.key), move(item.value));
+
+    for (auto include : document_data->preprocessor().included_paths()) {
+
+        auto included_document = get_or_create_document_data(document_path_from_include_path(include));
+        if (!included_document)
+            continue;
+        for (auto item : included_document->parser().definitions())
+            all_definitions.set(move(item.key), move(item.value));
+    }
+
+    document_data->m_parser = make<Parser>(document_data->preprocessor().processed_text(), filename, move(all_definitions));
+    return document_data;
+}
+
 }

Userland/DevTools/HackStudio/LanguageServers/Cpp/ParserAutoComplete.h

@@ -52,11 +52,33 @@ public:
 
 private:
     struct DocumentData {
-        DocumentData(String&& text, const String& filename);
+        const String& filename() const { return m_filename; }
-        String filename;
+        const String& text() const { return m_text; }
-        String text;
+        const Preprocessor& preprocessor() const
-        Preprocessor preprocessor;
+        {
-        Parser parser;
+            VERIFY(m_preprocessor);
+            return *m_preprocessor;
+        }
+        Preprocessor& preprocessor()
+        {
+            VERIFY(m_preprocessor);
+            return *m_preprocessor;
+        }
+        const Parser& parser() const
+        {
+            VERIFY(m_parser);
+            return *m_parser;
+        }
+        Parser& parser()
+        {
+            VERIFY(m_parser);
+            return *m_parser;
+        }
+
+        String m_filename;
+        String m_text;
+        OwnPtr<Preprocessor> m_preprocessor;
+        OwnPtr<Parser> m_parser;
     };
 
     Vector<GUI::AutocompleteProvider::Entry> autocomplete_property(const DocumentData&, const MemberExpression&, const StringView partial_text) const;
@@ -85,6 +107,8 @@ private:
     void update_declared_symbols(const DocumentData&);
     GUI::AutocompleteProvider::DeclarationType type_of_declaration(const Declaration&);
 
+    OwnPtr<DocumentData> create_document_data(String&& text, const String& filename);
+
     HashMap<String, OwnPtr<DocumentData>> m_documents;
 };
 

Userland/Libraries/LibCpp/Parser.cpp

@@ -37,26 +37,42 @@
 
 namespace Cpp {
 
-Parser::Parser(const StringView& program, const String& filename)
+Parser::Parser(const StringView& program, const String& filename, Preprocessor::Definitions&& definitions)
     : m_program(program)
+    , m_definitions(move(definitions))
     , m_lines(m_program.split_view("\n", true))
     , m_filename(filename)
 {
-    Lexer lexer(m_program);
+    initialize_program_tokens();
-    for (auto& token : lexer.lex()) {
-        if (token.m_type == Token::Type::Whitespace)
-            continue;
-        m_tokens.append(move(token));
-    }
 #if CPP_DEBUG
     dbgln("Program:");
     dbgln("{}", m_program);
     dbgln("Tokens:");
     for (auto& token : m_tokens) {
-        dbgln("{} ({}:{}-{}:{})", token.to_string(), token.start().line, token.start().column, token.end().line, token.end().column);
+        StringView text;
+        if (token.m_start.line != token.m_end.line || token.m_start.column > token.m_end.column)
+            text = {};
+        else
+            text = text_of_token(token);
+        dbgln("{}  {}:{}-{}:{} ({})", token.to_string(), token.start().line, token.start().column, token.end().line, token.end().column, text);
     }
 #endif
 }
+void Parser::initialize_program_tokens()
+{
+    Lexer lexer(m_program);
+    for (auto& token : lexer.lex()) {
+        if (token.m_type == Token::Type::Whitespace)
+            continue;
+        if (token.m_type == Token::Type::Identifier) {
+            if (auto defined_value = m_definitions.find(text_of_token(token)); defined_value != m_definitions.end()) {
+                add_tokens_for_preprocessor(token, defined_value->value);
+                continue;
+            }
+        }
+        m_tokens.append(move(token));
+    }
+}
 
 NonnullRefPtr<TranslationUnit> Parser::parse()
 {
@@ -1097,5 +1113,18 @@ bool Parser::match_ellipsis()
         return false;
     return peek().type() == Token::Type::Dot && peek().type() == Token::Type::Dot && peek().type() == Token::Type::Dot;
 }
+void Parser::add_tokens_for_preprocessor(Token& replaced_token, Preprocessor::DefinedValue& definition)
+{
+    if (!definition.value.has_value())
+        return;
+    Lexer lexer(definition.value.value());
+    for (auto token : lexer.lex()) {
+        if (token.type() == Token::Type::Whitespace)
+            continue;
+        token.m_start = replaced_token.start();
+        token.m_end = replaced_token.end();
+        m_tokens.append(move(token));
+    }
+}
 
 }

Userland/Libraries/LibCpp/Parser.h

@@ -27,14 +27,17 @@
 #pragma once
 
 #include "AK/NonnullRefPtr.h"
+#include <AK/Noncopyable.h>
 #include "AST.h"
+#include "Preprocessor.h"
 #include <LibCpp/Lexer.h>
 
 namespace Cpp {
 
 class Parser final {
+    AK_MAKE_NONCOPYABLE(Parser);
 public:
-    explicit Parser(const StringView& program, const String& filename);
+    explicit Parser(const StringView& program, const String& filename, Preprocessor::Definitions&& = {});
     ~Parser() = default;
 
     NonnullRefPtr<TranslationUnit> parse();
@@ -48,6 +51,7 @@ public:
     StringView text_of_token(const Cpp::Token& token) const;
     void print_tokens() const;
     Vector<String> errors() const { return m_errors; }
+    const Preprocessor::Definitions& definitions() const {return m_definitions;}
 
 private:
     enum class DeclarationType {
@@ -151,7 +155,15 @@ private:
         return node;
     }
 
+    bool match_attribute_specification();
+    void consume_attribute_specification();
+    bool match_ellipsis();
+    void initialize_program_tokens();
+    void add_tokens_for_preprocessor(Token& replaced_token, Preprocessor::DefinedValue&);
+    Vector<StringView> parse_type_qualifiers();
+
     StringView m_program;
+    Preprocessor::Definitions m_definitions;
     Vector<StringView> m_lines;
     String m_filename;
     Vector<Token> m_tokens;
@@ -160,10 +172,6 @@ private:
     RefPtr<TranslationUnit> m_root_node;
     NonnullRefPtrVector<ASTNode> m_nodes;
     Vector<String> m_errors;
-    Vector<StringView> parse_type_qualifiers();
-    bool match_attribute_specification();
-    void consume_attribute_specification();
-    bool match_ellipsis();
 };
 
 }

Userland/Libraries/LibCpp/Preprocessor.cpp

@@ -182,4 +182,10 @@ void Preprocessor::handle_preprocessor_line(const StringView& line)
     VERIFY_NOT_REACHED();
 }
 
+const String& Preprocessor::processed_text()
+{
+    VERIFY(!m_processed_text.is_null());
+    return m_processed_text;
+}
+
 };

Userland/Libraries/LibCpp/Preprocessor.h

@@ -41,14 +41,17 @@ public:
     const String& processed_text();
     Vector<StringView> included_paths() const { return m_included_paths; }
 
-private:
     struct DefinedValue {
         Optional<StringView> value;
     };
+    using Definitions = HashMap<StringView, DefinedValue>;
 
+    const Definitions& definitions() const { return m_definitions; }
+
+private:
     void handle_preprocessor_line(const StringView&);
 
-    HashMap<StringView, DefinedValue> m_definitions;
+    Definitions m_definitions;
     const StringView m_program;
     StringBuilder m_builder;
     Vector<StringView> m_lines;