Tests: Add regression tests for the LibCpp preprocessor

Similarly to the LibCpp parser regression tests, these tests run the
preprocessor on the .cpp test files under
Userland/LibCpp/Tests/preprocessor, and compare the output with existing
.txt ground truth files.

Meta/build-root-filesystem.sh

@@ -152,6 +152,7 @@ cp -r "$SERENITY_SOURCE_DIR"/Userland/Libraries/LibJS/Tests mnt/home/anon/js-tes
 cp -r "$SERENITY_SOURCE_DIR"/Userland/Libraries/LibWeb/Tests mnt/home/anon/web-tests
 cp -r "$SERENITY_SOURCE_DIR"/Userland/DevTools/HackStudio/LanguageServers/Cpp/Tests mnt/home/anon/cpp-tests/comprehension
 cp -r "$SERENITY_SOURCE_DIR"/Userland/Libraries/LibCpp/Tests/parser mnt/home/anon/cpp-tests/parser
+cp -r "$SERENITY_SOURCE_DIR"/Userland/Libraries/LibCpp/Tests/preprocessor mnt/home/anon/cpp-tests/preprocessor
 cp -r "$SERENITY_SOURCE_DIR"/Userland/Libraries/LibWasm/Tests mnt/home/anon/wasm-tests
 cp -r "$SERENITY_SOURCE_DIR"/Userland/Libraries/LibJS/Tests/test-common.js mnt/home/anon/wasm-tests
 chmod 700 mnt/root

Meta/check-newlines-at-eof.py

@@ -23,7 +23,8 @@ def run():
             "**/CMake*.txt",
             ":!:Kernel/FileSystem/ext2_fs.h",
             ':!:Userland/DevTools/HackStudio/LanguageServers/Cpp/Tests/*',
-            ':!:Userland/Libraries/LibCpp/Tests/parser/*'
+            ':!:Userland/Libraries/LibCpp/Tests/parser/*',
+            ':!:Userland/Libraries/LibCpp/Tests/preprocessor/*'
         ],
         check=True,
         capture_output=True

Meta/check-style.sh

@@ -12,7 +12,7 @@ cd "$script_path/.." || exit 1
 #  */
 GOOD_LICENSE_HEADER_PATTERN=$'^/\*\n( \* Copyright \(c\) [0-9]{4}(-[0-9]{4})?, .*\n)+ \*\n \* SPDX-License-Identifier: BSD-2-Clause\n \*/\n\n'
 BAD_LICENSE_HEADER_ERRORS=()
-LICENSE_HEADER_CHECK_EXCLUDES=(AK/Checked.h AK/Function.h Userland/Libraries/LibC/elf.h Userland/DevTools/HackStudio/LanguageServers/Cpp/Tests/* Userland/Libraries/LibCpp/Tests/parser/*)
+LICENSE_HEADER_CHECK_EXCLUDES=(AK/Checked.h AK/Function.h Userland/Libraries/LibC/elf.h Userland/DevTools/HackStudio/LanguageServers/Cpp/Tests/* Userland/Libraries/LibCpp/Tests/parser/* Userland/Libraries/LibCpp/Tests/preprocessor/*)
 
 # We check that "#pragma once" is present
 PRAGMA_ONCE_PATTERN='#pragma once'

Meta/lint-clang-format.sh

@@ -13,7 +13,8 @@ if [ "$#" -eq "1" ]; then
             ':!:Base' \
             ':!:Kernel/FileSystem/ext2_fs.h' \
             ':!:Userland/DevTools/HackStudio/LanguageServers/Cpp/Tests/*' \
-            ':!:Userland/Libraries/LibCpp/Tests/parser/*'
+            ':!:Userland/Libraries/LibCpp/Tests/parser/*' \
+            ':!:Userland/Libraries/LibCpp/Tests/preprocessor/*'
     )
 else
     files=()

Tests/LibCpp/test-cpp-preprocessor.cpp

@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2021, Itamar S. <itamar8910@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/LexicalPath.h>
+#include <LibCore/DirIterator.h>
+#include <LibCore/File.h>
+#include <LibCpp/Parser.h>
+#include <LibTest/TestCase.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+
+constexpr char TESTS_ROOT_DIR[] = "/home/anon/cpp-tests/preprocessor";
+
+static String read_all(const String& path)
+{
+    auto result = Core::File::open(path, Core::OpenMode::ReadOnly);
+    VERIFY(!result.is_error());
+    auto content = result.value()->read_all();
+    return { reinterpret_cast<const char*>(content.data()), content.size() };
+}
+
+TEST_CASE(test_regression)
+{
+    Core::DirIterator directory_iterator(TESTS_ROOT_DIR, Core::DirIterator::Flags::SkipDots);
+
+    while (directory_iterator.has_next()) {
+        auto file_path = directory_iterator.next_full_path();
+
+        auto path = LexicalPath { file_path };
+        if (!path.has_extension(".cpp"))
+            continue;
+
+        outln("Checking {}...", path.basename());
+
+        auto ast_file_path = String::formatted("{}.txt", file_path.substring(0, file_path.length() - sizeof(".cpp") + 1));
+
+        auto source = read_all(file_path);
+        auto target = read_all(ast_file_path);
+
+        StringView source_view(source);
+        Cpp::Preprocessor preprocessor(file_path, source_view);
+
+        auto target_lines = target.split_view('\n');
+        auto tokens = preprocessor.process_and_lex();
+
+        EXPECT_EQ(tokens.size(), target_lines.size());
+        for (size_t i = 0; i < tokens.size(); ++i) {
+            EXPECT_EQ(tokens[i].to_string(), target_lines[i]);
+        }
+    }
+}

Userland/Libraries/LibCpp/Tests/preprocessor/macro1.cpp

@@ -0,0 +1,2 @@
+#define ADD(x,y) x+y
+ADD(2,5);

Userland/Libraries/LibCpp/Tests/preprocessor/macro1.txt

@@ -0,0 +1,4 @@
+Integer  1:0-1:2 (2)
+Plus  1:0-1:2 (+)
+Integer  1:0-1:2 (5)
+Semicolon  1:8-1:8 (;)

Userland/Libraries/LibCpp/Tests/preprocessor/macro2.cpp

@@ -0,0 +1,3 @@
+#define M(x) String {x + "lo"}
+
+M("he" + "l")

Userland/Libraries/LibCpp/Tests/preprocessor/macro2.txt

@@ -0,0 +1,8 @@
+KnownType  2:0-2:0 (String)
+LeftCurly  2:0-2:0 ({)
+DoubleQuotedString  2:0-2:0 ("he")
+Plus  2:0-2:0 (+)
+DoubleQuotedString  2:0-2:0 ("l")
+Plus  2:0-2:0 (+)
+DoubleQuotedString  2:0-2:0 ("lo")
+RightCurly  2:0-2:0 (})

Userland/Libraries/LibCpp/Tests/preprocessor/macro3.cpp

@@ -0,0 +1,4 @@
+#define M(x, y, z) x y = z;
+
+M(Vector, vec, ({1,2}))
+

Userland/Libraries/LibCpp/Tests/preprocessor/macro3.txt

@@ -0,0 +1,11 @@
+KnownType  2:0-2:0 (Vector)
+Identifier  2:0-2:0 (vec)
+Equals  2:0-2:0 (=)
+LeftParen  2:0-2:0 (()
+LeftCurly  2:0-2:0 ({)
+Integer  2:0-2:0 (1)
+Comma  2:0-2:0 (,)
+Integer  2:0-2:0 (2)
+RightCurly  2:0-2:0 (})
+RightParen  2:0-2:0 ())
+Semicolon  2:0-2:0 (;)

Userland/Libraries/LibCpp/Tests/preprocessor/simple_define.cpp

@@ -0,0 +1,2 @@
+#define NUMBER 1337
+int x = NUMBER;

Userland/Libraries/LibCpp/Tests/preprocessor/simple_define.txt

@@ -0,0 +1,5 @@
+KnownType  1:0-1:2 (int)
+Identifier  1:4-1:4 (x)
+Equals  1:6-1:6 (=)
+Integer  1:8-1:13 (1337)
+Semicolon  1:14-1:14 (;)