Utilities: Add basic Package manager skeleton utility

Author: https://github.com/supercomputer7
Commit: https://github.com/SerenityOS/serenity/commit/ccaa423372
Pull-request: https://github.com/SerenityOS/serenity/pull/18944
Reviewed-by: https://github.com/ADKaster
Reviewed-by: https://github.com/alimpfard
Reviewed-by: https://github.com/gmta
Reviewed-by: https://github.com/kleinesfilmroellchen ✅
Reviewed-by: https://github.com/linusg
Reviewed-by: https://github.com/timschumi

Meta/build-root-filesystem.sh

@@ -193,6 +193,7 @@ chown -R 100:100 mnt/home/anon/Desktop
 echo "done"
 
 printf "installing shortcuts... "
+ln -sf /bin/PackageManager mnt/bin/pkg
 ln -sf Shell mnt/bin/sh
 ln -sf test mnt/bin/[
 ln -sf less mnt/bin/more

Userland/Utilities/CMakeLists.txt

@@ -171,3 +171,5 @@ foreach(name IN LISTS FUZZER_TARGETS)
     set_source_files_properties("${fuzz_source_file}" PROPERTIES COMPILE_OPTIONS "-Wno-missing-declarations")
     target_link_libraries(test-fuzz PRIVATE "${FUZZER_DEPENDENCIES_${name}}")
 endforeach()
+
+add_subdirectory(pkg)

Userland/Utilities/pkg/AvailablePort.cpp

@@ -0,0 +1,167 @@
+/*
+ * Copyright (c) 2023, Liav A. <liavalb@hotmail.co.il>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/Optional.h>
+#include <AK/Stream.h>
+#include <LibCore/ArgsParser.h>
+#include <LibCore/EventLoop.h>
+#include <LibCore/File.h>
+#include <LibCore/Proxy.h>
+#include <LibCore/System.h>
+#include <LibFileSystem/FileSystem.h>
+#include <LibHTTP/HttpResponse.h>
+#include <LibMain/Main.h>
+#include <LibMarkdown/Document.h>
+#include <LibMarkdown/Table.h>
+#include <LibProtocol/Request.h>
+#include <LibProtocol/RequestClient.h>
+
+#include "AvailablePort.h"
+#include "MarkdownTableFinder.h"
+
+#include <Shell/AST.h>
+#include <Shell/Formatter.h>
+#include <Shell/NodeVisitor.h>
+#include <Shell/PosixParser.h>
+#include <Shell/Shell.h>
+
+static bool is_installed(HashMap<String, InstalledPort>& installed_ports_database, StringView package_name)
+{
+    auto port = installed_ports_database.find(package_name);
+    return port != installed_ports_database.end();
+}
+
+static Optional<AvailablePort&> find_port_package(HashMap<String, AvailablePort>& available_ports, StringView package_name)
+{
+    auto port = available_ports.find(package_name);
+    if (port == available_ports.end())
+        return {};
+    return port->value;
+}
+
+ErrorOr<int> AvailablePort::query_details_for_package(HashMap<String, AvailablePort>& available_ports, HashMap<String, InstalledPort>& installed_ports, StringView package_name, bool verbose)
+{
+    auto possible_available_port = find_port_package(available_ports, package_name);
+    if (!possible_available_port.has_value()) {
+        outln("pkg: No match for queried name \"{}\"", package_name);
+        return 0;
+    }
+
+    auto& available_port = possible_available_port.release_value();
+
+    outln("{}: {}, {}", available_port.name(), available_port.version(), available_port.website());
+    if (verbose) {
+        out("Installed: ");
+        if (is_installed(installed_ports, package_name))
+            outln("Yes");
+        else
+            outln("No");
+    }
+    return 0;
+}
+
+static Optional<Markdown::Table::Column const&> get_column_in_table(Markdown::Table const& ports_table, StringView column_name)
+{
+    for (auto& column : ports_table.columns()) {
+        if (column_name == column.header.render_for_terminal())
+            return column;
+    }
+    return {};
+}
+
+ErrorOr<int> AvailablePort::update_available_ports_list_file()
+{
+    if (!Core::System::access("/usr/Ports/AvailablePorts.md"sv, R_OK).is_error() && FileSystem::remove("/usr/Ports/AvailablePorts.md"sv, FileSystem::RecursionMode::Disallowed).is_error()) {
+        outln("pkg: /usr/Ports/AvailablePorts.md exists, but can't delete it before updating it!");
+        return 0;
+    }
+    RefPtr<Protocol::Request> request;
+    auto protocol_client = TRY(Protocol::RequestClient::try_create());
+    HashMap<DeprecatedString, DeprecatedString, CaseInsensitiveStringTraits> request_headers;
+    Core::ProxyData proxy_data {};
+
+    auto output_stream = TRY(Core::File::open("/usr/Ports/AvailablePorts.md"sv, Core::File::OpenMode::ReadWrite, 0644));
+    Core::EventLoop loop;
+
+    URL url("https://raw.githubusercontent.com/SerenityOS/serenity/master/Ports/AvailablePorts.md");
+    DeprecatedString method = "GET";
+    outln("pkg: Syncing packages database...");
+    request = protocol_client->start_request(method, url, request_headers, ReadonlyBytes {}, proxy_data);
+    request->on_finish = [&](bool success, auto) {
+        if (!success)
+            outln("pkg: Syncing packages database failed.");
+        else
+            outln("pkg: Syncing packages database done.");
+        loop.quit(success ? 0 : 1);
+    };
+    request->stream_into(*output_stream);
+    return loop.exec();
+}
+
+static ErrorOr<String> extract_port_name_from_column(Markdown::Table::Column const& column, size_t row_index)
+{
+    struct : public Markdown::Visitor {
+        virtual RecursionDecision visit(Markdown::Text::LinkNode const& node) override
+        {
+            text_node = node.text.ptr();
+            return RecursionDecision::Break;
+        }
+
+    public:
+        Markdown::Text::Node* text_node;
+    } text_node_find_visitor;
+
+    column.rows[row_index].walk(text_node_find_visitor);
+    VERIFY(text_node_find_visitor.text_node);
+    StringBuilder string_builder;
+    text_node_find_visitor.text_node->render_for_raw_print(string_builder);
+    return string_builder.to_string();
+}
+
+ErrorOr<HashMap<String, AvailablePort>> AvailablePort::read_available_ports_list()
+{
+    auto available_ports_file = TRY(Core::File::open("/usr/Ports/AvailablePorts.md"sv, Core::File::OpenMode::Read, 0600));
+    auto content_buffer = TRY(available_ports_file->read_until_eof());
+    auto content = StringView(content_buffer);
+    auto document = Markdown::Document::parse(content);
+    auto finder = MarkdownTableFinder::analyze(*document);
+    if (finder.table_count() != 1)
+        return Error::from_string_literal("Invalid tables count in /usr/Ports/AvailablePorts.md");
+
+    VERIFY(finder.tables()[0]);
+    auto possible_port_name_column = get_column_in_table(*finder.tables()[0], "Port"sv);
+    auto possible_port_version_column = get_column_in_table(*finder.tables()[0], "Version"sv);
+    auto possible_port_website_column = get_column_in_table(*finder.tables()[0], "Website"sv);
+
+    if (!possible_port_name_column.has_value())
+        return Error::from_string_literal("pkg: Port column not found /usr/Ports/AvailablePorts.md");
+    if (!possible_port_version_column.has_value())
+        return Error::from_string_literal("pkg: Version column not found /usr/Ports/AvailablePorts.md");
+    if (!possible_port_website_column.has_value())
+        return Error::from_string_literal("pkg: Website column not found /usr/Ports/AvailablePorts.md");
+
+    auto& port_name_column = possible_port_name_column.release_value();
+    auto& port_version_column = possible_port_version_column.release_value();
+    auto& port_website_column = possible_port_website_column.release_value();
+
+    VERIFY(port_name_column.rows.size() == port_version_column.rows.size());
+    VERIFY(port_version_column.rows.size() == port_website_column.rows.size());
+
+    HashMap<String, AvailablePort> available_ports;
+    for (size_t port_index = 0; port_index < port_name_column.rows.size(); port_index++) {
+        auto name = TRY(extract_port_name_from_column(port_name_column, port_index));
+        auto website = TRY(String::from_deprecated_string(port_website_column.rows[port_index].render_for_terminal()));
+        if (website.is_empty())
+            website = TRY(String::from_utf8("n/a"sv));
+
+        auto version = TRY(String::from_deprecated_string(port_version_column.rows[port_index].render_for_terminal()));
+        if (version.is_empty())
+            version = TRY(String::from_utf8("n/a"sv));
+
+        TRY(available_ports.try_set(name, AvailablePort { name, version, website }));
+    }
+    return available_ports;
+}

Userland/Utilities/pkg/AvailablePort.h

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2023, Liav A. <liavalb@hotmail.co.il>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include "InstalledPort.h"
+#include <AK/HashMap.h>
+#include <AK/String.h>
+#include <AK/StringView.h>
+#include <AK/Types.h>
+
+class AvailablePort {
+public:
+    static ErrorOr<int> query_details_for_package(HashMap<String, AvailablePort>& available_ports, HashMap<String, InstalledPort>& installed_ports, StringView package_name, bool verbose);
+    static ErrorOr<HashMap<String, AvailablePort>> read_available_ports_list();
+    static ErrorOr<int> update_available_ports_list_file();
+
+    AvailablePort(String name, String version, String website)
+        : m_name(name)
+        , m_website(move(website))
+        , m_version(move(version))
+    {
+    }
+
+    StringView name() const { return m_name.bytes_as_string_view(); }
+    StringView version() const { return m_version.bytes_as_string_view(); }
+    StringView website() const { return m_website.bytes_as_string_view(); }
+
+private:
+    String m_name;
+    String m_website;
+    String m_version;
+};

Userland/Utilities/pkg/CMakeLists.txt

@@ -0,0 +1,15 @@
+serenity_component(
+    PackageManager
+    RECOMMENDED
+    TARGETS PackageManager
+    DEPENDS FileSystemAccessServer
+)
+
+set(SOURCES
+    AvailablePort.cpp
+    InstalledPort.cpp
+    main.cpp
+)
+
+serenity_app(PackageManager ICON app-assistant)
+target_link_libraries(PackageManager PRIVATE LibCore LibMain LibFileSystem LibProtocol LibHTTP LibMarkdown LibShell)

Userland/Utilities/pkg/InstalledPort.cpp

@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2023, Liav A. <liavalb@hotmail.co.il>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include "InstalledPort.h"
+#include <AK/Function.h>
+#include <LibCore/File.h>
+#include <LibCore/System.h>
+
+ErrorOr<HashMap<String, InstalledPort>> InstalledPort::read_ports_database()
+{
+    auto file = TRY(Core::File::open("/usr/Ports/installed.db"sv, Core::File::OpenMode::Read));
+    auto buffered_file = TRY(Core::InputBufferedFile::create(move(file)));
+    auto buffer = TRY(ByteBuffer::create_uninitialized(PAGE_SIZE));
+
+    HashMap<String, InstalledPort> ports;
+    while (TRY(buffered_file->can_read_line())) {
+        auto line = TRY(buffered_file->read_line(buffer));
+        if (line.is_empty()) {
+            continue;
+        } else if (line.starts_with("dependency"sv)) {
+            auto parts = line.split_view(' ');
+            VERIFY(parts.size() == 3);
+            auto type = InstalledPort::Type::Dependency;
+            // FIXME: Add versioning when printing these ports!
+            auto name = TRY(String::from_utf8(parts[2]));
+            TRY(ports.try_set(name, InstalledPort { TRY(String::from_utf8(parts[2])), type, TRY(String::from_utf8(""sv)) }));
+        } else if (line.starts_with("auto"sv)) {
+            auto parts = line.split_view(' ');
+            VERIFY(parts.size() == 3);
+            auto type = InstalledPort::Type::Auto;
+            auto name = TRY(String::from_utf8(parts[1]));
+            TRY(ports.try_set(name, InstalledPort { name, type, TRY(String::from_utf8(parts[2])) }));
+        } else if (line.starts_with("manual"sv)) {
+            auto parts = line.split_view(' ');
+            VERIFY(parts.size() == 3);
+            auto type = InstalledPort::Type::Manual;
+            auto name = TRY(String::from_utf8(parts[1]));
+            TRY(ports.try_set(name, InstalledPort { name, type, TRY(String::from_utf8(parts[2])) }));
+        } else {
+            return Error::from_string_literal("Unknown installed port type");
+        }
+    }
+    return ports;
+}
+
+ErrorOr<void> InstalledPort::for_each_by_type(HashMap<String, InstalledPort>& ports_database, InstalledPort::Type type, Function<ErrorOr<void>(InstalledPort const&)> callback)
+{
+    for (auto& port : ports_database) {
+        if (type == port.value.type())
+            TRY(callback(port.value));
+    }
+    return {};
+}

Userland/Utilities/pkg/InstalledPort.h

@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2023, Liav A. <liavalb@hotmail.co.il>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/HashMap.h>
+#include <AK/String.h>
+#include <AK/StringView.h>
+#include <AK/Types.h>
+
+class InstalledPort {
+public:
+    enum class Type {
+        Auto,
+        Dependency,
+        Manual,
+    };
+
+    static ErrorOr<HashMap<String, InstalledPort>> read_ports_database();
+    static ErrorOr<void> for_each_by_type(HashMap<String, InstalledPort>&, Type type, Function<ErrorOr<void>(InstalledPort const&)> callback);
+
+    InstalledPort(String name, Type type, String version)
+        : m_name(name)
+        , m_type(type)
+        , m_version(move(version))
+    {
+    }
+
+    Type type() const { return m_type; }
+    StringView type_as_string_view() const
+    {
+        if (m_type == Type::Auto)
+            return "Automatic"sv;
+        if (m_type == Type::Dependency)
+            return "Dependency"sv;
+        if (m_type == Type::Manual)
+            return "Manual"sv;
+        VERIFY_NOT_REACHED();
+    }
+
+    StringView name() const { return m_name.bytes_as_string_view(); }
+    StringView version() const { return m_version.bytes_as_string_view(); }
+
+private:
+    String m_name;
+    Type m_type;
+    String m_version;
+};

Userland/Utilities/pkg/MarkdownTableFinder.h

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2023, Liav A. <liavalb@hotmail.co.il>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/String.h>
+#include <AK/StringView.h>
+#include <AK/Types.h>
+#include <LibMarkdown/Document.h>
+#include <LibMarkdown/Visitor.h>
+
+class MarkdownTableFinder final : Markdown::Visitor {
+public:
+    ~MarkdownTableFinder() = default;
+
+    static MarkdownTableFinder analyze(Markdown::Document const& document)
+    {
+        MarkdownTableFinder finder;
+        document.walk(finder);
+        return finder;
+    }
+
+    size_t table_count() const { return m_tables.size(); }
+    Vector<Markdown::Table const*> const& tables() const { return m_tables; }
+
+private:
+    MarkdownTableFinder() { }
+
+    virtual RecursionDecision visit(Markdown::Table const& table) override
+    {
+        if (m_tables.size() >= 1)
+            return RecursionDecision::Break;
+        m_tables.append(&table);
+        return RecursionDecision::Recurse;
+    }
+
+    Vector<Markdown::Table const*> m_tables;
+};

Userland/Utilities/pkg/main.cpp

@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2023, Liav A. <liavalb@hotmail.co.il>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include "AvailablePort.h"
+#include "InstalledPort.h"
+#include <LibCore/ArgsParser.h>
+#include <LibCore/System.h>
+#include <LibMain/Main.h>
+#include <unistd.h>
+
+static void print_port_details(InstalledPort const& port)
+{
+    outln("{}, installed as {}, version {}", port.name(), port.type_as_string_view(), port.version());
+}
+
+static void print_port_details_without_version(InstalledPort const& port)
+{
+    outln("{}, installed as {}", port.name(), port.type_as_string_view());
+}
+
+ErrorOr<int> serenity_main(Main::Arguments arguments)
+{
+    TRY(Core::System::pledge("stdio recvfd thread unix rpath cpath wpath"));
+
+    TRY(Core::System::unveil("/tmp/session/%sid/portal/request", "rw"));
+    TRY(Core::System::unveil("/usr/Ports/installed.db"sv, "rwc"sv));
+    TRY(Core::System::unveil("/usr/Ports/AvailablePorts.md"sv, "rwc"sv));
+    TRY(Core::System::unveil("/res"sv, "r"sv));
+    TRY(Core::System::unveil("/usr/lib"sv, "r"sv));
+    TRY(Core::System::unveil(nullptr, nullptr));
+
+    bool verbose = false;
+    bool show_all_installed_ports = false;
+    bool show_all_dependency_ports = false;
+    bool update_packages_db = false;
+    StringView query_package {};
+
+    Core::ArgsParser args_parser;
+    args_parser.add_option(show_all_installed_ports, "Show all manually-installed ports", "list-manual-ports", 'l');
+    args_parser.add_option(show_all_dependency_ports, "Show all dependencies' ports", "list-dependency-ports", 'd');
+    args_parser.add_option(update_packages_db, "Sync/Update ports database", "update-ports-database", 'u');
+    args_parser.add_option(query_package, "Query ports database for package name", "query-package", 'q', "Package name to query");
+    args_parser.add_option(verbose, "Verbose", "verbose", 'v');
+    args_parser.parse(arguments);
+
+    if (!update_packages_db && !show_all_installed_ports && !show_all_dependency_ports && query_package.is_null()) {
+        outln("pkg: No action to be performed was specified.");
+        return 0;
+    }
+
+    HashMap<String, InstalledPort> installed_ports;
+    HashMap<String, AvailablePort> available_ports;
+    if (show_all_installed_ports || show_all_dependency_ports || !query_package.is_null()) {
+        installed_ports = TRY(InstalledPort::read_ports_database());
+    }
+
+    int return_value = 0;
+    if (update_packages_db) {
+        if (getuid() != 0) {
+            outln("pkg: Requires root to update packages database.");
+            return 1;
+        }
+        return_value = TRY(AvailablePort::update_available_ports_list_file());
+    }
+
+    if (!query_package.is_null()) {
+        if (Core::System::access("/usr/Ports/AvailablePorts.md"sv, R_OK).is_error()) {
+            outln("pkg: Please run this program with -u first!");
+            return 0;
+        }
+        available_ports = TRY(AvailablePort::read_available_ports_list());
+    }
+
+    if (show_all_installed_ports) {
+        outln("Manually-installed ports:");
+        TRY(InstalledPort::for_each_by_type(installed_ports, InstalledPort::Type::Manual, [](auto& port) -> ErrorOr<void> {
+            print_port_details(port);
+            return {};
+        }));
+    }
+
+    if (show_all_dependency_ports) {
+        outln("Dependencies-installed ports:");
+        TRY(InstalledPort::for_each_by_type(installed_ports, InstalledPort::Type::Dependency, [](auto& port) -> ErrorOr<void> {
+            // NOTE: Dependency entries don't specify versions, so we don't
+            // try to print it.
+            print_port_details_without_version(port);
+            return {};
+        }));
+    }
+
+    if (!query_package.is_null()) {
+        if (query_package.is_empty()) {
+            outln("pkg: Queried package name is empty.");
+            return 0;
+        }
+        return_value = TRY(AvailablePort::query_details_for_package(available_ports, installed_ports, query_package, verbose));
+    }
+
+    return return_value;
+}