Shell: Add (basic) support for history event designators

Closes #4888

Userland/Shell/AST.cpp

@@ -1201,6 +1201,157 @@ Glob::~Glob()
 {
 }
 
+void HistoryEvent::dump(int level) const
+{
+    Node::dump(level);
+    print_indented("Event Selector", level + 1);
+    switch (m_selector.event.kind) {
+    case HistorySelector::EventKind::IndexFromStart:
+        print_indented("IndexFromStart", level + 2);
+        break;
+    case HistorySelector::EventKind::IndexFromEnd:
+        print_indented("IndexFromEnd", level + 2);
+        break;
+    case HistorySelector::EventKind::ContainingStringLookup:
+        print_indented("ContainingStringLookup", level + 2);
+        break;
+    case HistorySelector::EventKind::StartingStringLookup:
+        print_indented("StartingStringLookup", level + 2);
+        break;
+    }
+    print_indented(String::formatted("{}({})", m_selector.event.index, m_selector.event.text), level + 3);
+
+    print_indented("Word Selector", level + 1);
+    auto print_word_selector = [&](const HistorySelector::WordSelector& selector) {
+        switch (selector.kind) {
+        case HistorySelector::WordSelectorKind::Index:
+            print_indented(String::formatted("Index {}", selector.selector), level + 3);
+            break;
+        case HistorySelector::WordSelectorKind::Last:
+            print_indented(String::formatted("Last"), level + 3);
+            break;
+        }
+    };
+
+    if (m_selector.word_selector_range.end.has_value()) {
+        print_indented("Range Start", level + 2);
+        print_word_selector(m_selector.word_selector_range.start);
+        print_indented("Range End", level + 2);
+        print_word_selector(m_selector.word_selector_range.end.value());
+    } else {
+        print_indented("Direct Address", level + 2);
+        print_word_selector(m_selector.word_selector_range.start);
+    }
+}
+
+RefPtr<Value> HistoryEvent::run(RefPtr<Shell> shell)
+{
+    if (!shell)
+        return create<AST::ListValue>({});
+
+    auto editor = shell->editor();
+    if (!editor) {
+        shell->raise_error(Shell::ShellError::EvaluatedSyntaxError, "No history available!", position());
+        return create<AST::ListValue>({});
+    }
+    auto& history = editor->history();
+
+    // FIXME: Implement reverse iterators and find()?
+    auto find_reverse = [](auto it_start, auto it_end, auto finder) {
+        auto it = it_end;
+        while (it != it_start) {
+            --it;
+            if (finder(*it))
+                return it;
+        }
+        return it_end;
+    };
+    // First, resolve the event itself.
+    String resolved_history;
+    switch (m_selector.event.kind) {
+    case HistorySelector::EventKind::IndexFromStart:
+        if (m_selector.event.index >= history.size()) {
+            shell->raise_error(Shell::ShellError::EvaluatedSyntaxError, "History event index out of bounds", m_selector.event.text_position);
+            return create<AST::ListValue>({});
+        }
+        resolved_history = history[m_selector.event.index].entry;
+        break;
+    case HistorySelector::EventKind::IndexFromEnd:
+        if (m_selector.event.index >= history.size()) {
+            shell->raise_error(Shell::ShellError::EvaluatedSyntaxError, "History event index out of bounds", m_selector.event.text_position);
+            return create<AST::ListValue>({});
+        }
+        resolved_history = history[history.size() - m_selector.event.index - 1].entry;
+        break;
+    case HistorySelector::EventKind::ContainingStringLookup: {
+        auto it = find_reverse(history.begin(), history.end(), [&](auto& entry) { return entry.entry.contains(m_selector.event.text); });
+        if (it.is_end()) {
+            shell->raise_error(Shell::ShellError::EvaluatedSyntaxError, "History event did not match any entry", m_selector.event.text_position);
+            return create<AST::ListValue>({});
+        }
+        resolved_history = it->entry;
+        break;
+    }
+    case HistorySelector::EventKind::StartingStringLookup: {
+        auto it = find_reverse(history.begin(), history.end(), [&](auto& entry) { return entry.entry.starts_with(m_selector.event.text); });
+        if (it.is_end()) {
+            shell->raise_error(Shell::ShellError::EvaluatedSyntaxError, "History event did not match any entry", m_selector.event.text_position);
+            return create<AST::ListValue>({});
+        }
+        resolved_history = it->entry;
+        break;
+    }
+    }
+
+    // Then, split it up to "words".
+    auto nodes = Parser { resolved_history }.parse_as_multiple_expressions();
+
+    // Now take the "words" as described by the word selectors.
+    bool is_range = m_selector.word_selector_range.end.has_value();
+    if (is_range) {
+        auto start_index = m_selector.word_selector_range.start.resolve(nodes.size());
+        auto end_index = m_selector.word_selector_range.end->resolve(nodes.size());
+        if (start_index >= nodes.size()) {
+            shell->raise_error(Shell::ShellError::EvaluatedSyntaxError, "History word index out of bounds", m_selector.word_selector_range.start.position);
+            return create<AST::ListValue>({});
+        }
+        if (end_index >= nodes.size()) {
+            shell->raise_error(Shell::ShellError::EvaluatedSyntaxError, "History word index out of bounds", m_selector.word_selector_range.end->position);
+            return create<AST::ListValue>({});
+        }
+
+        decltype(nodes) resolved_nodes;
+        resolved_nodes.append(nodes.data() + start_index, end_index - start_index + 1);
+        NonnullRefPtr<AST::Node> list = create<AST::ListConcatenate>(position(), move(resolved_nodes));
+        return list->run(shell);
+    }
+
+    auto index = m_selector.word_selector_range.start.resolve(nodes.size());
+    if (index >= nodes.size()) {
+        shell->raise_error(Shell::ShellError::EvaluatedSyntaxError, "History word index out of bounds", m_selector.word_selector_range.start.position);
+        return create<AST::ListValue>({});
+    }
+    return nodes[index].run(shell);
+}
+
+void HistoryEvent::highlight_in_editor(Line::Editor& editor, Shell&, HighlightMetadata metadata)
+{
+    Line::Style style { Line::Style::Foreground(Line::Style::XtermColor::Green) };
+    if (metadata.is_first_in_list)
+        style.unify_with({ Line::Style::Bold });
+    editor.stylize({ m_position.start_offset, m_position.end_offset }, move(style));
+}
+
+HistoryEvent::HistoryEvent(Position position, HistorySelector selector)
+    : Node(move(position))
+    , m_selector(move(selector))
+{
+}
+
+HistoryEvent::~HistoryEvent()
+{
+}
+
 void Execute::dump(int level) const
 {
     Node::dump(level);

Userland/Shell/AST.h

@@ -454,7 +454,6 @@ public:
 
     enum class Kind : u32 {
         And,
-        ListConcatenate,
         Background,
         BarewordLiteral,
         BraceExpansion,
@@ -464,15 +463,18 @@ public:
         CommandLiteral,
         Comment,
         ContinuationControl,
-        DynamicEvaluate,
         DoubleQuotedString,
-        Fd2FdRedirection,
-        FunctionDeclaration,
-        ForLoop,
-        Glob,
+        DynamicEvaluate,
         Execute,
+        Fd2FdRedirection,
+        ForLoop,
+        FunctionDeclaration,
+        Glob,
+        HistoryEvent,
         IfCond,
         Join,
+        Juxtaposition,
+        ListConcatenate,
         MatchExpr,
         Or,
         Pipe,
@@ -480,12 +482,11 @@ public:
         ReadRedirection,
         ReadWriteRedirection,
         Sequence,
-        Subshell,
         SimpleVariable,
         SpecialVariable,
-        Juxtaposition,
         StringLiteral,
         StringPartCompose,
+        Subshell,
         SyntaxError,
         Tilde,
         VariableDeclarations,
@@ -881,6 +882,62 @@ private:
     String m_text;
 };
 
+struct HistorySelector {
+    enum EventKind {
+        IndexFromStart,
+        IndexFromEnd,
+        StartingStringLookup,
+        ContainingStringLookup,
+    };
+    enum WordSelectorKind {
+        Index,
+        Last,
+    };
+
+    struct {
+        EventKind kind { IndexFromStart };
+        size_t index { 0 };
+        Position text_position;
+        String text;
+    } event;
+
+    struct WordSelector {
+        WordSelectorKind kind { Index };
+        size_t selector { 0 };
+        Position position;
+
+        size_t resolve(size_t size) const
+        {
+            if (kind == Index)
+                return selector;
+            if (kind == Last)
+                return size - 1;
+            ASSERT_NOT_REACHED();
+        }
+    };
+    struct {
+        WordSelector start;
+        Optional<WordSelector> end;
+    } word_selector_range;
+};
+
+class HistoryEvent final : public Node {
+public:
+    HistoryEvent(Position, HistorySelector);
+    virtual ~HistoryEvent();
+    virtual void visit(NodeVisitor& visitor) override { visitor.visit(this); }
+
+    const HistorySelector& selector() const { return m_selector; }
+
+private:
+    NODE(HistoryEvent);
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(RefPtr<Shell>) override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+
+    HistorySelector m_selector;
+};
+
 class Execute final : public Node {
 public:
     Execute(Position, NonnullRefPtr<Node>, bool capture_stdout = false);

Userland/Shell/Forward.h

@@ -54,6 +54,7 @@ class Fd2FdRedirection;
 class FunctionDeclaration;
 class ForLoop;
 class Glob;
+class HistoryEvent;
 class Execute;
 class IfCond;
 class Join;

Userland/Shell/NodeVisitor.cpp

@@ -121,6 +121,10 @@ void NodeVisitor::visit(const AST::Glob*)
 {
 }
 
+void NodeVisitor::visit(const AST::HistoryEvent*)
+{
+}
+
 void NodeVisitor::visit(const AST::Execute* node)
 {
     node->command()->visit(*this);

Userland/Shell/NodeVisitor.h

@@ -50,6 +50,7 @@ public:
     virtual void visit(const AST::FunctionDeclaration*);
     virtual void visit(const AST::ForLoop*);
     virtual void visit(const AST::Glob*);
+    virtual void visit(const AST::HistoryEvent*);
     virtual void visit(const AST::Execute*);
     virtual void visit(const AST::IfCond*);
     virtual void visit(const AST::Join*);

Userland/Shell/Parser.cpp

@@ -25,6 +25,8 @@
  */
 
 #include "Parser.h"
+#include "Shell.h"
+#include <AK/AllOf.h>
 #include <AK/TemporaryChange.h>
 #include <ctype.h>
 #include <stdio.h>
@@ -114,11 +116,6 @@ static constexpr bool is_whitespace(char c)
     return c == ' ' || c == '\t';
 }
 
-static constexpr bool is_word_character(char c)
-{
-    return (c <= '9' && c >= '0') || (c <= 'Z' && c >= 'A') || (c <= 'z' && c >= 'a') || c == '_';
-}
-
 static constexpr bool is_digit(char c)
 {
     return c <= '9' && c >= '0';
@@ -157,6 +154,28 @@ RefPtr<AST::Node> Parser::parse()
     return toplevel;
 }
 
+RefPtr<AST::Node> Parser::parse_as_single_expression()
+{
+    auto input = Shell::escape_token_for_double_quotes(m_input);
+    Parser parser { input };
+    return parser.parse_expression();
+}
+
+NonnullRefPtrVector<AST::Node> Parser::parse_as_multiple_expressions()
+{
+    NonnullRefPtrVector<AST::Node> nodes;
+    for (;;) {
+        consume_while(is_whitespace);
+        auto node = parse_expression();
+        if (!node)
+            node = parse_redirection();
+        if (!node)
+            return nodes;
+        nodes.append(node.release_nonnull());
+    }
+    return nodes;
+}
+
 RefPtr<AST::Node> Parser::parse_toplevel()
 {
     auto rule_start = push_start();
@@ -1053,7 +1072,7 @@ RefPtr<AST::Node> Parser::parse_expression()
     if (strchr("&|)} ;<>\n", starting_char) != nullptr)
         return nullptr;
 
-    if (m_is_in_brace_expansion_spec && starting_char == ',')
+    if (m_extra_chars_not_allowed_in_barewords.contains_slow(starting_char))
         return nullptr;
 
     if (m_is_in_brace_expansion_spec && next_is(".."))
@@ -1088,6 +1107,11 @@ RefPtr<AST::Node> Parser::parse_expression()
         return read_concat(create<AST::CastToList>(move(list))); // Cast To List
     }
 
+    if (starting_char == '!') {
+        if (auto designator = parse_history_designator())
+            return designator;
+    }
+
     if (auto composite = parse_string_composite())
         return read_concat(composite.release_nonnull());
 
@@ -1329,6 +1353,126 @@ RefPtr<AST::Node> Parser::parse_evaluate()
     return inner;
 }
 
+RefPtr<AST::Node> Parser::parse_history_designator()
+{
+    auto rule_start = push_start();
+
+    ASSERT(peek() == '!');
+    consume();
+
+    // Event selector
+    AST::HistorySelector selector;
+    RefPtr<AST::Node> syntax_error;
+    selector.event.kind = AST::HistorySelector::EventKind::StartingStringLookup;
+    selector.event.text_position = { m_offset, m_offset, m_line, m_line };
+    selector.word_selector_range = {
+        { AST::HistorySelector::WordSelectorKind::Index, 0, { m_offset, m_offset, m_line, m_line } },
+        AST::HistorySelector::WordSelector {
+            AST::HistorySelector::WordSelectorKind::Last, 0, { m_offset, m_offset, m_line, m_line } },
+    };
+
+    switch (peek()) {
+    case '!':
+        consume();
+        selector.event.kind = AST::HistorySelector::EventKind::IndexFromEnd;
+        selector.event.index = 0;
+        selector.event.text = "!";
+        break;
+    case '?':
+        consume();
+        selector.event.kind = AST::HistorySelector::EventKind::ContainingStringLookup;
+        [[fallthrough]];
+    default: {
+        TemporaryChange chars_change { m_extra_chars_not_allowed_in_barewords, { ':' } };
+
+        auto bareword = parse_bareword();
+        if (!bareword || !bareword->is_bareword()) {
+            restore_to(*rule_start);
+            return nullptr;
+        }
+
+        selector.event.text = static_ptr_cast<AST::BarewordLiteral>(bareword)->text();
+        selector.event.text_position = (bareword ?: syntax_error)->position();
+        auto it = selector.event.text.begin();
+        bool is_negative = false;
+        if (*it == '-') {
+            ++it;
+            is_negative = true;
+        }
+        if (it != selector.event.text.end() && AK::all_of(it, selector.event.text.end(), is_digit)) {
+            if (is_negative)
+                selector.event.kind = AST::HistorySelector::EventKind::IndexFromEnd;
+            else
+                selector.event.kind = AST::HistorySelector::EventKind::IndexFromStart;
+            selector.event.index = abs(selector.event.text.to_int().value());
+        }
+        break;
+    }
+    }
+
+    if (peek() != ':')
+        return create<AST::HistoryEvent>(move(selector));
+
+    consume();
+
+    // Word selectors
+    auto parse_word_selector = [&]() -> Optional<AST::HistorySelector::WordSelector> {
+        auto rule_start = push_start();
+        auto c = peek();
+        if (isdigit(c)) {
+            auto num = consume_while(is_digit);
+            auto value = num.to_uint();
+            return AST::HistorySelector::WordSelector {
+                AST::HistorySelector::WordSelectorKind::Index,
+                value.value(),
+                { m_rule_start_offsets.last(), m_offset, m_rule_start_lines.last(), line() }
+            };
+        }
+        if (c == '^') {
+            consume();
+            return AST::HistorySelector::WordSelector {
+                AST::HistorySelector::WordSelectorKind::Index,
+                0,
+                { m_rule_start_offsets.last(), m_offset, m_rule_start_lines.last(), line() }
+            };
+        }
+        if (c == '$') {
+            consume();
+            return AST::HistorySelector::WordSelector {
+                AST::HistorySelector::WordSelectorKind::Last,
+                0,
+                { m_rule_start_offsets.last(), m_offset, m_rule_start_lines.last(), line() }
+            };
+        }
+        return {};
+    };
+
+    auto start = parse_word_selector();
+    if (!start.has_value()) {
+        syntax_error = create<AST::SyntaxError>("Expected a word selector after ':' in a history event designator", true);
+        auto node = create<AST::HistoryEvent>(move(selector));
+        node->set_is_syntax_error(syntax_error->syntax_error_node());
+        return node;
+    }
+    selector.word_selector_range.start = start.release_value();
+
+    if (peek() == '-') {
+        consume();
+        auto end = parse_word_selector();
+        if (!end.has_value()) {
+            syntax_error = create<AST::SyntaxError>("Expected a word selector after '-' in a history event designator word selector", true);
+            auto node = create<AST::HistoryEvent>(move(selector));
+            node->set_is_syntax_error(syntax_error->syntax_error_node());
+            return node;
+        }
+        selector.word_selector_range.end = move(end);
+    } else {
+        selector.word_selector_range.end.clear();
+    }
+
+    return create<AST::HistoryEvent>(move(selector));
+}
+
 RefPtr<AST::Node> Parser::parse_comment()
 {
     if (at_end())
@@ -1348,7 +1492,7 @@ RefPtr<AST::Node> Parser::parse_bareword()
     StringBuilder builder;
     auto is_acceptable_bareword_character = [&](char c) {
         return strchr("\\\"'*$&#|(){} ?;<>\n", c) == nullptr
-            && ((m_is_in_brace_expansion_spec && c != ',') || !m_is_in_brace_expansion_spec);
+            && !m_extra_chars_not_allowed_in_barewords.contains_slow(c);
     };
     while (!at_end()) {
         char ch = peek();
@@ -1497,6 +1641,8 @@ RefPtr<AST::Node> Parser::parse_brace_expansion()
 RefPtr<AST::Node> Parser::parse_brace_expansion_spec()
 {
     TemporaryChange is_in_brace_expansion { m_is_in_brace_expansion_spec, true };
+    TemporaryChange chars_change { m_extra_chars_not_allowed_in_barewords, { ',' } };
+
     auto rule_start = push_start();
     auto start_expr = parse_expression();
     if (start_expr) {

Userland/Shell/Parser.h

@@ -43,6 +43,10 @@ public:
     }
 
     RefPtr<AST::Node> parse();
+    /// Parse the given string *as* an expression
+    /// that is to forefully enclose it in double-quotes.
+    RefPtr<AST::Node> parse_as_single_expression();
+    NonnullRefPtrVector<AST::Node> parse_as_multiple_expressions();
 
     struct SavedOffset {
         size_t offset;
@@ -77,6 +81,7 @@ private:
     RefPtr<AST::Node> parse_doublequoted_string_inner();
     RefPtr<AST::Node> parse_variable();
     RefPtr<AST::Node> parse_evaluate();
+    RefPtr<AST::Node> parse_history_designator();
     RefPtr<AST::Node> parse_comment();
     RefPtr<AST::Node> parse_bareword();
     RefPtr<AST::Node> parse_glob();
@@ -140,6 +145,7 @@ private:
     Vector<size_t> m_rule_start_offsets;
     Vector<AST::Position::Line> m_rule_start_lines;
 
+    Vector<char> m_extra_chars_not_allowed_in_barewords;
     bool m_is_in_brace_expansion_spec { false };
     bool m_continuation_controls_allowed { false };
 };
@@ -215,6 +221,7 @@ list_expression :: ' '* expression (' '+ list_expression)?
 expression :: evaluate expression?
             | string_composite expression?
             | comment expression?
+            | history_designator expression?
             | '(' list_expression ')' expression?
 
 evaluate :: '$' '(' pipe_sequence ')'
@@ -244,6 +251,18 @@ variable :: '$' identifier
 
 comment :: '#' [^\n]*
 
+history_designator :: '!' event_selector (':' word_selector_composite)?
+
+event_selector :: '!'                  {== '-0'}
+                | '?' bareword '?'
+                | bareword             {number: index, otherwise: lookup}
+
+word_selector_composite :: word_selector ('-' word_selector)?
+
+word_selector :: number
+               | '^'                   {== 0}
+               | '$'                   {== end}
+
 bareword :: [^"'*$&#|()[\]{} ?;<>] bareword?
           | '\' [^"'*$&#|()[\]{} ?;<>] bareword?
 

Userland/Shell/Shell.cpp

@@ -1099,19 +1099,57 @@ String Shell::get_history_path()
 
 String Shell::escape_token_for_single_quotes(const String& token)
 {
+    // `foo bar \n '` -> `'foo bar \n '"'"`
+
     StringBuilder builder;
+    builder.append("'");
+    auto started_single_quote = true;
 
     for (auto c : token) {
         switch (c) {
         case '\'':
-            builder.append("'\\'");
-            break;
+            builder.append("\"'\"");
+            started_single_quote = false;
+            continue;
         default:
+            builder.append(c);
+            if (!started_single_quote) {
+                started_single_quote = true;
+                builder.append("'");
+            }
             break;
         }
-        builder.append(c);
     }
 
+    if (started_single_quote)
+        builder.append("'");
+
+    return builder.build();
+}
+
+String Shell::escape_token_for_double_quotes(const String& token)
+{
+    // `foo bar \n $x 'blah "hello` -> `"foo bar \\n $x 'blah \"hello"`
+
+    StringBuilder builder;
+    builder.append('"');
+
+    for (auto c : token) {
+        switch (c) {
+        case '\"':
+            builder.append("\\\"");
+            continue;
+        case '\\':
+            builder.append("\\\\");
+            continue;
+        default:
+            builder.append(c);
+            break;
+        }
+    }
+
+    builder.append('"');
+
     return builder.build();
 }
 
@@ -1499,6 +1537,22 @@ void Shell::bring_cursor_to_beginning_of_a_line() const
     putc('\r', stderr);
 }
 
+bool Shell::has_history_event(StringView source)
+{
+    struct : public AST::NodeVisitor {
+        virtual void visit(const AST::HistoryEvent* node)
+        {
+            has_history_event = true;
+            AST::NodeVisitor::visit(node);
+        }
+
+        bool has_history_event { false };
+    } visitor;
+
+    Parser { source }.parse()->visit(visitor);
+    return visitor.has_history_event;
+}
+
 bool Shell::read_single_line()
 {
     restore_ios();
@@ -1523,7 +1577,9 @@ bool Shell::read_single_line()
 
     run_command(line);
 
-    m_editor->add_to_history(line);
+    if (!has_history_event(line))
+        m_editor->add_to_history(line);
+
     return true;
 }
 

Userland/Shell/Shell.h

@@ -109,6 +109,8 @@ public:
     String resolve_path(String) const;
     String resolve_alias(const String&) const;
 
+    static bool has_history_event(StringView);
+
     RefPtr<AST::Value> get_argument(size_t);
     RefPtr<AST::Value> lookup_local_variable(const String&);
     String local_variable_or(const String&, const String&);
@@ -153,6 +155,7 @@ public:
     [[nodiscard]] Frame push_frame(String name);
     void pop_frame();
 
+    static String escape_token_for_double_quotes(const String& token);
     static String escape_token_for_single_quotes(const String& token);
     static String escape_token(const String& token);
     static String unescape_token(const String& token);