LibJS: Implement basic for..in and for..of loops

Author: https://github.com/linusg
Commit: https://github.com/SerenityOS/serenity/commit/07af2e6b2cf
Pull-request: https://github.com/SerenityOS/serenity/pull/1909
Reviewed-by: https://github.com/alimpfard
Reviewed-by: https://github.com/sunverwerth

Libraries/LibJS/AST.cpp

@@ -322,6 +322,128 @@ Value ForStatement::execute(Interpreter& interpreter) const
     return last_value;
 }
 
+static FlyString variable_from_for_declaration(Interpreter& interpreter, NonnullRefPtr<ASTNode> node, RefPtr<BlockStatement> wrapper)
+{
+    FlyString variable_name;
+    if (node->is_variable_declaration()) {
+        auto* variable_declaration = static_cast<const VariableDeclaration*>(node.ptr());
+        ASSERT(!variable_declaration->declarations().is_empty());
+        if (variable_declaration->declaration_kind() != DeclarationKind::Var) {
+            wrapper = create_ast_node<BlockStatement>();
+            interpreter.enter_scope(*wrapper, {}, ScopeType::Block);
+        }
+        variable_declaration->execute(interpreter);
+        variable_name = variable_declaration->declarations().first().id().string();
+    } else if (node->is_identifier()) {
+        variable_name = static_cast<const Identifier&>(*node).string();
+    } else {
+        ASSERT_NOT_REACHED();
+    }
+    return variable_name;
+}
+
+Value ForInStatement::execute(Interpreter& interpreter) const
+{
+    if (!m_lhs->is_variable_declaration() && !m_lhs->is_identifier()) {
+        // FIXME: Implement "for (foo.bar in baz)", "for (foo[0] in bar)"
+        ASSERT_NOT_REACHED();
+    }
+    RefPtr<BlockStatement> wrapper;
+    auto variable_name = variable_from_for_declaration(interpreter, m_lhs, wrapper);
+    auto wrapper_cleanup = ScopeGuard([&] {
+        if (wrapper)
+            interpreter.exit_scope(*wrapper);
+    });
+    auto last_value = js_undefined();
+    auto rhs_result = m_rhs->execute(interpreter);
+    if (interpreter.exception())
+        return {};
+    auto* object = rhs_result.to_object(interpreter);
+    while (object) {
+        auto property_names = object->get_own_properties(*object, Object::GetOwnPropertyMode::Key, Attribute::Enumerable);
+        for (auto& property_name : static_cast<Array&>(property_names.as_object()).elements()) {
+            interpreter.set_variable(variable_name, property_name);
+            last_value = interpreter.run(*m_body);
+            if (interpreter.exception())
+                return {};
+            if (interpreter.should_unwind()) {
+                if (interpreter.should_unwind_until(ScopeType::Continuable)) {
+                    interpreter.stop_unwind();
+                } else if (interpreter.should_unwind_until(ScopeType::Breakable)) {
+                    interpreter.stop_unwind();
+                    break;
+                } else {
+                    return js_undefined();
+                }
+            }
+        }
+        object = object->prototype();
+    }
+    return last_value;
+}
+
+Value ForOfStatement::execute(Interpreter& interpreter) const
+{
+    if (!m_lhs->is_variable_declaration() && !m_lhs->is_identifier()) {
+        // FIXME: Implement "for (foo.bar of baz)", "for (foo[0] of bar)"
+        ASSERT_NOT_REACHED();
+    }
+    RefPtr<BlockStatement> wrapper;
+    auto variable_name = variable_from_for_declaration(interpreter, m_lhs, wrapper);
+    auto wrapper_cleanup = ScopeGuard([&] {
+        if (wrapper)
+            interpreter.exit_scope(*wrapper);
+    });
+    auto last_value = js_undefined();
+    auto rhs_result = m_rhs->execute(interpreter);
+    if (interpreter.exception())
+        return {};
+    // FIXME: We need to properly implement the iterator protocol
+    auto is_iterable = rhs_result.is_array() || rhs_result.is_string() || (rhs_result.is_object() && rhs_result.as_object().is_string_object());
+    if (!is_iterable)
+        return interpreter.throw_exception<TypeError>("for..of right-hand side must be iterable");
+
+    size_t index = 0;
+    auto next = [&]() -> Optional<Value> {
+        if (rhs_result.is_array()) {
+            auto array_elements = static_cast<Array*>(&rhs_result.as_object())->elements();
+            if (index < array_elements.size())
+                return Value(array_elements.at(index));
+        } else if (rhs_result.is_string()) {
+            auto string = rhs_result.as_string().string();
+            if (index < string.length())
+                return js_string(interpreter, string.substring(index, 1));
+        } else if (rhs_result.is_object() && rhs_result.as_object().is_string_object()) {
+            auto string = static_cast<StringObject*>(&rhs_result.as_object())->primitive_string().string();
+            if (index < string.length())
+                return js_string(interpreter, string.substring(index, 1));
+        }
+        return {};
+    };
+
+    for (;;) {
+        auto next_item = next();
+        if (!next_item.has_value())
+            break;
+        interpreter.set_variable(variable_name, next_item.value());
+        last_value = interpreter.run(*m_body);
+        if (interpreter.exception())
+            return {};
+        if (interpreter.should_unwind()) {
+            if (interpreter.should_unwind_until(ScopeType::Continuable)) {
+                interpreter.stop_unwind();
+            } else if (interpreter.should_unwind_until(ScopeType::Breakable)) {
+                interpreter.stop_unwind();
+                break;
+            } else {
+                return js_undefined();
+            }
+        }
+        ++index;
+    }
+    return last_value;
+}
+
 Value BinaryExpression::execute(Interpreter& interpreter) const
 {
     auto lhs_result = m_lhs->execute(interpreter);
@@ -801,6 +923,28 @@ void ForStatement::dump(int indent) const
     body().dump(indent + 1);
 }
 
+void ForInStatement::dump(int indent) const
+{
+    ASTNode::dump(indent);
+
+    print_indent(indent);
+    printf("ForIn\n");
+    lhs().dump(indent + 1);
+    rhs().dump(indent + 1);
+    body().dump(indent + 1);
+}
+
+void ForOfStatement::dump(int indent) const
+{
+    ASTNode::dump(indent);
+
+    print_indent(indent);
+    printf("ForOf\n");
+    lhs().dump(indent + 1);
+    rhs().dump(indent + 1);
+    body().dump(indent + 1);
+}
+
 Value Identifier::execute(Interpreter& interpreter) const
 {
     auto value = interpreter.get_variable(string());

Libraries/LibJS/AST.h

@@ -341,6 +341,54 @@ private:
     NonnullRefPtr<Statement> m_body;
 };
 
+class ForInStatement : public Statement {
+public:
+    ForInStatement(NonnullRefPtr<ASTNode> lhs, NonnullRefPtr<Expression> rhs, NonnullRefPtr<Statement> body)
+        : m_lhs(move(lhs))
+        , m_rhs(move(rhs))
+        , m_body(move(body))
+    {
+    }
+
+    const ASTNode& lhs() const { return *m_lhs; }
+    const Expression& rhs() const { return *m_rhs; }
+    const Statement& body() const { return *m_body; }
+
+    virtual Value execute(Interpreter&) const override;
+    virtual void dump(int indent) const override;
+
+private:
+    virtual const char* class_name() const override { return "ForInStatement"; }
+
+    NonnullRefPtr<ASTNode> m_lhs;
+    NonnullRefPtr<Expression> m_rhs;
+    NonnullRefPtr<Statement> m_body;
+};
+
+class ForOfStatement : public Statement {
+public:
+    ForOfStatement(NonnullRefPtr<ASTNode> lhs, NonnullRefPtr<Expression> rhs, NonnullRefPtr<Statement> body)
+        : m_lhs(move(lhs))
+        , m_rhs(move(rhs))
+        , m_body(move(body))
+    {
+    }
+
+    const ASTNode& lhs() const { return *m_lhs; }
+    const Expression& rhs() const { return *m_rhs; }
+    const Statement& body() const { return *m_body; }
+
+    virtual Value execute(Interpreter&) const override;
+    virtual void dump(int indent) const override;
+
+private:
+    virtual const char* class_name() const override { return "ForOfStatement"; }
+
+    NonnullRefPtr<ASTNode> m_lhs;
+    NonnullRefPtr<Expression> m_rhs;
+    NonnullRefPtr<Statement> m_body;
+};
+
 enum class BinaryOp {
     Addition,
     Subtraction,
@@ -678,6 +726,11 @@ enum class DeclarationKind {
 
 class VariableDeclarator final : public ASTNode {
 public:
+    VariableDeclarator(NonnullRefPtr<Identifier> id)
+        : m_id(move(id))
+    {
+    }
+
     VariableDeclarator(NonnullRefPtr<Identifier> id, RefPtr<Expression> init)
         : m_id(move(id))
         , m_init(move(init))

Libraries/LibJS/Parser.cpp

@@ -592,8 +592,7 @@ NonnullRefPtr<StringLiteral> Parser::parse_string_literal(Token token)
         syntax_error(
             message,
             m_parser_state.m_current_token.line_number(),
-            m_parser_state.m_current_token.line_column()
-        );
+            m_parser_state.m_current_token.line_column());
     }
     return create_ast_node<StringLiteral>(string);
 }
@@ -651,14 +650,14 @@ NonnullRefPtr<TemplateLiteral> Parser::parse_template_literal(bool is_tagged)
     return create_ast_node<TemplateLiteral>(expressions);
 }
 
-NonnullRefPtr<Expression> Parser::parse_expression(int min_precedence, Associativity associativity)
+NonnullRefPtr<Expression> Parser::parse_expression(int min_precedence, Associativity associativity, Vector<TokenType> forbidden)
 {
     auto expression = parse_primary_expression();
     while (match(TokenType::TemplateLiteralStart)) {
         auto template_literal = parse_template_literal(true);
         expression = create_ast_node<TaggedTemplateLiteral>(move(expression), move(template_literal));
     }
-    while (match_secondary_expression()) {
+    while (match_secondary_expression(forbidden)) {
         int new_precedence = operator_precedence(m_parser_state.m_current_token.type());
         if (new_precedence < min_precedence)
             break;
@@ -974,7 +973,7 @@ NonnullRefPtr<FunctionNodeType> Parser::parse_function_node(bool needs_function_
     return create_ast_node<FunctionNodeType>(name, move(body), move(parameters), function_length, NonnullRefPtrVector<VariableDeclaration>());
 }
 
-NonnullRefPtr<VariableDeclaration> Parser::parse_variable_declaration()
+NonnullRefPtr<VariableDeclaration> Parser::parse_variable_declaration(bool with_semicolon)
 {
     DeclarationKind declaration_kind;
 
@@ -1010,10 +1009,11 @@ NonnullRefPtr<VariableDeclaration> Parser::parse_variable_declaration()
         }
         break;
     }
-    consume_or_insert_semicolon();
+    if (with_semicolon)
+        consume_or_insert_semicolon();
 
     auto declaration = create_ast_node<VariableDeclaration>(declaration_kind, move(declarations));
-    if (declaration->declaration_kind() == DeclarationKind::Var)
+    if (declaration_kind == DeclarationKind::Var)
         m_parser_state.m_var_scopes.last().append(declaration);
     else
         m_parser_state.m_let_scopes.last().append(declaration);
@@ -1177,57 +1177,46 @@ NonnullRefPtr<IfStatement> Parser::parse_if_statement()
     return create_ast_node<IfStatement>(move(predicate), move(consequent), move(alternate));
 }
 
-NonnullRefPtr<ForStatement> Parser::parse_for_statement()
+NonnullRefPtr<Statement> Parser::parse_for_statement()
 {
+    auto match_for_in_of = [&]() {
+        return match(TokenType::In) || (match(TokenType::Identifier) && m_parser_state.m_current_token.value() == "of");
+    };
+
     consume(TokenType::For);
 
     consume(TokenType::ParenOpen);
 
-    bool first_semicolon_consumed = false;
     bool in_scope = false;
     RefPtr<ASTNode> init;
-    switch (m_parser_state.m_current_token.type()) {
-    case TokenType::Semicolon:
-        break;
-    default:
+    if (!match(TokenType::Semicolon)) {
         if (match_expression()) {
-            init = parse_expression(0);
+            init = parse_expression(0, Associativity::Right, { TokenType::In });
+            if (match_for_in_of())
+                return parse_for_in_of_statement(*init);
         } else if (match_variable_declaration()) {
-            if (m_parser_state.m_current_token.type() != TokenType::Var) {
+            if (!match(TokenType::Var)) {
                 m_parser_state.m_let_scopes.append(NonnullRefPtrVector<VariableDeclaration>());
                 in_scope = true;
             }
-
-            init = parse_variable_declaration();
-            first_semicolon_consumed = true;
+            init = parse_variable_declaration(false);
+            if (match_for_in_of())
+                return parse_for_in_of_statement(*init);
         } else {
             ASSERT_NOT_REACHED();
         }
-        break;
     }
-
-    if (!first_semicolon_consumed)
-        consume(TokenType::Semicolon);
+    consume(TokenType::Semicolon);
 
     RefPtr<Expression> test;
-    switch (m_parser_state.m_current_token.type()) {
-    case TokenType::Semicolon:
-        break;
-    default:
+    if (!match(TokenType::Semicolon))
         test = parse_expression(0);
-        break;
-    }
 
     consume(TokenType::Semicolon);
 
     RefPtr<Expression> update;
-    switch (m_parser_state.m_current_token.type()) {
-    case TokenType::ParenClose:
-        break;
-    default:
+    if (!match(TokenType::ParenClose))
         update = parse_expression(0);
-        break;
-    }
 
     consume(TokenType::ParenClose);
 
@@ -1240,6 +1229,28 @@ NonnullRefPtr<ForStatement> Parser::parse_for_statement()
     return create_ast_node<ForStatement>(move(init), move(test), move(update), move(body));
 }
 
+NonnullRefPtr<Statement> Parser::parse_for_in_of_statement(NonnullRefPtr<ASTNode> lhs)
+{
+    if (lhs->is_variable_declaration()) {
+        auto declarations = static_cast<VariableDeclaration*>(lhs.ptr())->declarations();
+        if (declarations.size() > 1) {
+            syntax_error("multiple declarations not allowed in for..in/of");
+            lhs = create_ast_node<ErrorExpression>();
+        }
+        if (declarations.first().init() != nullptr) {
+            syntax_error("variable initializer not allowed in for..in/of");
+            lhs = create_ast_node<ErrorExpression>();
+        }
+    }
+    auto in_or_of = consume();
+    auto rhs = parse_expression(0);
+    consume(TokenType::ParenClose);
+    auto body = parse_statement();
+    if (in_or_of.type() == TokenType::In)
+        return create_ast_node<ForInStatement>(move(lhs), move(rhs), move(body));
+    return create_ast_node<ForOfStatement>(move(lhs), move(rhs), move(body));
+}
+
 NonnullRefPtr<DebuggerStatement> Parser::parse_debugger_statement()
 {
     consume(TokenType::Debugger);
@@ -1296,9 +1307,11 @@ bool Parser::match_unary_prefixed_expression() const
         || type == TokenType::Delete;
 }
 
-bool Parser::match_secondary_expression() const
+bool Parser::match_secondary_expression(Vector<TokenType> forbidden) const
 {
     auto type = m_parser_state.m_current_token.type();
+    if (forbidden.contains_slow(type))
+        return false;
     return type == TokenType::Plus
         || type == TokenType::PlusEquals
         || type == TokenType::Minus
@@ -1410,7 +1423,7 @@ void Parser::consume_or_insert_semicolon()
 
 Token Parser::consume(TokenType expected_type)
 {
-    if (m_parser_state.m_current_token.type() != expected_type) {
+    if (!match(expected_type)) {
         expected(Token::name(expected_type));
     }
     return consume();

Libraries/LibJS/Parser.h

@@ -50,8 +50,9 @@ public:
     NonnullRefPtr<Statement> parse_statement();
     NonnullRefPtr<BlockStatement> parse_block_statement();
     NonnullRefPtr<ReturnStatement> parse_return_statement();
-    NonnullRefPtr<VariableDeclaration> parse_variable_declaration();
-    NonnullRefPtr<ForStatement> parse_for_statement();
+    NonnullRefPtr<VariableDeclaration> parse_variable_declaration(bool with_semicolon = true);
+    NonnullRefPtr<Statement> parse_for_statement();
+    NonnullRefPtr<Statement> parse_for_in_of_statement(NonnullRefPtr<ASTNode> lhs);
     NonnullRefPtr<IfStatement> parse_if_statement();
     NonnullRefPtr<ThrowStatement> parse_throw_statement();
     NonnullRefPtr<TryStatement> parse_try_statement();
@@ -65,7 +66,7 @@ public:
     NonnullRefPtr<DebuggerStatement> parse_debugger_statement();
     NonnullRefPtr<ConditionalExpression> parse_conditional_expression(NonnullRefPtr<Expression> test);
 
-    NonnullRefPtr<Expression> parse_expression(int min_precedence, Associativity associate = Associativity::Right);
+    NonnullRefPtr<Expression> parse_expression(int min_precedence, Associativity associate = Associativity::Right, Vector<TokenType> forbidden = {});
     NonnullRefPtr<Expression> parse_primary_expression();
     NonnullRefPtr<Expression> parse_unary_prefixed_expression();
     NonnullRefPtr<ObjectExpression> parse_object_expression();
@@ -105,7 +106,7 @@ private:
     Associativity operator_associativity(TokenType) const;
     bool match_expression() const;
     bool match_unary_prefixed_expression() const;
-    bool match_secondary_expression() const;
+    bool match_secondary_expression(Vector<TokenType> forbidden = {}) const;
     bool match_statement() const;
     bool match_variable_declaration() const;
     bool match_identifier_name() const;

Libraries/LibJS/Tests/Array.prototype-generic-functions.js

@@ -73,67 +73,47 @@ try {
     const o = { length: 5, 0: "foo", 1: "bar", 3: "baz" };
 
     {
-        const visited = [];
-        Array.prototype.every.call(o, function (value) {
-            visited.push(value);
-            return true;
-        });
-        assert(visited.length === 3);
-        assert(visited[0] === "foo");
-        assert(visited[1] === "bar");
-        assert(visited[2] === "baz");
+        assertVisitsAll(visit => {
+            Array.prototype.every.call(o, function (value) {
+                visit(value);
+                return true;
+            });
+        }, ["foo", "bar", "baz"]);
     }
 
     ["find", "findIndex"].forEach(name => {
-        const visited = [];
-        Array.prototype[name].call(o, function (value) {
-            visited.push(value);
-            return false;
-        });
-        assert(visited.length === 5);
-        assert(visited[0] === "foo");
-        assert(visited[1] === "bar");
-        assert(visited[2] === undefined);
-        assert(visited[3] === "baz");
-        assert(visited[4] === undefined);
+        assertVisitsAll(visit => {
+            Array.prototype[name].call(o, function (value) {
+                visit(value);
+                return false;
+            });
+        }, ["foo", "bar", undefined, "baz", undefined]);
     });
 
     ["filter", "forEach", "map", "some"].forEach(name => {
-        const visited = [];
-        Array.prototype[name].call(o, function (value) {
-            visited.push(value);
-            return false;
-        });
-        assert(visited.length === 3);
-        assert(visited[0] === "foo");
-        assert(visited[1] === "bar");
-        assert(visited[2] === "baz");
+        assertVisitsAll(visit => {
+            Array.prototype[name].call(o, function (value) {
+                visit(value);
+                return false;
+            });
+        }, ["foo", "bar", "baz"]);
     });
-
     {
-        const visited = [];
-        Array.prototype.reduce.call(o, function (_, value) {
-            visited.push(value);
-            return false;
-        }, "initial");
-
-        assert(visited.length === 3);
-        assert(visited[0] === "foo");
-        assert(visited[1] === "bar");
-        assert(visited[2] === "baz");
+        assertVisitsAll(visit => {
+            Array.prototype.reduce.call(o, function (_, value) {
+                visit(value);
+                return false;
+            }, "initial");
+        }, ["foo", "bar", "baz"]);
     }
 
     {
-        const visited = [];
-        Array.prototype.reduceRight.call(o, function (_, value) {
-            visited.push(value);
-            return false;
-        }, "initial");
-
-        assert(visited.length === 3);
-        assert(visited[2] === "foo");
-        assert(visited[1] === "bar");
-        assert(visited[0] === "baz");
+        assertVisitsAll(visit => {
+            Array.prototype.reduceRight.call(o, function (_, value) {
+                visit(value);
+                return false;
+            }, "initial");
+        }, ["baz", "bar", "foo"]);
     }
 
     console.log("PASS");

Libraries/LibJS/Tests/for-in-basic.js

@@ -0,0 +1,41 @@
+load("test-common.js");
+
+try {
+    assertVisitsAll(visit => {
+        for (const property in "") {
+            visit(property);
+        }
+    }, []);
+
+    assertVisitsAll(visit => {
+        for (const property in 123) {
+            visit(property);
+        }
+    }, []);
+
+    assertVisitsAll(visit => {
+        for (const property in {}) {
+            visit(property);
+        }
+    }, []);
+
+    assertVisitsAll(visit => {
+        for (const property in "hello") {
+            visit(property);
+        }
+    }, ["0", "1", "2", "3", "4"]);
+
+    assertVisitsAll(visit => {
+        for (const property in {a: 1, b: 2, c: 2}) {
+            visit(property);
+        }
+    }, ["a", "b", "c"]);
+
+    var property;
+    for (property in "abc");
+    assert(property === "2");
+
+    console.log("PASS");
+} catch (e) {
+    console.log("FAIL: " + e);
+}

Libraries/LibJS/Tests/for-of-basic.js

@@ -0,0 +1,43 @@
+load("test-common.js");
+
+try {
+    assertThrowsError(() => {
+        for (const _ of 123) {}
+    }, {
+        error: TypeError,
+        message: "for..of right-hand side must be iterable"
+    });
+
+    assertThrowsError(() => {
+        for (const _ of {foo: 1, bar: 2}) {}
+    }, {
+        error: TypeError,
+        message: "for..of right-hand side must be iterable"
+    });
+
+    assertVisitsAll(visit => {
+        for (const num of [1, 2, 3]) {
+            visit(num);
+        }
+    }, [1, 2, 3]);
+
+    assertVisitsAll(visit => {
+        for (const char of "hello") {
+            visit(char);
+        }
+    }, ["h", "e", "l", "l", "o"]);
+
+    assertVisitsAll(visit => {
+        for (const char of new String("hello")) {
+            visit(char);
+        }
+    }, ["h", "e", "l", "l", "o"]);
+
+    var char;
+    for (char of "abc");
+    assert(char === "c");
+
+    console.log("PASS");
+} catch (e) {
+    console.log("FAIL: " + e);
+}

Libraries/LibJS/Tests/test-common.js

@@ -50,6 +50,13 @@ function assertThrowsError(testFunction, options) {
     }
 }
 
+const assertVisitsAll = (testFunction, expectedOutput) => {
+    const visited = [];
+    testFunction(value => visited.push(value));
+    assert(visited.length === expectedOutput.length);
+    expectedOutput.forEach((value, i) => assert(visited[i] === value));
+};
+
 /**
  * Check whether the difference between two numbers is less than 0.000001.
  * @param {Number} a First number