From a2161d8eb718d8b2dde5aaa6120ad60524028875 Mon Sep 17 00:00:00 2001 From: DexesTTP Date: Sun, 3 Apr 2022 18:20:05 +0200 Subject: [PATCH] Utilities: Add networking to headless-browser With this, the headless browser can now connect to the web. Thanks a lot to Ali and Sin-ack for their help with this! Co-authored-by: Ali Mohammad Pur Co-authored-by: sin-ack --- Userland/Utilities/CMakeLists.txt | 2 +- Userland/Utilities/headless-browser.cpp | 365 +++++++++++++++++++++++- 2 files changed, 362 insertions(+), 5 deletions(-) diff --git a/Userland/Utilities/CMakeLists.txt b/Userland/Utilities/CMakeLists.txt index 24488b99b47..bb40193d38a 100644 --- a/Userland/Utilities/CMakeLists.txt +++ b/Userland/Utilities/CMakeLists.txt @@ -128,7 +128,7 @@ target_link_libraries(groups LibMain) target_link_libraries(gunzip LibCompress LibMain) target_link_libraries(gzip LibCompress LibMain) target_link_libraries(head LibMain) -target_link_libraries(headless-browser LibCore LibGfx LibWeb LibMain) +target_link_libraries(headless-browser LibCore LibGemini LibGfx LibHTTP LibWeb LibWebSocket LibMain) target_link_libraries(hexdump LibMain) target_link_libraries(host LibMain) target_link_libraries(hostname LibMain) diff --git a/Userland/Utilities/headless-browser.cpp b/Userland/Utilities/headless-browser.cpp index 94fb5d76e08..924e6192221 100644 --- a/Userland/Utilities/headless-browser.cpp +++ b/Userland/Utilities/headless-browser.cpp @@ -14,12 +14,22 @@ #include #include #include +#include +#include +#include #include +#include +#include +#include #include #include #include #include #include +#include +#include +#include +#include #include #include #include @@ -30,6 +40,9 @@ #include #include #include +#include +#include +#include class HeadlessBrowserPageClient final : public Web::PageClient { public: @@ -255,8 +268,237 @@ private: explicit HeadlessImageDecoderClient() = default; }; +static HashTable> s_all_requests; + class HeadlessRequestServer : public Web::ResourceLoaderConnector { public: + class HTTPHeadlessRequest + : public Web::ResourceLoaderConnectorRequest + , public Weakable { + public: + static ErrorOr> create(String const& method, AK::URL const& url, HashMap const& request_headers, ReadonlyBytes request_body, Core::ProxyData const&) + { + auto stream_backing_buffer = TRY(ByteBuffer::create_uninitialized(1 * MiB)); + auto underlying_socket = TRY(Core::Stream::TCPSocket::connect(url.host(), url.port().value_or(80))); + TRY(underlying_socket->set_blocking(false)); + auto socket = TRY(Core::Stream::BufferedSocket::create(move(underlying_socket))); + + HTTP::HttpRequest request; + if (method.equals_ignoring_case("head"sv)) + request.set_method(HTTP::HttpRequest::HEAD); + else if (method.equals_ignoring_case("get"sv)) + request.set_method(HTTP::HttpRequest::GET); + else if (method.equals_ignoring_case("post"sv)) + request.set_method(HTTP::HttpRequest::POST); + else + request.set_method(HTTP::HttpRequest::Invalid); + request.set_url(move(url)); + request.set_headers(request_headers); + request.set_body(TRY(ByteBuffer::copy(request_body))); + + return adopt_ref(*new HTTPHeadlessRequest(move(request), move(socket), move(stream_backing_buffer))); + } + + virtual ~HTTPHeadlessRequest() override + { + } + + virtual void set_should_buffer_all_input(bool) override + { + } + + virtual bool stop() override + { + return false; + } + + virtual void stream_into(Core::Stream::Stream&) override + { + } + + private: + HTTPHeadlessRequest(HTTP::HttpRequest&& request, NonnullOwnPtr socket, ByteBuffer&& stream_backing_buffer) + : m_stream_backing_buffer(move(stream_backing_buffer)) + , m_output_stream(Core::Stream::MemoryStream::construct(m_stream_backing_buffer.bytes()).release_value_but_fixme_should_propagate_errors()) + , m_socket(move(socket)) + , m_job(HTTP::Job::construct(move(request), *m_output_stream)) + { + m_job->on_headers_received = [weak_this = make_weak_ptr()](auto& response_headers, auto response_code) mutable { + if (auto strong_this = weak_this.strong_ref()) { + strong_this->m_response_code = response_code; + for (auto& header : response_headers) { + strong_this->m_response_headers.set(header.key, header.value); + } + } + }; + m_job->on_finish = [weak_this = make_weak_ptr()](bool success) mutable { + Core::deferred_invoke([weak_this, success]() mutable { + if (auto strong_this = weak_this.strong_ref()) { + ReadonlyBytes response_bytes { strong_this->m_output_stream->bytes().data(), strong_this->m_output_stream->offset() }; + auto response_buffer = ByteBuffer::copy(response_bytes).release_value_but_fixme_should_propagate_errors(); + strong_this->on_buffered_request_finish(success, strong_this->m_output_stream->offset(), strong_this->m_response_headers, strong_this->m_response_code, response_buffer); + } + }); + }; + m_job->start(*m_socket); + } + + Optional m_response_code; + ByteBuffer m_stream_backing_buffer; + NonnullOwnPtr m_output_stream; + NonnullOwnPtr m_socket; + NonnullRefPtr m_job; + HashMap m_response_headers; + }; + + class HTTPSHeadlessRequest + : public Web::ResourceLoaderConnectorRequest + , public Weakable { + public: + static ErrorOr> create(String const& method, AK::URL const& url, HashMap const& request_headers, ReadonlyBytes request_body, Core::ProxyData const&) + { + auto stream_backing_buffer = TRY(ByteBuffer::create_uninitialized(1 * MiB)); + auto underlying_socket = TRY(TLS::TLSv12::connect(url.host(), url.port().value_or(80))); + TRY(underlying_socket->set_blocking(false)); + auto socket = TRY(Core::Stream::BufferedSocket::create(move(underlying_socket))); + + HTTP::HttpRequest request; + if (method.equals_ignoring_case("head"sv)) + request.set_method(HTTP::HttpRequest::HEAD); + else if (method.equals_ignoring_case("get"sv)) + request.set_method(HTTP::HttpRequest::GET); + else if (method.equals_ignoring_case("post"sv)) + request.set_method(HTTP::HttpRequest::POST); + else + request.set_method(HTTP::HttpRequest::Invalid); + request.set_url(move(url)); + request.set_headers(request_headers); + request.set_body(TRY(ByteBuffer::copy(request_body))); + + return adopt_ref(*new HTTPSHeadlessRequest(move(request), move(socket), move(stream_backing_buffer))); + } + + virtual ~HTTPSHeadlessRequest() override + { + } + + virtual void set_should_buffer_all_input(bool) override + { + } + + virtual bool stop() override + { + return false; + } + + virtual void stream_into(Core::Stream::Stream&) override + { + } + + private: + HTTPSHeadlessRequest(HTTP::HttpRequest&& request, NonnullOwnPtr socket, ByteBuffer&& stream_backing_buffer) + : m_stream_backing_buffer(move(stream_backing_buffer)) + , m_output_stream(Core::Stream::MemoryStream::construct(m_stream_backing_buffer.bytes()).release_value_but_fixme_should_propagate_errors()) + , m_socket(move(socket)) + , m_job(HTTP::HttpsJob::construct(move(request), *m_output_stream)) + { + m_job->on_headers_received = [weak_this = make_weak_ptr()](auto& response_headers, auto response_code) mutable { + if (auto strong_this = weak_this.strong_ref()) { + strong_this->m_response_code = response_code; + for (auto& header : response_headers) { + strong_this->m_response_headers.set(header.key, header.value); + } + } + }; + m_job->on_finish = [weak_this = make_weak_ptr()](bool success) mutable { + Core::deferred_invoke([weak_this, success]() mutable { + if (auto strong_this = weak_this.strong_ref()) { + ReadonlyBytes response_bytes { strong_this->m_output_stream->bytes().data(), strong_this->m_output_stream->offset() }; + auto response_buffer = ByteBuffer::copy(response_bytes).release_value_but_fixme_should_propagate_errors(); + strong_this->on_buffered_request_finish(success, strong_this->m_output_stream->offset(), strong_this->m_response_headers, strong_this->m_response_code, response_buffer); + } + }); + }; + m_job->start(*m_socket); + } + + Optional m_response_code; + ByteBuffer m_stream_backing_buffer; + NonnullOwnPtr m_output_stream; + NonnullOwnPtr m_socket; + NonnullRefPtr m_job; + HashMap m_response_headers; + }; + + class GeminiHeadlessRequest + : public Web::ResourceLoaderConnectorRequest + , public Weakable { + public: + static ErrorOr> create(String const&, AK::URL const& url, HashMap const&, ReadonlyBytes, Core::ProxyData const&) + { + auto stream_backing_buffer = TRY(ByteBuffer::create_uninitialized(1 * MiB)); + auto underlying_socket = TRY(Core::Stream::TCPSocket::connect(url.host(), url.port().value_or(80))); + TRY(underlying_socket->set_blocking(false)); + auto socket = TRY(Core::Stream::BufferedSocket::create(move(underlying_socket))); + + Gemini::GeminiRequest request; + request.set_url(url); + + return adopt_ref(*new GeminiHeadlessRequest(move(request), move(socket), move(stream_backing_buffer))); + } + + virtual ~GeminiHeadlessRequest() override + { + } + + virtual void set_should_buffer_all_input(bool) override + { + } + + virtual bool stop() override + { + return false; + } + + virtual void stream_into(Core::Stream::Stream&) override + { + } + + private: + GeminiHeadlessRequest(Gemini::GeminiRequest&& request, NonnullOwnPtr socket, ByteBuffer&& stream_backing_buffer) + : m_stream_backing_buffer(move(stream_backing_buffer)) + , m_output_stream(Core::Stream::MemoryStream::construct(m_stream_backing_buffer.bytes()).release_value_but_fixme_should_propagate_errors()) + , m_socket(move(socket)) + , m_job(Gemini::Job::construct(move(request), *m_output_stream)) + { + m_job->on_headers_received = [weak_this = make_weak_ptr()](auto& response_headers, auto response_code) mutable { + if (auto strong_this = weak_this.strong_ref()) { + strong_this->m_response_code = response_code; + for (auto& header : response_headers) { + strong_this->m_response_headers.set(header.key, header.value); + } + } + }; + m_job->on_finish = [weak_this = make_weak_ptr()](bool success) mutable { + Core::deferred_invoke([weak_this, success]() mutable { + if (auto strong_this = weak_this.strong_ref()) { + ReadonlyBytes response_bytes { strong_this->m_output_stream->bytes().data(), strong_this->m_output_stream->offset() }; + auto response_buffer = ByteBuffer::copy(response_bytes).release_value_but_fixme_should_propagate_errors(); + strong_this->on_buffered_request_finish(success, strong_this->m_output_stream->offset(), strong_this->m_response_headers, strong_this->m_response_code, response_buffer); + } + }); + }; + m_job->start(*m_socket); + } + + Optional m_response_code; + ByteBuffer m_stream_backing_buffer; + NonnullOwnPtr m_output_stream; + NonnullOwnPtr m_socket; + NonnullRefPtr m_job; + HashMap m_response_headers; + }; + static NonnullRefPtr create() { return adopt_ref(*new HeadlessRequestServer()); @@ -267,9 +509,30 @@ public: virtual void prefetch_dns(AK::URL const&) override { } virtual void preconnect(AK::URL const&) override { } - virtual RefPtr start_request(String const&, AK::URL const&, HashMap const&, ReadonlyBytes, Core::ProxyData const&) override + virtual RefPtr start_request(String const& method, AK::URL const& url, HashMap const& request_headers, ReadonlyBytes request_body, Core::ProxyData const& proxy) override { - return {}; + RefPtr request; + if (url.protocol().equals_ignoring_case("http"sv)) { + auto request_or_error = HTTPHeadlessRequest::create(method, url, request_headers, request_body, proxy); + if (request_or_error.is_error()) + return {}; + request = request_or_error.release_value(); + } + if (url.protocol().equals_ignoring_case("https"sv)) { + auto request_or_error = HTTPSHeadlessRequest::create(method, url, request_headers, request_body, proxy); + if (request_or_error.is_error()) + return {}; + request = request_or_error.release_value(); + } + if (url.protocol().equals_ignoring_case("gemini"sv)) { + auto request_or_error = GeminiHeadlessRequest::create(method, url, request_headers, request_body, proxy); + if (request_or_error.is_error()) + return {}; + request = request_or_error.release_value(); + } + if (request) + s_all_requests.set(request); + return request; } private: @@ -278,6 +541,96 @@ private: class HeadlessWebSocketClientManager : public Web::WebSockets::WebSocketClientManager { public: + class HeadlessWebSocket + : public Web::WebSockets::WebSocketClientSocket + , public Weakable { + public: + static NonnullRefPtr create(NonnullRefPtr underlying_socket) + { + return adopt_ref(*new HeadlessWebSocket(move(underlying_socket))); + } + + virtual ~HeadlessWebSocket() override + { + } + + virtual Web::WebSockets::WebSocket::ReadyState ready_state() override + { + switch (m_websocket->ready_state()) { + case WebSocket::ReadyState::Connecting: + return Web::WebSockets::WebSocket::ReadyState::Connecting; + case WebSocket::ReadyState::Open: + return Web::WebSockets::WebSocket::ReadyState::Open; + case WebSocket::ReadyState::Closing: + return Web::WebSockets::WebSocket::ReadyState::Closing; + case WebSocket::ReadyState::Closed: + return Web::WebSockets::WebSocket::ReadyState::Closed; + } + VERIFY_NOT_REACHED(); + } + + virtual void send(ByteBuffer binary_or_text_message, bool is_text) override + { + m_websocket->send(WebSocket::Message(binary_or_text_message, is_text)); + } + + virtual void send(StringView message) override + { + m_websocket->send(WebSocket::Message(message)); + } + + virtual void close(u16 code, String reason) override + { + m_websocket->close(code, reason); + } + + private: + HeadlessWebSocket(NonnullRefPtr underlying_socket) + : m_websocket(move(underlying_socket)) + { + m_websocket->on_open = [weak_this = make_weak_ptr()] { + if (auto strong_this = weak_this.strong_ref()) + if (strong_this->on_open) + strong_this->on_open(); + }; + m_websocket->on_message = [weak_this = make_weak_ptr()](auto message) { + if (auto strong_this = weak_this.strong_ref()) { + if (strong_this->on_message) { + strong_this->on_message(Web::WebSockets::WebSocketClientSocket::Message { + .data = move(message.data()), + .is_text = message.is_text(), + }); + } + } + }; + m_websocket->on_error = [weak_this = make_weak_ptr()](auto error) { + if (auto strong_this = weak_this.strong_ref()) { + if (strong_this->on_error) { + switch (error) { + case WebSocket::WebSocket::Error::CouldNotEstablishConnection: + strong_this->on_error(Web::WebSockets::WebSocketClientSocket::Error::CouldNotEstablishConnection); + return; + case WebSocket::WebSocket::Error::ConnectionUpgradeFailed: + strong_this->on_error(Web::WebSockets::WebSocketClientSocket::Error::ConnectionUpgradeFailed); + return; + case WebSocket::WebSocket::Error::ServerClosedSocket: + strong_this->on_error(Web::WebSockets::WebSocketClientSocket::Error::ServerClosedSocket); + return; + } + VERIFY_NOT_REACHED(); + } + } + }; + m_websocket->on_close = [weak_this = make_weak_ptr()](u16 code, String reason, bool was_clean) { + if (auto strong_this = weak_this.strong_ref()) + if (strong_this->on_close) + strong_this->on_close(code, move(reason), was_clean); + }; + } + + NonnullRefPtr m_websocket; + }; + static NonnullRefPtr create() { return adopt_ref(*new HeadlessWebSocketClientManager()); @@ -285,9 +638,13 @@ public: virtual ~HeadlessWebSocketClientManager() override { } - virtual RefPtr connect(AK::URL const&, String const&) override + virtual RefPtr connect(AK::URL const& url, String const& origin) override { - return {}; + WebSocket::ConnectionInfo connection_info(url); + connection_info.set_origin(origin); + + auto connection = HeadlessWebSocket::create(WebSocket::WebSocket::create(move(connection_info))); + return connection; } private: