Ladybird: Use QtNetwork for HTTP and HTTPS requests

Until we can get our own RequestServer infrastructure up and running,
running the TLS and HTTP code in-process was causing lots of crashes
due to unexpected reentrancy via nested event loops.

This patch adds a simple backend for HTTP and HTTPS requests that simply
funnels them through QNetworkAccessManager.
This commit is contained in:
Andreas Kling 2022-07-05 21:13:38 +02:00 committed by Andrew Kaster
parent 69d264828f
commit 419d3ec646
Notes: sideshowbarker 2024-07-17 07:06:47 +09:00
4 changed files with 152 additions and 278 deletions

View File

@ -30,19 +30,20 @@ include(${Lagom_SOURCE_DIR}/../CMake/lagom_compile_options.cmake)
add_compile_options(-Wno-expansion-to-defined)
set(CMAKE_AUTOMOC ON)
find_package(Qt6 REQUIRED COMPONENTS Core Widgets)
find_package(Qt6 REQUIRED COMPONENTS Core Widgets Network)
# FIXME: Stop using deprecated declarations from QT :^)
add_compile_options(-Wno-deprecated-declarations)
set(SOURCES
BrowserWindow.cpp
RequestManagerQt.cpp
main.cpp
WebView.cpp
)
add_executable(ladybird ${SOURCES})
target_link_libraries(ladybird PRIVATE Qt6::Widgets Lagom::Web Lagom::HTTP Lagom::WebSocket Lagom::Main)
target_link_libraries(ladybird PRIVATE Qt6::Widgets Qt::Network Lagom::Web Lagom::WebSocket Lagom::Main)
get_filename_component(
SERENITY_SOURCE_DIR "${Lagom_SOURCE_DIR}/../.."

View File

@ -0,0 +1,86 @@
/*
* Copyright (c) 2022, Andreas Kling <kling@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include "RequestManagerQt.h"
#include <AK/JsonArray.h>
RequestManagerQt::RequestManagerQt()
{
m_qnam = new QNetworkAccessManager(this);
QObject::connect(m_qnam, &QNetworkAccessManager::finished, this, &RequestManagerQt::reply_finished);
}
void RequestManagerQt::reply_finished(QNetworkReply* reply)
{
auto request = m_pending.get(reply).value();
m_pending.remove(reply);
request->did_finish();
}
RefPtr<Web::ResourceLoaderConnectorRequest> RequestManagerQt::start_request(String const& method, AK::URL const& url, HashMap<String, String> const& request_headers, ReadonlyBytes request_body, Core::ProxyData const& proxy)
{
if (!url.protocol().is_one_of_ignoring_case("http"sv, "https"sv)) {
return nullptr;
}
auto request_or_error = Request::create(*m_qnam, method, url, request_headers, request_body, proxy);
if (request_or_error.is_error()) {
return nullptr;
}
auto request = request_or_error.release_value();
m_pending.set(&request->reply(), *request);
return request;
}
ErrorOr<NonnullRefPtr<RequestManagerQt::Request>> RequestManagerQt::Request::create(QNetworkAccessManager& qnam, String const& method, AK::URL const& url, HashMap<String, String> const& request_headers, ReadonlyBytes request_body, Core::ProxyData const&)
{
QNetworkRequest request { QString(url.to_string().characters()) };
QNetworkReply* reply = nullptr;
if (method.equals_ignoring_case("head"sv)) {
reply = qnam.head(request);
} else if (method.equals_ignoring_case("get"sv)) {
reply = qnam.get(request);
} else if (method.equals_ignoring_case("post"sv)) {
reply = qnam.post(request, QByteArray((char const*)request_body.data(), request_body.size()));
}
for (auto& it : request_headers) {
request.setRawHeader(it.key.characters(), it.value.characters());
}
return adopt_ref(*new Request(*reply));
}
RequestManagerQt::Request::Request(QNetworkReply& reply)
: m_reply(reply)
{
}
RequestManagerQt::Request::~Request() = default;
void RequestManagerQt::Request::did_finish()
{
bool success = m_reply.error() == QNetworkReply::NetworkError::NoError;
auto buffer = m_reply.readAll();
auto http_status_code = m_reply.attribute(QNetworkRequest::Attribute::HttpStatusCodeAttribute).toInt();
HashMap<String, String, CaseInsensitiveStringTraits> response_headers;
Vector<String> set_cookie_headers;
for (auto& it : m_reply.rawHeaderPairs()) {
auto name = String(it.first.data(), it.first.length());
auto value = String(it.second.data(), it.second.length());
if (name.equals_ignoring_case("set-cookie")) {
set_cookie_headers.append(value);
} else {
response_headers.set(name, value);
}
}
if (!set_cookie_headers.is_empty()) {
response_headers.set("set-cookie", JsonArray { set_cookie_headers }.to_string());
}
on_buffered_request_finish(success, buffer.length(), response_headers, http_status_code, ReadonlyBytes { buffer.data(), (size_t)buffer.size() });
}

View File

@ -0,0 +1,61 @@
/*
* Copyright (c) 2022, Andreas Kling <kling@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#define AK_DONT_REPLACE_STD
#include <LibWeb/Loader/ResourceLoader.h>
#include <QtNetwork/QNetworkAccessManager>
#include <QtNetwork/QNetworkReply>
class RequestManagerQt
: public QObject
, public Web::ResourceLoaderConnector {
Q_OBJECT
public:
static NonnullRefPtr<RequestManagerQt> create()
{
return adopt_ref(*new RequestManagerQt());
}
virtual ~RequestManagerQt() override { }
virtual void prefetch_dns(AK::URL const&) override { }
virtual void preconnect(AK::URL const&) override { }
virtual RefPtr<Web::ResourceLoaderConnectorRequest> start_request(String const& method, AK::URL const&, HashMap<String, String> const& request_headers, ReadonlyBytes request_body, Core::ProxyData const&) override;
private slots:
void reply_finished(QNetworkReply*);
private:
RequestManagerQt();
class Request
: public Web::ResourceLoaderConnectorRequest {
public:
static ErrorOr<NonnullRefPtr<Request>> create(QNetworkAccessManager& qnam, String const& method, AK::URL const& url, HashMap<String, String> const& request_headers, ReadonlyBytes request_body, Core::ProxyData const&);
virtual ~Request() override;
virtual void set_should_buffer_all_input(bool) override { }
virtual bool stop() override { return false; }
virtual void stream_into(Core::Stream::Stream&) override { }
void did_finish();
QNetworkReply& reply() { return m_reply; }
private:
Request(QNetworkReply&);
QNetworkReply& m_reply;
};
HashMap<QNetworkReply*, NonnullRefPtr<Request>> m_pending;
QNetworkAccessManager* m_qnam { nullptr };
};

View File

@ -8,6 +8,7 @@
#define AK_DONT_REPLACE_STD
#include "WebView.h"
#include "RequestManagerQt.h"
#include <AK/Assertions.h>
#include <AK/ByteBuffer.h>
#include <AK/Format.h>
@ -23,18 +24,11 @@
#include <LibCore/Stream.h>
#include <LibCore/System.h>
#include <LibCore/Timer.h>
#include <LibGemini/GeminiRequest.h>
#include <LibGemini/GeminiResponse.h>
#include <LibGemini/Job.h>
#include <LibGfx/Bitmap.h>
#include <LibGfx/Font/FontDatabase.h>
#include <LibGfx/ImageDecoder.h>
#include <LibGfx/PNGWriter.h>
#include <LibGfx/Rect.h>
#include <LibHTTP/HttpRequest.h>
#include <LibHTTP/HttpResponse.h>
#include <LibHTTP/HttpsJob.h>
#include <LibHTTP/Job.h>
#include <LibMain/Main.h>
#include <LibWeb/Cookie/ParsedCookie.h>
#include <LibWeb/DOM/Document.h>
@ -440,274 +434,6 @@ private:
explicit HeadlessImageDecoderClient() = default;
};
static HashTable<RefPtr<Web::ResourceLoaderConnectorRequest>> s_all_requests;
class HeadlessRequestServer : public Web::ResourceLoaderConnector {
public:
class HTTPHeadlessRequest
: public Web::ResourceLoaderConnectorRequest
, public Weakable<HTTPHeadlessRequest> {
public:
static ErrorOr<NonnullRefPtr<HTTPHeadlessRequest>> create(String const& method, AK::URL const& url, HashMap<String, String> const& request_headers, ReadonlyBytes request_body, Core::ProxyData const&)
{
auto stream_backing_buffer = TRY(ByteBuffer::create_uninitialized(1 * MiB));
auto underlying_socket = TRY(Core::Stream::TCPSocket::connect(url.host(), url.port().value_or(80)));
TRY(underlying_socket->set_blocking(false));
auto socket = TRY(Core::Stream::BufferedSocket<Core::Stream::TCPSocket>::create(move(underlying_socket)));
HTTP::HttpRequest request;
if (method.equals_ignoring_case("head"sv))
request.set_method(HTTP::HttpRequest::HEAD);
else if (method.equals_ignoring_case("get"sv))
request.set_method(HTTP::HttpRequest::GET);
else if (method.equals_ignoring_case("post"sv))
request.set_method(HTTP::HttpRequest::POST);
else
request.set_method(HTTP::HttpRequest::Invalid);
request.set_url(move(url));
request.set_headers(request_headers);
request.set_body(TRY(ByteBuffer::copy(request_body)));
return adopt_ref(*new HTTPHeadlessRequest(move(request), move(socket), move(stream_backing_buffer)));
}
virtual ~HTTPHeadlessRequest() override
{
}
virtual void set_should_buffer_all_input(bool) override
{
}
virtual bool stop() override
{
return false;
}
virtual void stream_into(Core::Stream::Stream&) override
{
}
private:
HTTPHeadlessRequest(HTTP::HttpRequest&& request, NonnullOwnPtr<Core::Stream::BufferedSocketBase> socket, ByteBuffer&& stream_backing_buffer)
: m_stream_backing_buffer(move(stream_backing_buffer))
, m_output_stream(Core::Stream::MemoryStream::construct(m_stream_backing_buffer.bytes()).release_value_but_fixme_should_propagate_errors())
, m_socket(move(socket))
, m_job(HTTP::Job::construct(move(request), *m_output_stream))
{
m_job->on_headers_received = [weak_this = make_weak_ptr()](auto& response_headers, auto response_code) mutable {
if (auto strong_this = weak_this.strong_ref()) {
strong_this->m_response_code = response_code;
for (auto& header : response_headers) {
strong_this->m_response_headers.set(header.key, header.value);
}
}
};
m_job->on_finish = [weak_this = make_weak_ptr()](bool success) mutable {
Core::deferred_invoke([weak_this, success]() mutable {
if (auto strong_this = weak_this.strong_ref()) {
ReadonlyBytes response_bytes { strong_this->m_output_stream->bytes().data(), strong_this->m_output_stream->offset() };
auto response_buffer = ByteBuffer::copy(response_bytes).release_value_but_fixme_should_propagate_errors();
strong_this->on_buffered_request_finish(success, strong_this->m_output_stream->offset(), strong_this->m_response_headers, strong_this->m_response_code, response_buffer);
} });
};
m_job->start(*m_socket);
}
Optional<u32> m_response_code;
ByteBuffer m_stream_backing_buffer;
NonnullOwnPtr<Core::Stream::MemoryStream> m_output_stream;
NonnullOwnPtr<Core::Stream::BufferedSocketBase> m_socket;
NonnullRefPtr<HTTP::Job> m_job;
HashMap<String, String, CaseInsensitiveStringTraits> m_response_headers;
};
class HTTPSHeadlessRequest
: public Web::ResourceLoaderConnectorRequest
, public Weakable<HTTPSHeadlessRequest> {
public:
static ErrorOr<NonnullRefPtr<HTTPSHeadlessRequest>> create(String const& method, AK::URL const& url, HashMap<String, String> const& request_headers, ReadonlyBytes request_body, Core::ProxyData const&)
{
auto stream_backing_buffer = TRY(ByteBuffer::create_uninitialized(1 * MiB));
auto underlying_socket = TRY(TLS::TLSv12::connect(url.host(), url.port().value_or(443)));
TRY(underlying_socket->set_blocking(false));
auto socket = TRY(Core::Stream::BufferedSocket<TLS::TLSv12>::create(move(underlying_socket)));
HTTP::HttpRequest request;
if (method.equals_ignoring_case("head"sv))
request.set_method(HTTP::HttpRequest::HEAD);
else if (method.equals_ignoring_case("get"sv))
request.set_method(HTTP::HttpRequest::GET);
else if (method.equals_ignoring_case("post"sv))
request.set_method(HTTP::HttpRequest::POST);
else
request.set_method(HTTP::HttpRequest::Invalid);
request.set_url(move(url));
request.set_headers(request_headers);
request.set_body(TRY(ByteBuffer::copy(request_body)));
return adopt_ref(*new HTTPSHeadlessRequest(move(request), move(socket), move(stream_backing_buffer)));
}
virtual ~HTTPSHeadlessRequest() override
{
}
virtual void set_should_buffer_all_input(bool) override
{
}
virtual bool stop() override
{
return false;
}
virtual void stream_into(Core::Stream::Stream&) override
{
}
private:
HTTPSHeadlessRequest(HTTP::HttpRequest&& request, NonnullOwnPtr<Core::Stream::BufferedSocketBase> socket, ByteBuffer&& stream_backing_buffer)
: m_stream_backing_buffer(move(stream_backing_buffer))
, m_output_stream(Core::Stream::MemoryStream::construct(m_stream_backing_buffer.bytes()).release_value_but_fixme_should_propagate_errors())
, m_socket(move(socket))
, m_job(HTTP::HttpsJob::construct(move(request), *m_output_stream))
{
m_job->on_headers_received = [weak_this = make_weak_ptr()](auto& response_headers, auto response_code) mutable {
if (auto strong_this = weak_this.strong_ref()) {
strong_this->m_response_code = response_code;
for (auto& header : response_headers) {
strong_this->m_response_headers.set(header.key, header.value);
}
}
};
m_job->on_finish = [weak_this = make_weak_ptr()](bool success) mutable {
Core::deferred_invoke([weak_this, success]() mutable {
if (auto strong_this = weak_this.strong_ref()) {
ReadonlyBytes response_bytes { strong_this->m_output_stream->bytes().data(), strong_this->m_output_stream->offset() };
auto response_buffer = ByteBuffer::copy(response_bytes).release_value_but_fixme_should_propagate_errors();
strong_this->on_buffered_request_finish(success, strong_this->m_output_stream->offset(), strong_this->m_response_headers, strong_this->m_response_code, response_buffer);
} });
};
m_job->start(*m_socket);
}
Optional<u32> m_response_code;
ByteBuffer m_stream_backing_buffer;
NonnullOwnPtr<Core::Stream::MemoryStream> m_output_stream;
NonnullOwnPtr<Core::Stream::BufferedSocketBase> m_socket;
NonnullRefPtr<HTTP::HttpsJob> m_job;
HashMap<String, String, CaseInsensitiveStringTraits> m_response_headers;
};
class GeminiHeadlessRequest
: public Web::ResourceLoaderConnectorRequest
, public Weakable<GeminiHeadlessRequest> {
public:
static ErrorOr<NonnullRefPtr<GeminiHeadlessRequest>> create(String const&, AK::URL const& url, HashMap<String, String> const&, ReadonlyBytes, Core::ProxyData const&)
{
auto stream_backing_buffer = TRY(ByteBuffer::create_uninitialized(1 * MiB));
auto underlying_socket = TRY(Core::Stream::TCPSocket::connect(url.host(), url.port().value_or(80)));
TRY(underlying_socket->set_blocking(false));
auto socket = TRY(Core::Stream::BufferedSocket<Core::Stream::TCPSocket>::create(move(underlying_socket)));
Gemini::GeminiRequest request;
request.set_url(url);
return adopt_ref(*new GeminiHeadlessRequest(move(request), move(socket), move(stream_backing_buffer)));
}
virtual ~GeminiHeadlessRequest() override
{
}
virtual void set_should_buffer_all_input(bool) override
{
}
virtual bool stop() override
{
return false;
}
virtual void stream_into(Core::Stream::Stream&) override
{
}
private:
GeminiHeadlessRequest(Gemini::GeminiRequest&& request, NonnullOwnPtr<Core::Stream::BufferedSocketBase> socket, ByteBuffer&& stream_backing_buffer)
: m_stream_backing_buffer(move(stream_backing_buffer))
, m_output_stream(Core::Stream::MemoryStream::construct(m_stream_backing_buffer.bytes()).release_value_but_fixme_should_propagate_errors())
, m_socket(move(socket))
, m_job(Gemini::Job::construct(move(request), *m_output_stream))
{
m_job->on_headers_received = [weak_this = make_weak_ptr()](auto& response_headers, auto response_code) mutable {
if (auto strong_this = weak_this.strong_ref()) {
strong_this->m_response_code = response_code;
for (auto& header : response_headers) {
strong_this->m_response_headers.set(header.key, header.value);
}
}
};
m_job->on_finish = [weak_this = make_weak_ptr()](bool success) mutable {
Core::deferred_invoke([weak_this, success]() mutable {
if (auto strong_this = weak_this.strong_ref()) {
ReadonlyBytes response_bytes { strong_this->m_output_stream->bytes().data(), strong_this->m_output_stream->offset() };
auto response_buffer = ByteBuffer::copy(response_bytes).release_value_but_fixme_should_propagate_errors();
strong_this->on_buffered_request_finish(success, strong_this->m_output_stream->offset(), strong_this->m_response_headers, strong_this->m_response_code, response_buffer);
} });
};
m_job->start(*m_socket);
}
Optional<u32> m_response_code;
ByteBuffer m_stream_backing_buffer;
NonnullOwnPtr<Core::Stream::MemoryStream> m_output_stream;
NonnullOwnPtr<Core::Stream::BufferedSocketBase> m_socket;
NonnullRefPtr<Gemini::Job> m_job;
HashMap<String, String, CaseInsensitiveStringTraits> m_response_headers;
};
static NonnullRefPtr<HeadlessRequestServer> create()
{
return adopt_ref(*new HeadlessRequestServer());
}
virtual ~HeadlessRequestServer() override { }
virtual void prefetch_dns(AK::URL const&) override { }
virtual void preconnect(AK::URL const&) override { }
virtual RefPtr<Web::ResourceLoaderConnectorRequest> start_request(String const& method, AK::URL const& url, HashMap<String, String> const& request_headers, ReadonlyBytes request_body, Core::ProxyData const& proxy) override
{
RefPtr<Web::ResourceLoaderConnectorRequest> request;
if (url.protocol().equals_ignoring_case("http"sv)) {
auto request_or_error = HTTPHeadlessRequest::create(method, url, request_headers, request_body, proxy);
if (request_or_error.is_error())
return {};
request = request_or_error.release_value();
}
if (url.protocol().equals_ignoring_case("https"sv)) {
auto request_or_error = HTTPSHeadlessRequest::create(method, url, request_headers, request_body, proxy);
if (request_or_error.is_error())
return {};
request = request_or_error.release_value();
}
if (url.protocol().equals_ignoring_case("gemini"sv)) {
auto request_or_error = GeminiHeadlessRequest::create(method, url, request_headers, request_body, proxy);
if (request_or_error.is_error())
return {};
request = request_or_error.release_value();
}
if (request)
s_all_requests.set(request);
return request;
}
private:
HeadlessRequestServer() { }
};
class HeadlessWebSocketClientManager : public Web::WebSockets::WebSocketClientManager {
public:
class HeadlessWebSocket
@ -823,7 +549,7 @@ private:
void initialize_web_engine()
{
Web::ImageDecoding::Decoder::initialize(HeadlessImageDecoderClient::create());
Web::ResourceLoader::initialize(HeadlessRequestServer::create());
Web::ResourceLoader::initialize(RequestManagerQt::create());
Web::WebSockets::WebSocketClientManager::initialize(HeadlessWebSocketClientManager::create());
Web::FrameLoader::set_default_favicon_path(String::formatted("{}/res/icons/16x16/app-browser.png", s_serenity_resource_root));