LibHTTP+RequestServer: Add HTTP::HeaderMap and use for response headers

Instead of using a HashMap<ByteString, ByteString, CaseInsensitive...>
everywhere, we now encapsulate this in a class.

Even better, the new class also allows keeping track of multiple headers
with the same name! This will make it possible for HTTP responses to
actually retain all their headers on the perilous journey from
RequestServer to LibWeb.
This commit is contained in:
Andreas Kling 2024-06-09 11:28:37 +02:00 committed by Andreas Kling
parent c096608dd9
commit e636851481
Notes: sideshowbarker 2024-07-17 06:33:00 +09:00
21 changed files with 159 additions and 53 deletions

View File

@ -112,7 +112,7 @@ void RequestManagerQt::Request::did_finish()
{
auto buffer = m_reply.readAll();
auto http_status_code = m_reply.attribute(QNetworkRequest::Attribute::HttpStatusCodeAttribute).toInt();
HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> response_headers;
HTTP::HeaderMap response_headers;
Vector<ByteString> set_cookie_headers;
for (auto& it : m_reply.rawHeaderPairs()) {
auto name = ByteString(it.first.data(), it.first.length());

View File

@ -11,6 +11,7 @@
#include <AK/Stream.h>
#include <LibCore/EventReceiver.h>
#include <LibCore/Forward.h>
#include <LibHTTP/HeaderMap.h>
namespace Core {
@ -27,7 +28,7 @@ public:
virtual ~NetworkJob() override = default;
// Could fire twice, after Headers and after Trailers!
Function<void(HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> const& response_headers, Optional<u32> response_code)> on_headers_received;
Function<void(HTTP::HeaderMap const& response_headers, Optional<u32> response_code)> on_headers_received;
Function<void(bool success)> on_finish;
Function<void(Optional<u64>, u64)> on_progress;

View File

@ -0,0 +1,40 @@
/*
* Copyright (c) 2024, Andreas Kling <andreas@ladybird.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/ByteString.h>
#include <LibIPC/Decoder.h>
#include <LibIPC/Encoder.h>
namespace HTTP {
struct Header {
ByteString name;
ByteString value;
};
}
namespace IPC {
template<>
inline ErrorOr<void> encode(Encoder& encoder, HTTP::Header const& header)
{
TRY(encoder.encode(header.name));
TRY(encoder.encode(header.value));
return {};
}
template<>
inline ErrorOr<HTTP::Header> decode(Decoder& decoder)
{
auto name = TRY(decoder.decode<ByteString>());
auto value = TRY(decoder.decode<ByteString>());
return HTTP::Header { move(name), move(value) };
}
}

View File

@ -0,0 +1,65 @@
/*
* Copyright (c) 2024, Andreas Kling <andreas@ladybird.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <LibHTTP/Header.h>
namespace HTTP {
class HeaderMap {
public:
HeaderMap() = default;
~HeaderMap() = default;
void set(ByteString name, ByteString value)
{
m_map.set(name, value);
m_headers.append({ move(name), move(value) });
}
[[nodiscard]] bool contains(ByteString const& name) const
{
return m_map.contains(name);
}
[[nodiscard]] Optional<ByteString> get(ByteString const& name) const
{
return m_map.get(name);
}
[[nodiscard]] Vector<Header> const& headers() const
{
return m_headers;
}
private:
HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> m_map;
Vector<Header> m_headers;
};
}
namespace IPC {
template<>
inline ErrorOr<void> encode(Encoder& encoder, HTTP::HeaderMap const& header_map)
{
TRY(encoder.encode(header_map.headers()));
return {};
}
template<>
inline ErrorOr<HTTP::HeaderMap> decode(Decoder& decoder)
{
auto headers = TRY(decoder.decode<Vector<HTTP::Header>>());
HTTP::HeaderMap header_map;
for (auto& header : headers)
header_map.set(move(header.name), move(header.value));
return header_map;
}
}

View File

@ -9,7 +9,7 @@
namespace HTTP {
HttpResponse::HttpResponse(int code, HashMap<ByteString, ByteString, CaseInsensitiveStringTraits>&& headers, size_t size)
HttpResponse::HttpResponse(int code, HeaderMap&& headers, size_t size)
: m_code(code)
, m_headers(move(headers))
, m_downloaded_size(size)

View File

@ -10,13 +10,14 @@
#include <AK/ByteString.h>
#include <AK/HashMap.h>
#include <LibCore/NetworkResponse.h>
#include <LibHTTP/HeaderMap.h>
namespace HTTP {
class HttpResponse : public Core::NetworkResponse {
public:
virtual ~HttpResponse() override = default;
static NonnullRefPtr<HttpResponse> create(int code, HashMap<ByteString, ByteString, CaseInsensitiveStringTraits>&& headers, size_t downloaded_size)
static NonnullRefPtr<HttpResponse> create(int code, HeaderMap&& headers, size_t downloaded_size)
{
return adopt_ref(*new HttpResponse(code, move(headers), downloaded_size));
}
@ -24,15 +25,15 @@ public:
int code() const { return m_code; }
size_t downloaded_size() const { return m_downloaded_size; }
StringView reason_phrase() const { return reason_phrase_for_code(m_code); }
HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> const& headers() const { return m_headers; }
HeaderMap const& headers() const { return m_headers; }
static StringView reason_phrase_for_code(int code);
private:
HttpResponse(int code, HashMap<ByteString, ByteString, CaseInsensitiveStringTraits>&&, size_t size);
HttpResponse(int code, HeaderMap&&, size_t size);
int m_code { 0 };
HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> m_headers;
HeaderMap m_headers;
size_t m_downloaded_size { 0 };
};

View File

@ -53,7 +53,7 @@ protected:
Core::BufferedSocketBase* m_socket { nullptr };
bool m_legacy_connection { false };
int m_code { -1 };
HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> m_headers;
HTTP::HeaderMap m_headers;
Vector<ByteString> m_set_cookie_headers;
struct ReceivedBuffer {

View File

@ -94,7 +94,7 @@ void Request::did_progress(Badge<RequestClient>, Optional<u64> total_size, u64 d
on_progress(total_size, downloaded_size);
}
void Request::did_receive_headers(Badge<RequestClient>, HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> const& response_headers, Optional<u32> response_code)
void Request::did_receive_headers(Badge<RequestClient>, HTTP::HeaderMap const& response_headers, Optional<u32> response_code)
{
if (on_headers_received)
on_headers_received(response_headers, response_code);

View File

@ -14,6 +14,7 @@
#include <AK/RefCounted.h>
#include <AK/WeakPtr.h>
#include <LibCore/Notifier.h>
#include <LibHTTP/HeaderMap.h>
#include <LibIPC/Forward.h>
namespace Protocol {
@ -36,13 +37,13 @@ public:
int fd() const { return m_fd; }
bool stop();
using BufferedRequestFinished = Function<void(bool success, u64 total_size, HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> const& response_headers, Optional<u32> response_code, ReadonlyBytes payload)>;
using BufferedRequestFinished = Function<void(bool success, u64 total_size, HTTP::HeaderMap const& response_headers, Optional<u32> response_code, ReadonlyBytes payload)>;
// Configure the request such that the entirety of the response data is buffered. The callback receives that data and
// the response headers all at once. Using this method is mutually exclusive with `set_unbuffered_data_received_callback`.
void set_buffered_request_finished_callback(BufferedRequestFinished);
using HeadersReceived = Function<void(HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> const& response_headers, Optional<u32> response_code)>;
using HeadersReceived = Function<void(HTTP::HeaderMap const& response_headers, Optional<u32> response_code)>;
using DataReceived = Function<void(ReadonlyBytes data)>;
using RequestFinished = Function<void(bool success, u64 total_size)>;
@ -55,7 +56,7 @@ public:
void did_finish(Badge<RequestClient>, bool success, u64 total_size);
void did_progress(Badge<RequestClient>, Optional<u64> total_size, u64 downloaded_size);
void did_receive_headers(Badge<RequestClient>, HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> const& response_headers, Optional<u32> response_code);
void did_receive_headers(Badge<RequestClient>, HTTP::HeaderMap const& response_headers, Optional<u32> response_code);
void did_request_certificates(Badge<RequestClient>);
RefPtr<Core::Notifier>& write_notifier(Badge<RequestClient>) { return m_write_notifier; }
@ -83,7 +84,7 @@ private:
struct InternalBufferedData {
AllocatingMemoryStream payload_stream;
HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> response_headers;
HTTP::HeaderMap response_headers;
Optional<u32> response_code;
};

View File

@ -87,20 +87,14 @@ void RequestClient::request_progress(i32 request_id, Optional<u64> const& total_
}
}
void RequestClient::headers_became_available(i32 request_id, HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> const& response_headers, Optional<u32> const& status_code)
void RequestClient::headers_became_available(i32 request_id, HTTP::HeaderMap const& response_headers, Optional<u32> const& status_code)
{
auto request = const_cast<Request*>(m_requests.get(request_id).value_or(nullptr));
if (!request) {
warnln("Received headers for non-existent request {}", request_id);
return;
}
auto response_headers_clone_or_error = response_headers.clone();
if (response_headers_clone_or_error.is_error()) {
warnln("Error while receiving headers for request {}: {}", request_id, response_headers_clone_or_error.error());
return;
}
request->did_receive_headers({}, response_headers_clone_or_error.release_value(), status_code);
request->did_receive_headers({}, response_headers, status_code);
}
void RequestClient::certificate_requested(i32 request_id)

View File

@ -7,6 +7,7 @@
#pragma once
#include <AK/HashMap.h>
#include <LibHTTP/HeaderMap.h>
#include <LibIPC/ConnectionToServer.h>
#include <LibProtocol/WebSocket.h>
#include <LibWebSocket/WebSocket.h>
@ -42,7 +43,7 @@ private:
virtual void request_progress(i32, Optional<u64> const&, u64) override;
virtual void request_finished(i32, bool, u64) override;
virtual void certificate_requested(i32) override;
virtual void headers_became_available(i32, HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> const&, Optional<u32> const&) override;
virtual void headers_became_available(i32, HTTP::HeaderMap const&, Optional<u32> const&) override;
virtual void websocket_connected(i32) override;
virtual void websocket_received(i32, bool, ByteBuffer const&) override;

View File

@ -1985,7 +1985,7 @@ WebIDL::ExceptionOr<JS::NonnullGCPtr<PendingResponse>> nonstandard_resource_load
log_response(status_code, response_headers, ReadonlyBytes {});
}
for (auto const& [name, value] : response_headers) {
for (auto const& [name, value] : response_headers.headers()) {
auto header = Infrastructure::Header::from_string_pair(name, value);
response->header_list()->append(move(header));
}
@ -2050,7 +2050,7 @@ WebIDL::ExceptionOr<JS::NonnullGCPtr<PendingResponse>> nonstandard_resource_load
auto response = Infrastructure::Response::create(vm);
response->set_status(status_code.value_or(200));
response->set_body(move(body));
for (auto const& [name, value] : response_headers) {
for (auto const& [name, value] : response_headers.headers()) {
auto header = Infrastructure::Header::from_string_pair(name, value);
response->header_list()->append(move(header));
}
@ -2071,7 +2071,7 @@ WebIDL::ExceptionOr<JS::NonnullGCPtr<PendingResponse>> nonstandard_resource_load
response->set_status(status_code.value_or(400));
auto [body, _] = TRY_OR_IGNORE(extract_body(realm, data));
response->set_body(move(body));
for (auto const& [name, value] : response_headers) {
for (auto const& [name, value] : response_headers.headers()) {
auto header = Infrastructure::Header::from_string_pair(name, value);
response->header_list()->append(move(header));
}

View File

@ -183,13 +183,15 @@ void HTMLObjectElement::resource_did_load()
// 4. Run the appropriate set of steps from the following list:
// * If the resource has associated Content-Type metadata
if (auto it = resource()->response_headers().find("Content-Type"sv); it != resource()->response_headers().end()) {
if (auto maybe_content_type = resource()->response_headers().get("Content-Type"sv); maybe_content_type.has_value()) {
auto& content_type = maybe_content_type.value();
// 1. Let binary be false.
bool binary = false;
// 2. If the type specified in the resource's Content-Type metadata is "text/plain", and the result of applying the rules for distinguishing if a resource is text or binary to the resource is that the resource is not text/plain, then set binary to true.
if (it->value == "text/plain"sv) {
auto supplied_type = MimeSniff::MimeType::parse(it->value).release_value_but_fixme_should_propagate_errors();
if (content_type == "text/plain"sv) {
auto supplied_type = MimeSniff::MimeType::parse(content_type).release_value_but_fixme_should_propagate_errors();
auto computed_type = MimeSniff::Resource::sniff(resource()->encoded_data(), MimeSniff::SniffingConfiguration {
.sniffing_context = MimeSniff::SniffingContext::TextOrBinary,
.supplied_type = move(supplied_type),
@ -200,12 +202,12 @@ void HTMLObjectElement::resource_did_load()
}
// 3. If the type specified in the resource's Content-Type metadata is "application/octet-stream", then set binary to true.
if (it->value == "application/octet-stream"sv)
if (content_type == "application/octet-stream"sv)
binary = true;
// 4. If binary is false, then let the resource type be the type specified in the resource's Content-Type metadata, and jump to the step below labeled handler.
if (!binary)
return run_object_representation_handler_steps(it->value);
return run_object_representation_handler_steps(content_type);
// 5. If there is a type attribute present on the object element, and its value is not application/octet-stream, then run the following steps:
if (auto type = this->type(); !type.is_empty() && (type != "application/octet-stream"sv)) {

View File

@ -83,12 +83,12 @@ static bool is_valid_encoding(StringView encoding)
return TextCodec::decoder_for(encoding).has_value();
}
void Resource::did_load(Badge<ResourceLoader>, ReadonlyBytes data, HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> const& headers, Optional<u32> status_code)
void Resource::did_load(Badge<ResourceLoader>, ReadonlyBytes data, HTTP::HeaderMap const& headers, Optional<u32> status_code)
{
VERIFY(m_state == State::Pending);
// FIXME: Handle OOM failure.
m_encoded_data = ByteBuffer::copy(data).release_value_but_fixme_should_propagate_errors();
m_response_headers = headers.clone().release_value_but_fixme_should_propagate_errors();
m_response_headers = headers;
m_status_code = move(status_code);
m_state = State::Loaded;

View File

@ -14,6 +14,7 @@
#include <AK/WeakPtr.h>
#include <AK/Weakable.h>
#include <LibGfx/Forward.h>
#include <LibHTTP/HeaderMap.h>
#include <LibURL/URL.h>
#include <LibWeb/Forward.h>
#include <LibWeb/Loader/LoadRequest.h>
@ -53,7 +54,7 @@ public:
const URL::URL& url() const { return m_request.url(); }
ByteBuffer const& encoded_data() const { return m_encoded_data; }
HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> const& response_headers() const { return m_response_headers; }
[[nodiscard]] HTTP::HeaderMap const& response_headers() const { return m_response_headers; }
[[nodiscard]] Optional<u32> status_code() const { return m_status_code; }
@ -66,7 +67,7 @@ public:
void for_each_client(Function<void(ResourceClient&)>);
void did_load(Badge<ResourceLoader>, ReadonlyBytes data, HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> const& headers, Optional<u32> status_code);
void did_load(Badge<ResourceLoader>, ReadonlyBytes data, HTTP::HeaderMap const&, Optional<u32> status_code);
void did_fail(Badge<ResourceLoader>, ByteString const& error, Optional<u32> status_code);
protected:
@ -84,7 +85,7 @@ private:
Optional<ByteString> m_encoding;
ByteString m_mime_type;
HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> m_response_headers;
HTTP::HeaderMap m_response_headers;
Optional<u32> m_status_code;
HashTable<ResourceClient*> m_clients;
};

View File

@ -166,13 +166,13 @@ static void store_response_cookies(Page& page, URL::URL const& url, ByteString c
}
}
static HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> response_headers_for_file(StringView path, Optional<time_t> const& modified_time)
static HTTP::HeaderMap response_headers_for_file(StringView path, Optional<time_t> const& modified_time)
{
// For file:// and resource:// URLs, we have to guess the MIME type, since there's no HTTP header to tell us what
// it is. We insert a fake Content-Type header here, so that clients can use it to learn the MIME type.
auto mime_type = Core::guess_mime_type_based_on_filename(path);
HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> response_headers;
HTTP::HeaderMap response_headers;
response_headers.set("Content-Type"sv, mime_type);
if (modified_time.has_value()) {
@ -258,7 +258,7 @@ void ResourceLoader::load(LoadRequest& request, SuccessCallback success_callback
}
log_success(request);
HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> response_headers;
HTTP::HeaderMap response_headers;
response_headers.set("Content-Type"sv, "text/html"sv);
success_callback(maybe_response.release_value().bytes(), response_headers, {});
};
@ -267,7 +267,7 @@ void ResourceLoader::load(LoadRequest& request, SuccessCallback success_callback
dbgln_if(SPAM_DEBUG, "Loading about: URL {}", url);
log_success(request);
HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> response_headers;
HTTP::HeaderMap response_headers;
response_headers.set("Content-Type", "text/html; charset=UTF-8");
// About version page
@ -304,7 +304,7 @@ void ResourceLoader::load(LoadRequest& request, SuccessCallback success_callback
MUST(data_url.mime_type.serialized()),
StringView(data_url.body.bytes()));
HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> response_headers;
HTTP::HeaderMap response_headers;
response_headers.set("Content-Type", MUST(data_url.mime_type.serialized()).to_byte_string());
log_success(request);
@ -537,7 +537,7 @@ RefPtr<ResourceLoaderConnectorRequest> ResourceLoader::start_network_request(Loa
return protocol_request;
}
void ResourceLoader::handle_network_response_headers(LoadRequest const& request, HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> const& response_headers)
void ResourceLoader::handle_network_response_headers(LoadRequest const& request, HTTP::HeaderMap const& response_headers)
{
if (!request.page())
return;

View File

@ -72,13 +72,13 @@ public:
RefPtr<Resource> load_resource(Resource::Type, LoadRequest&);
using SuccessCallback = JS::SafeFunction<void(ReadonlyBytes, HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> const& response_headers, Optional<u32> status_code)>;
using ErrorCallback = JS::SafeFunction<void(ByteString const&, Optional<u32> status_code, ReadonlyBytes payload, HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> const& response_headers)>;
using SuccessCallback = JS::SafeFunction<void(ReadonlyBytes, HTTP::HeaderMap const& response_headers, Optional<u32> status_code)>;
using ErrorCallback = JS::SafeFunction<void(ByteString const&, Optional<u32> status_code, ReadonlyBytes payload, HTTP::HeaderMap const& response_headers)>;
using TimeoutCallback = JS::SafeFunction<void()>;
void load(LoadRequest&, SuccessCallback success_callback, ErrorCallback error_callback = nullptr, Optional<u32> timeout = {}, TimeoutCallback timeout_callback = nullptr);
using OnHeadersReceived = JS::SafeFunction<void(HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> const& response_headers, Optional<u32> status_code)>;
using OnHeadersReceived = JS::SafeFunction<void(HTTP::HeaderMap const& response_headers, Optional<u32> status_code)>;
using OnDataReceived = JS::SafeFunction<void(ReadonlyBytes data)>;
using OnComplete = JS::SafeFunction<void(bool success, Optional<StringView> error_message)>;
@ -107,7 +107,7 @@ private:
static ErrorOr<NonnullRefPtr<ResourceLoader>> try_create(NonnullRefPtr<ResourceLoaderConnector>);
RefPtr<ResourceLoaderConnectorRequest> start_network_request(LoadRequest const&);
void handle_network_response_headers(LoadRequest const&, HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> const&);
void handle_network_response_headers(LoadRequest const&, HTTP::HeaderMap const&);
void finish_network_request(NonnullRefPtr<ResourceLoaderConnectorRequest> const&);
int m_pending_loads { 0 };

View File

@ -231,9 +231,8 @@ Messages::RequestServer::StopRequestResponse ConnectionFromClient::stop_request(
void ConnectionFromClient::did_receive_headers(Badge<Request>, Request& request)
{
auto response_headers = request.response_headers().clone().release_value_but_fixme_should_propagate_errors();
auto lock = Threading::MutexLocker(m_ipc_mutex);
async_headers_became_available(request.id(), move(response_headers), request.status_code());
async_headers_became_available(request.id(), request.response_headers(), request.status_code());
}
void ConnectionFromClient::did_finish_request(Badge<Request>, Request& request, bool success)

View File

@ -21,9 +21,9 @@ void Request::stop()
m_client.did_finish_request({}, *this, false);
}
void Request::set_response_headers(HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> const& response_headers)
void Request::set_response_headers(HTTP::HeaderMap response_headers)
{
m_response_headers = response_headers;
m_response_headers = move(response_headers);
m_client.did_receive_headers({}, *this);
}

View File

@ -25,7 +25,7 @@ public:
Optional<u32> status_code() const { return m_status_code; }
Optional<u64> total_size() const { return m_total_size; }
size_t downloaded_size() const { return m_downloaded_size; }
HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> const& response_headers() const { return m_response_headers; }
HTTP::HeaderMap const& response_headers() const { return m_response_headers; }
void stop();
virtual void set_certificate(ByteString, ByteString);
@ -38,7 +38,7 @@ public:
void did_progress(Optional<u64> total_size, u64 downloaded_size);
void set_status_code(u32 status_code) { m_status_code = status_code; }
void did_request_certificates();
void set_response_headers(HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> const&);
void set_response_headers(HTTP::HeaderMap);
void set_downloaded_size(size_t size) { m_downloaded_size = size; }
Core::File const& output_stream() const { return *m_output_stream; }
@ -53,7 +53,7 @@ private:
Optional<u64> m_total_size {};
size_t m_downloaded_size { 0 };
NonnullOwnPtr<Core::File> m_output_stream;
HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> m_response_headers;
HTTP::HeaderMap m_response_headers;
};
}

View File

@ -1,3 +1,4 @@
#include <LibHTTP/HeaderMap.h>
#include <LibURL/URL.h>
endpoint RequestClient
@ -5,7 +6,7 @@ endpoint RequestClient
request_started(i32 request_id, IPC::File fd) =|
request_progress(i32 request_id, Optional<u64> total_size, u64 downloaded_size) =|
request_finished(i32 request_id, bool success, u64 total_size) =|
headers_became_available(i32 request_id, HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> response_headers, Optional<u32> status_code) =|
headers_became_available(i32 request_id, HTTP::HeaderMap response_headers, Optional<u32> status_code) =|
// Websocket API
// FIXME: See if this can be merged with the regular APIs