From 79acb998e1493ef479a140a01e415251fe3d736a Mon Sep 17 00:00:00 2001 From: Aliaksandr Kalenik Date: Wed, 26 Jun 2024 17:56:55 +0200 Subject: [PATCH] LibCore+LibWeb: Use Metal backend for Skia painter on macOS If Metal context and IOSurface are available, Skia painter will use Ganesh GPU backend on macOS, which is noticeably faster than the default CPU backend. Painting pipeline: 1. (WebContent) Allocate IOSurface for backing store 2. (WebContent) Allocate MTLTexture that wraps IOSurface 3. (WebContent) Paint into MTLTexture using Skia 4. (Browser) Wrap IOSurface into Gfx::Painter and use QPainter/CoreGraphics to blit backing store into viewport. Things we should improve in the future: 1. Upload textures for images in advance instead of doing that before every repaint. 2. Teach AppKit client to read directly from IOSurface instead of copying. --- Userland/Libraries/LibCore/CMakeLists.txt | 2 + Userland/Libraries/LibCore/IOSurface.cpp | 5 + Userland/Libraries/LibCore/IOSurface.h | 2 + Userland/Libraries/LibCore/MetalContext.h | 39 ++++++++ Userland/Libraries/LibCore/MetalContext.mm | 92 ++++++++++++++++++ .../LibWeb/HTML/TraversableNavigable.cpp | 24 +++-- .../LibWeb/HTML/TraversableNavigable.h | 10 ++ .../LibWeb/Painting/DisplayListPlayerSkia.cpp | 94 ++++++++++++++++--- .../LibWeb/Painting/DisplayListPlayerSkia.h | 25 ++++- vcpkg.json | 16 +++- 10 files changed, 288 insertions(+), 21 deletions(-) create mode 100644 Userland/Libraries/LibCore/MetalContext.h create mode 100644 Userland/Libraries/LibCore/MetalContext.mm diff --git a/Userland/Libraries/LibCore/CMakeLists.txt b/Userland/Libraries/LibCore/CMakeLists.txt index a8cf281bb1b..f32f618fd66 100644 --- a/Userland/Libraries/LibCore/CMakeLists.txt +++ b/Userland/Libraries/LibCore/CMakeLists.txt @@ -85,6 +85,7 @@ endif() if (APPLE) list(APPEND SOURCES IOSurface.cpp) + list(APPEND SOURCES MetalContext.mm) endif() serenity_lib(LibCore core) @@ -96,6 +97,7 @@ if (APPLE) target_link_libraries(LibCore PUBLIC "-framework CoreServices") target_link_libraries(LibCore PUBLIC "-framework Foundation") target_link_libraries(LibCore PUBLIC "-framework IOSurface") + target_link_libraries(LibCore PUBLIC "-framework Metal") endif() if (ANDROID) diff --git a/Userland/Libraries/LibCore/IOSurface.cpp b/Userland/Libraries/LibCore/IOSurface.cpp index 4ede4210c16..5c721c4e201 100644 --- a/Userland/Libraries/LibCore/IOSurface.cpp +++ b/Userland/Libraries/LibCore/IOSurface.cpp @@ -124,4 +124,9 @@ void* IOSurfaceHandle::data() const return IOSurfaceGetBaseAddress(m_ref_wrapper->ref); } +void* IOSurfaceHandle::core_foundation_pointer() const +{ + return m_ref_wrapper->ref; +} + } diff --git a/Userland/Libraries/LibCore/IOSurface.h b/Userland/Libraries/LibCore/IOSurface.h index b33b7078d7a..9fc77480765 100644 --- a/Userland/Libraries/LibCore/IOSurface.h +++ b/Userland/Libraries/LibCore/IOSurface.h @@ -31,6 +31,8 @@ public: size_t bytes_per_row() const; void* data() const; + void* core_foundation_pointer() const; + ~IOSurfaceHandle(); private: diff --git a/Userland/Libraries/LibCore/MetalContext.h b/Userland/Libraries/LibCore/MetalContext.h new file mode 100644 index 00000000000..244643a7893 --- /dev/null +++ b/Userland/Libraries/LibCore/MetalContext.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2024, Aliaksandr Kalenik + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#if !defined(AK_OS_MACOS) +static_assert(false, "This file must only be used for macOS"); +#endif + +#include +#include + +namespace Core { + +class MetalTexture { +public: + virtual void const* texture() const = 0; + virtual size_t width() const = 0; + virtual size_t height() const = 0; + + virtual ~MetalTexture() {}; +}; + +class MetalContext { +public: + virtual void const* device() const = 0; + virtual void const* queue() const = 0; + + virtual OwnPtr create_texture_from_iosurface(IOSurfaceHandle const&) = 0; + + virtual ~MetalContext() {}; +}; + +OwnPtr get_metal_context(); + +} diff --git a/Userland/Libraries/LibCore/MetalContext.mm b/Userland/Libraries/LibCore/MetalContext.mm new file mode 100644 index 00000000000..33fcb9b2fa7 --- /dev/null +++ b/Userland/Libraries/LibCore/MetalContext.mm @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2024, Aliaksandr Kalenik + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include + +#define FixedPoint FixedPointMacOS +#define Duration DurationMacOS +#include +#undef FixedPoint +#undef Duration + +namespace Core { + +class MetalTextureImpl final : public MetalTexture { +public: + MetalTextureImpl(id texture) + : m_texture(texture) + { + } + + void const* texture() const override { return m_texture; } + size_t width() const override { return m_texture.width; } + size_t height() const override { return m_texture.height; } + + virtual ~MetalTextureImpl() + { + [m_texture release]; + } + +private: + id m_texture; +}; + +class MetalContextImpl final : public MetalContext { +public: + MetalContextImpl(id device, id queue) + : m_device(device) + , m_queue(queue) + { + } + + void const* device() const override { return m_device; } + void const* queue() const override { return m_queue; } + + OwnPtr create_texture_from_iosurface(IOSurfaceHandle const& iosurface) override + { + auto* const descriptor = [[MTLTextureDescriptor alloc] init]; + descriptor.pixelFormat = MTLPixelFormatBGRA8Unorm; + descriptor.width = iosurface.width(); + descriptor.height = iosurface.height(); + descriptor.storageMode = MTLStorageModeShared; + descriptor.usage = MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead; + + id texture = [m_device newTextureWithDescriptor:descriptor iosurface:(IOSurfaceRef)iosurface.core_foundation_pointer() plane:0]; + [descriptor release]; + return make(texture); + } + + virtual ~MetalContextImpl() override + { + [m_queue release]; + [m_device release]; + } + +private: + id m_device; + id m_queue; +}; + +OwnPtr get_metal_context() +{ + auto device = MTLCreateSystemDefaultDevice(); + if (!device) { + dbgln("Failed to create Metal device"); + return {}; + } + + auto queue = [device newCommandQueue]; + if (!queue) { + dbgln("Failed to create Metal command queue"); + [device release]; + return {}; + } + + return make(device, queue); +} + +} diff --git a/Userland/Libraries/LibWeb/HTML/TraversableNavigable.cpp b/Userland/Libraries/LibWeb/HTML/TraversableNavigable.cpp index 9204b33af6a..f315c848aa1 100644 --- a/Userland/Libraries/LibWeb/HTML/TraversableNavigable.cpp +++ b/Userland/Libraries/LibWeb/HTML/TraversableNavigable.cpp @@ -18,7 +18,6 @@ #include #include #include -#include #include #ifdef HAS_ACCELERATED_GRAPHICS @@ -33,6 +32,10 @@ TraversableNavigable::TraversableNavigable(JS::NonnullGCPtr page) : Navigable(page) , m_session_history_traversal_queue(vm().heap().allocate_without_realm()) { +#ifdef AK_OS_MACOS + m_metal_context = Core::get_metal_context(); + m_skia_backend_context = Web::Painting::DisplayListPlayerSkia::create_metal_context(*m_metal_context); +#endif } TraversableNavigable::~TraversableNavigable() = default; @@ -1174,10 +1177,8 @@ JS::GCPtr TraversableNavigable::currently_focused_area() return candidate; } -void TraversableNavigable::paint(Web::DevicePixelRect const& content_rect, Painting::BackingStore& backing_store, Web::PaintOptions paint_options) +void TraversableNavigable::paint(Web::DevicePixelRect const& content_rect, Painting::BackingStore& target, Web::PaintOptions paint_options) { - auto& target = backing_store.bitmap(); - Painting::DisplayList display_list; Painting::DisplayListRecorder display_list_recorder(display_list); @@ -1193,7 +1194,7 @@ void TraversableNavigable::paint(Web::DevicePixelRect const& content_rect, Paint auto display_list_player_type = page().client().display_list_player_type(); if (display_list_player_type == DisplayListPlayerType::GPU) { #ifdef HAS_ACCELERATED_GRAPHICS - Web::Painting::DisplayListPlayerGPU player(*paint_options.accelerated_graphics_context, target); + Web::Painting::DisplayListPlayerGPU player(*paint_options.accelerated_graphics_context, target.bitmap()); display_list.execute(player); #else static bool has_warned_about_configuration = false; @@ -1204,10 +1205,19 @@ void TraversableNavigable::paint(Web::DevicePixelRect const& content_rect, Paint } #endif } else if (display_list_player_type == DisplayListPlayerType::Skia) { - Painting::DisplayListPlayerSkia player(target); +#ifdef AK_OS_MACOS + if (m_metal_context && m_skia_backend_context && is(target)) { + auto& iosurface_backing_store = static_cast(target); + auto texture = m_metal_context->create_texture_from_iosurface(iosurface_backing_store.iosurface_handle()); + Painting::DisplayListPlayerSkia player(*m_skia_backend_context, *texture); + display_list.execute(player); + return; + } +#endif + Web::Painting::DisplayListPlayerSkia player(target.bitmap()); display_list.execute(player); } else { - Web::Painting::DisplayListPlayerCPU player(target); + Web::Painting::DisplayListPlayerCPU player(target.bitmap()); display_list.execute(player); } } diff --git a/Userland/Libraries/LibWeb/HTML/TraversableNavigable.h b/Userland/Libraries/LibWeb/HTML/TraversableNavigable.h index 82388c87ac2..6072474b7e6 100644 --- a/Userland/Libraries/LibWeb/HTML/TraversableNavigable.h +++ b/Userland/Libraries/LibWeb/HTML/TraversableNavigable.h @@ -12,8 +12,13 @@ #include #include #include +#include #include +#ifdef AK_OS_MACOS +# include +#endif + namespace Web::HTML { // https://html.spec.whatwg.org/multipage/document-sequences.html#traversable-navigable @@ -124,6 +129,11 @@ private: JS::NonnullGCPtr m_session_history_traversal_queue; String m_window_handle; + +#ifdef AK_OS_MACOS + OwnPtr m_skia_backend_context; + OwnPtr m_metal_context; +#endif }; struct BrowsingContextAndDocument { diff --git a/Userland/Libraries/LibWeb/Painting/DisplayListPlayerSkia.cpp b/Userland/Libraries/LibWeb/Painting/DisplayListPlayerSkia.cpp index 0945ba2cc95..797d7e5a521 100644 --- a/Userland/Libraries/LibWeb/Painting/DisplayListPlayerSkia.cpp +++ b/Userland/Libraries/LibWeb/Painting/DisplayListPlayerSkia.cpp @@ -17,6 +17,8 @@ #include #include #include +#include +#include #include #include @@ -24,8 +26,74 @@ #include #include +#ifdef AK_OS_MACOS +# define FixedPoint FixedPointMacOS +# define Duration DurationMacOS +# include +# include +# include +# undef FixedPoint +# undef Duration +#endif + namespace Web::Painting { +#ifdef AK_OS_MACOS +class SkiaMetalBackendContext final : public SkiaBackendContext { + AK_MAKE_NONCOPYABLE(SkiaMetalBackendContext); + AK_MAKE_NONMOVABLE(SkiaMetalBackendContext); + +public: + SkiaMetalBackendContext(sk_sp context) + : m_context(move(context)) + { + } + + ~SkiaMetalBackendContext() override {}; + + sk_sp wrap_metal_texture(Core::MetalTexture& metal_texture) + { + GrMtlTextureInfo mtl_info; + mtl_info.fTexture = sk_ret_cfp(metal_texture.texture()); + auto backend_render_target = GrBackendRenderTarget(metal_texture.width(), metal_texture.height(), mtl_info); + return SkSurfaces::WrapBackendRenderTarget(m_context.get(), backend_render_target, kTopLeft_GrSurfaceOrigin, kBGRA_8888_SkColorType, nullptr, nullptr); + } + + void flush_and_submit() override + { + m_context->flush(); + m_context->submit(GrSyncCpu::kYes); + } + +private: + sk_sp m_context; +}; + +OwnPtr DisplayListPlayerSkia::create_metal_context(Core::MetalContext const& metal_context) +{ + GrMtlBackendContext backend_context; + backend_context.fDevice.retain((GrMTLHandle)metal_context.device()); + backend_context.fQueue.retain((GrMTLHandle)metal_context.queue()); + sk_sp ctx = GrDirectContexts::MakeMetal(backend_context); + return make(ctx); +} + +DisplayListPlayerSkia::DisplayListPlayerSkia(SkiaBackendContext& context, Core::MetalTexture& metal_texture) +{ + auto image_info = SkImageInfo::Make(metal_texture.width(), metal_texture.height(), kBGRA_8888_SkColorType, kPremul_SkAlphaType); + VERIFY(is(context)); + auto surface = static_cast(context).wrap_metal_texture(metal_texture); + if (!surface) { + dbgln("Failed to create Skia surface from Metal texture"); + VERIFY_NOT_REACHED(); + } + m_surface = make(surface); + m_flush_context = [&context] mutable { + context.flush_and_submit(); + }; +} +#endif + class DisplayListPlayerSkia::SkiaSurface { public: SkCanvas& canvas() const { return *surface->getCanvas(); } @@ -39,6 +107,21 @@ private: sk_sp surface; }; +DisplayListPlayerSkia::DisplayListPlayerSkia(Gfx::Bitmap& bitmap) +{ + VERIFY(bitmap.format() == Gfx::BitmapFormat::BGRA8888); + auto image_info = SkImageInfo::Make(bitmap.width(), bitmap.height(), kBGRA_8888_SkColorType, kPremul_SkAlphaType); + auto surface = SkSurfaces::WrapPixels(image_info, bitmap.begin(), bitmap.pitch()); + VERIFY(surface); + m_surface = make(surface); +} + +DisplayListPlayerSkia::~DisplayListPlayerSkia() +{ + if (m_flush_context) + m_flush_context(); +} + static SkRect to_skia_rect(auto const& rect) { return SkRect::MakeXYWH(rect.x(), rect.y(), rect.width(), rect.height()); @@ -195,17 +278,6 @@ static SkSamplingOptions to_skia_sampling_options(Gfx::ScalingMode scaling_mode) surface().canvas().clipPath(to_skia_path(path), true); \ } -DisplayListPlayerSkia::DisplayListPlayerSkia(Gfx::Bitmap& bitmap) -{ - VERIFY(bitmap.format() == Gfx::BitmapFormat::BGRA8888); - auto image_info = SkImageInfo::Make(bitmap.width(), bitmap.height(), kBGRA_8888_SkColorType, kPremul_SkAlphaType); - auto surface = SkSurfaces::WrapPixels(image_info, bitmap.begin(), bitmap.width() * 4); - VERIFY(surface); - m_surface = make(surface); -} - -DisplayListPlayerSkia::~DisplayListPlayerSkia() = default; - DisplayListPlayerSkia::SkiaSurface& DisplayListPlayerSkia::surface() const { return static_cast(*m_surface); diff --git a/Userland/Libraries/LibWeb/Painting/DisplayListPlayerSkia.h b/Userland/Libraries/LibWeb/Painting/DisplayListPlayerSkia.h index 5c859adf4e3..c49d6f91023 100644 --- a/Userland/Libraries/LibWeb/Painting/DisplayListPlayerSkia.h +++ b/Userland/Libraries/LibWeb/Painting/DisplayListPlayerSkia.h @@ -9,8 +9,24 @@ #include #include +#ifdef AK_OS_MACOS +# include +# include +#endif + namespace Web::Painting { +class SkiaBackendContext { + AK_MAKE_NONCOPYABLE(SkiaBackendContext); + AK_MAKE_NONMOVABLE(SkiaBackendContext); + +public: + SkiaBackendContext() {}; + virtual ~SkiaBackendContext() {}; + + virtual void flush_and_submit() {}; +}; + class DisplayListPlayerSkia : public DisplayListPlayer { public: CommandResult draw_glyph_run(DrawGlyphRun const&) override; @@ -52,7 +68,13 @@ public: bool needs_update_immutable_bitmap_texture_cache() const override { return false; } void update_immutable_bitmap_texture_cache(HashMap&) override {}; - DisplayListPlayerSkia(Gfx::Bitmap& bitmap); + DisplayListPlayerSkia(Gfx::Bitmap&); + +#ifdef AK_OS_MACOS + static OwnPtr create_metal_context(Core::MetalContext const&); + DisplayListPlayerSkia(SkiaBackendContext&, Core::MetalTexture&); +#endif + virtual ~DisplayListPlayerSkia() override; private: @@ -60,6 +82,7 @@ private: SkiaSurface& surface() const; OwnPtr m_surface; + Function m_flush_context; }; } diff --git a/vcpkg.json b/vcpkg.json index 515ac4c5bf1..50b7396a6b0 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -9,9 +9,21 @@ "libjpeg-turbo", { "name": "libpng", - "features": [ "apng" ] + "features": [ + "apng" + ] + }, + { + "name": "skia", + "platform": "osx", + "features": [ + "metal" + ] + }, + { + "name": "skia", + "platform": "linux | freebsd | openbsd" }, - "skia", "sqlite3", "woff2" ],