LibCore+LibWeb: Use Metal backend for Skia painter on macOS

If Metal context and IOSurface are available, Skia painter will use
Ganesh GPU backend on macOS, which is noticeably faster than the default
CPU backend.

Painting pipeline:
1. (WebContent) Allocate IOSurface for backing store
2. (WebContent) Allocate MTLTexture that wraps IOSurface
3. (WebContent) Paint into MTLTexture using Skia
4. (Browser) Wrap IOSurface into Gfx::Painter and use
   QPainter/CoreGraphics to blit backing store into viewport.

Things we should improve in the future:
1. Upload textures for images in advance instead of doing that before
   every repaint.
2. Teach AppKit client to read directly from IOSurface instead of
   copying.

Userland/Libraries/LibCore/CMakeLists.txt

@@ -85,6 +85,7 @@ endif()
 
 if (APPLE)
     list(APPEND SOURCES IOSurface.cpp)
+    list(APPEND SOURCES MetalContext.mm)
 endif()
 
 serenity_lib(LibCore core)
@@ -96,6 +97,7 @@ if (APPLE)
     target_link_libraries(LibCore PUBLIC "-framework CoreServices")
     target_link_libraries(LibCore PUBLIC "-framework Foundation")
     target_link_libraries(LibCore PUBLIC "-framework IOSurface")
+    target_link_libraries(LibCore PUBLIC "-framework Metal")
 endif()
 
 if (ANDROID)

Userland/Libraries/LibCore/IOSurface.cpp

@@ -124,4 +124,9 @@ void* IOSurfaceHandle::data() const
     return IOSurfaceGetBaseAddress(m_ref_wrapper->ref);
 }
 
+void* IOSurfaceHandle::core_foundation_pointer() const
+{
+    return m_ref_wrapper->ref;
+}
+
 }

Userland/Libraries/LibCore/IOSurface.h

@@ -31,6 +31,8 @@ public:
     size_t bytes_per_row() const;
     void* data() const;
 
+    void* core_foundation_pointer() const;
+
     ~IOSurfaceHandle();
 
 private:

Userland/Libraries/LibCore/MetalContext.h

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2024, Aliaksandr Kalenik <kalenik.aliaksandr@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#if !defined(AK_OS_MACOS)
+static_assert(false, "This file must only be used for macOS");
+#endif
+
+#include <AK/Forward.h>
+#include <LibCore/IOSurface.h>
+
+namespace Core {
+
+class MetalTexture {
+public:
+    virtual void const* texture() const = 0;
+    virtual size_t width() const = 0;
+    virtual size_t height() const = 0;
+
+    virtual ~MetalTexture() {};
+};
+
+class MetalContext {
+public:
+    virtual void const* device() const = 0;
+    virtual void const* queue() const = 0;
+
+    virtual OwnPtr<MetalTexture> create_texture_from_iosurface(IOSurfaceHandle const&) = 0;
+
+    virtual ~MetalContext() {};
+};
+
+OwnPtr<MetalContext> get_metal_context();
+
+}

Userland/Libraries/LibCore/MetalContext.mm

@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 2024, Aliaksandr Kalenik <kalenik.aliaksandr@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/OwnPtr.h>
+#include <LibCore/MetalContext.h>
+
+#define FixedPoint FixedPointMacOS
+#define Duration DurationMacOS
+#include <Metal/Metal.h>
+#undef FixedPoint
+#undef Duration
+
+namespace Core {
+
+class MetalTextureImpl final : public MetalTexture {
+public:
+    MetalTextureImpl(id<MTLTexture> texture)
+        : m_texture(texture)
+    {
+    }
+
+    void const* texture() const override { return m_texture; }
+    size_t width() const override { return m_texture.width; }
+    size_t height() const override { return m_texture.height; }
+
+    virtual ~MetalTextureImpl()
+    {
+        [m_texture release];
+    }
+
+private:
+    id<MTLTexture> m_texture;
+};
+
+class MetalContextImpl final : public MetalContext {
+public:
+    MetalContextImpl(id<MTLDevice> device, id<MTLCommandQueue> queue)
+        : m_device(device)
+        , m_queue(queue)
+    {
+    }
+
+    void const* device() const override { return m_device; }
+    void const* queue() const override { return m_queue; }
+
+    OwnPtr<MetalTexture> create_texture_from_iosurface(IOSurfaceHandle const& iosurface) override
+    {
+        auto* const descriptor = [[MTLTextureDescriptor alloc] init];
+        descriptor.pixelFormat = MTLPixelFormatBGRA8Unorm;
+        descriptor.width = iosurface.width();
+        descriptor.height = iosurface.height();
+        descriptor.storageMode = MTLStorageModeShared;
+        descriptor.usage = MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead;
+
+        id<MTLTexture> texture = [m_device newTextureWithDescriptor:descriptor iosurface:(IOSurfaceRef)iosurface.core_foundation_pointer() plane:0];
+        [descriptor release];
+        return make<MetalTextureImpl>(texture);
+    }
+
+    virtual ~MetalContextImpl() override
+    {
+        [m_queue release];
+        [m_device release];
+    }
+
+private:
+    id<MTLDevice> m_device;
+    id<MTLCommandQueue> m_queue;
+};
+
+OwnPtr<MetalContext> get_metal_context()
+{
+    auto device = MTLCreateSystemDefaultDevice();
+    if (!device) {
+        dbgln("Failed to create Metal device");
+        return {};
+    }
+
+    auto queue = [device newCommandQueue];
+    if (!queue) {
+        dbgln("Failed to create Metal command queue");
+        [device release];
+        return {};
+    }
+
+    return make<MetalContextImpl>(device, queue);
+}
+
+}

Userland/Libraries/LibWeb/HTML/TraversableNavigable.cpp

@@ -18,7 +18,6 @@
 #include <LibWeb/HTML/Window.h>
 #include <LibWeb/Page/Page.h>
 #include <LibWeb/Painting/DisplayListPlayerCPU.h>
-#include <LibWeb/Painting/DisplayListPlayerSkia.h>
 #include <LibWeb/Platform/EventLoopPlugin.h>
 
 #ifdef HAS_ACCELERATED_GRAPHICS
@@ -33,6 +32,10 @@ TraversableNavigable::TraversableNavigable(JS::NonnullGCPtr<Page> page)
     : Navigable(page)
     , m_session_history_traversal_queue(vm().heap().allocate_without_realm<SessionHistoryTraversalQueue>())
 {
+#ifdef AK_OS_MACOS
+    m_metal_context = Core::get_metal_context();
+    m_skia_backend_context = Web::Painting::DisplayListPlayerSkia::create_metal_context(*m_metal_context);
+#endif
 }
 
 TraversableNavigable::~TraversableNavigable() = default;
@@ -1174,10 +1177,8 @@ JS::GCPtr<DOM::Node> TraversableNavigable::currently_focused_area()
     return candidate;
 }
 
-void TraversableNavigable::paint(Web::DevicePixelRect const& content_rect, Painting::BackingStore& backing_store, Web::PaintOptions paint_options)
+void TraversableNavigable::paint(Web::DevicePixelRect const& content_rect, Painting::BackingStore& target, Web::PaintOptions paint_options)
 {
-    auto& target = backing_store.bitmap();
-
     Painting::DisplayList display_list;
     Painting::DisplayListRecorder display_list_recorder(display_list);
 
@@ -1193,7 +1194,7 @@ void TraversableNavigable::paint(Web::DevicePixelRect const& content_rect, Paint
     auto display_list_player_type = page().client().display_list_player_type();
     if (display_list_player_type == DisplayListPlayerType::GPU) {
 #ifdef HAS_ACCELERATED_GRAPHICS
-        Web::Painting::DisplayListPlayerGPU player(*paint_options.accelerated_graphics_context, target);
+        Web::Painting::DisplayListPlayerGPU player(*paint_options.accelerated_graphics_context, target.bitmap());
         display_list.execute(player);
 #else
         static bool has_warned_about_configuration = false;
@@ -1204,10 +1205,19 @@ void TraversableNavigable::paint(Web::DevicePixelRect const& content_rect, Paint
         }
 #endif
     } else if (display_list_player_type == DisplayListPlayerType::Skia) {
-        Painting::DisplayListPlayerSkia player(target);
+#ifdef AK_OS_MACOS
+        if (m_metal_context && m_skia_backend_context && is<Painting::IOSurfaceBackingStore>(target)) {
+            auto& iosurface_backing_store = static_cast<Painting::IOSurfaceBackingStore&>(target);
+            auto texture = m_metal_context->create_texture_from_iosurface(iosurface_backing_store.iosurface_handle());
+            Painting::DisplayListPlayerSkia player(*m_skia_backend_context, *texture);
+            display_list.execute(player);
+            return;
+        }
+#endif
+        Web::Painting::DisplayListPlayerSkia player(target.bitmap());
         display_list.execute(player);
     } else {
-        Web::Painting::DisplayListPlayerCPU player(target);
+        Web::Painting::DisplayListPlayerCPU player(target.bitmap());
         display_list.execute(player);
     }
 }

Userland/Libraries/LibWeb/HTML/TraversableNavigable.h

@@ -12,8 +12,13 @@
 #include <LibWeb/HTML/SessionHistoryTraversalQueue.h>
 #include <LibWeb/HTML/VisibilityState.h>
 #include <LibWeb/Page/Page.h>
+#include <LibWeb/Painting/DisplayListPlayerSkia.h>
 #include <WebContent/BackingStoreManager.h>
 
+#ifdef AK_OS_MACOS
+#    include <LibCore/MetalContext.h>
+#endif
+
 namespace Web::HTML {
 
 // https://html.spec.whatwg.org/multipage/document-sequences.html#traversable-navigable
@@ -124,6 +129,11 @@ private:
     JS::NonnullGCPtr<SessionHistoryTraversalQueue> m_session_history_traversal_queue;
 
     String m_window_handle;
+
+#ifdef AK_OS_MACOS
+    OwnPtr<Web::Painting::SkiaBackendContext> m_skia_backend_context;
+    OwnPtr<Core::MetalContext> m_metal_context;
+#endif
 };
 
 struct BrowsingContextAndDocument {

Userland/Libraries/LibWeb/Painting/DisplayListPlayerSkia.cpp

@@ -17,6 +17,8 @@
 #include <core/SkSurface.h>
 #include <effects/SkGradientShader.h>
 #include <effects/SkImageFilters.h>
+#include <gpu/GrDirectContext.h>
+#include <gpu/ganesh/SkSurfaceGanesh.h>
 #include <pathops/SkPathOps.h>
 
 #include <LibGfx/Filters/StackBlurFilter.h>
@@ -24,8 +26,74 @@
 #include <LibWeb/Painting/DisplayListPlayerSkia.h>
 #include <LibWeb/Painting/ShadowPainting.h>
 
+#ifdef AK_OS_MACOS
+#    define FixedPoint FixedPointMacOS
+#    define Duration DurationMacOS
+#    include <gpu/GrBackendSurface.h>
+#    include <gpu/ganesh/mtl/GrMtlBackendContext.h>
+#    include <gpu/ganesh/mtl/GrMtlDirectContext.h>
+#    undef FixedPoint
+#    undef Duration
+#endif
+
 namespace Web::Painting {
 
+#ifdef AK_OS_MACOS
+class SkiaMetalBackendContext final : public SkiaBackendContext {
+    AK_MAKE_NONCOPYABLE(SkiaMetalBackendContext);
+    AK_MAKE_NONMOVABLE(SkiaMetalBackendContext);
+
+public:
+    SkiaMetalBackendContext(sk_sp<GrDirectContext> context)
+        : m_context(move(context))
+    {
+    }
+
+    ~SkiaMetalBackendContext() override {};
+
+    sk_sp<SkSurface> wrap_metal_texture(Core::MetalTexture& metal_texture)
+    {
+        GrMtlTextureInfo mtl_info;
+        mtl_info.fTexture = sk_ret_cfp(metal_texture.texture());
+        auto backend_render_target = GrBackendRenderTarget(metal_texture.width(), metal_texture.height(), mtl_info);
+        return SkSurfaces::WrapBackendRenderTarget(m_context.get(), backend_render_target, kTopLeft_GrSurfaceOrigin, kBGRA_8888_SkColorType, nullptr, nullptr);
+    }
+
+    void flush_and_submit() override
+    {
+        m_context->flush();
+        m_context->submit(GrSyncCpu::kYes);
+    }
+
+private:
+    sk_sp<GrDirectContext> m_context;
+};
+
+OwnPtr<SkiaBackendContext> DisplayListPlayerSkia::create_metal_context(Core::MetalContext const& metal_context)
+{
+    GrMtlBackendContext backend_context;
+    backend_context.fDevice.retain((GrMTLHandle)metal_context.device());
+    backend_context.fQueue.retain((GrMTLHandle)metal_context.queue());
+    sk_sp<GrDirectContext> ctx = GrDirectContexts::MakeMetal(backend_context);
+    return make<SkiaMetalBackendContext>(ctx);
+}
+
+DisplayListPlayerSkia::DisplayListPlayerSkia(SkiaBackendContext& context, Core::MetalTexture& metal_texture)
+{
+    auto image_info = SkImageInfo::Make(metal_texture.width(), metal_texture.height(), kBGRA_8888_SkColorType, kPremul_SkAlphaType);
+    VERIFY(is<SkiaMetalBackendContext>(context));
+    auto surface = static_cast<SkiaMetalBackendContext&>(context).wrap_metal_texture(metal_texture);
+    if (!surface) {
+        dbgln("Failed to create Skia surface from Metal texture");
+        VERIFY_NOT_REACHED();
+    }
+    m_surface = make<SkiaSurface>(surface);
+    m_flush_context = [&context] mutable {
+        context.flush_and_submit();
+    };
+}
+#endif
+
 class DisplayListPlayerSkia::SkiaSurface {
 public:
     SkCanvas& canvas() const { return *surface->getCanvas(); }
@@ -39,6 +107,21 @@ private:
     sk_sp<SkSurface> surface;
 };
 
+DisplayListPlayerSkia::DisplayListPlayerSkia(Gfx::Bitmap& bitmap)
+{
+    VERIFY(bitmap.format() == Gfx::BitmapFormat::BGRA8888);
+    auto image_info = SkImageInfo::Make(bitmap.width(), bitmap.height(), kBGRA_8888_SkColorType, kPremul_SkAlphaType);
+    auto surface = SkSurfaces::WrapPixels(image_info, bitmap.begin(), bitmap.pitch());
+    VERIFY(surface);
+    m_surface = make<SkiaSurface>(surface);
+}
+
+DisplayListPlayerSkia::~DisplayListPlayerSkia()
+{
+    if (m_flush_context)
+        m_flush_context();
+}
+
 static SkRect to_skia_rect(auto const& rect)
 {
     return SkRect::MakeXYWH(rect.x(), rect.y(), rect.width(), rect.height());
@@ -195,17 +278,6 @@ static SkSamplingOptions to_skia_sampling_options(Gfx::ScalingMode scaling_mode)
             surface().canvas().clipPath(to_skia_path(path), true); \
     }
 
-DisplayListPlayerSkia::DisplayListPlayerSkia(Gfx::Bitmap& bitmap)
-{
-    VERIFY(bitmap.format() == Gfx::BitmapFormat::BGRA8888);
-    auto image_info = SkImageInfo::Make(bitmap.width(), bitmap.height(), kBGRA_8888_SkColorType, kPremul_SkAlphaType);
-    auto surface = SkSurfaces::WrapPixels(image_info, bitmap.begin(), bitmap.width() * 4);
-    VERIFY(surface);
-    m_surface = make<SkiaSurface>(surface);
-}
-
-DisplayListPlayerSkia::~DisplayListPlayerSkia() = default;
-
 DisplayListPlayerSkia::SkiaSurface& DisplayListPlayerSkia::surface() const
 {
     return static_cast<SkiaSurface&>(*m_surface);

Userland/Libraries/LibWeb/Painting/DisplayListPlayerSkia.h

@@ -9,8 +9,24 @@
 #include <LibGfx/Bitmap.h>
 #include <LibWeb/Painting/DisplayListRecorder.h>
 
+#ifdef AK_OS_MACOS
+#    include <LibCore/IOSurface.h>
+#    include <LibCore/MetalContext.h>
+#endif
+
 namespace Web::Painting {
 
+class SkiaBackendContext {
+    AK_MAKE_NONCOPYABLE(SkiaBackendContext);
+    AK_MAKE_NONMOVABLE(SkiaBackendContext);
+
+public:
+    SkiaBackendContext() {};
+    virtual ~SkiaBackendContext() {};
+
+    virtual void flush_and_submit() {};
+};
+
 class DisplayListPlayerSkia : public DisplayListPlayer {
 public:
     CommandResult draw_glyph_run(DrawGlyphRun const&) override;
@@ -52,7 +68,13 @@ public:
     bool needs_update_immutable_bitmap_texture_cache() const override { return false; }
     void update_immutable_bitmap_texture_cache(HashMap<u32, Gfx::ImmutableBitmap const*>&) override {};
 
-    DisplayListPlayerSkia(Gfx::Bitmap& bitmap);
+    DisplayListPlayerSkia(Gfx::Bitmap&);
+
+#ifdef AK_OS_MACOS
+    static OwnPtr<SkiaBackendContext> create_metal_context(Core::MetalContext const&);
+    DisplayListPlayerSkia(SkiaBackendContext&, Core::MetalTexture&);
+#endif
+
     virtual ~DisplayListPlayerSkia() override;
 
 private:
@@ -60,6 +82,7 @@ private:
     SkiaSurface& surface() const;
 
     OwnPtr<SkiaSurface> m_surface;
+    Function<void()> m_flush_context;
 };
 
 }

vcpkg.json

@@ -9,9 +9,21 @@
     "libjpeg-turbo",
     {
       "name": "libpng",
-      "features": [ "apng" ]
+      "features": [
+        "apng"
+      ]
+    },
+    {
+      "name": "skia",
+      "platform": "osx",
+      "features": [
+        "metal"
+      ]
+    },
+    {
+      "name": "skia",
+      "platform": "linux | freebsd | openbsd"
     },
-    "skia",
     "sqlite3",
     "woff2"
   ],