subcompositor/renderer: fixup handling of subsurfaces below the main one

some apps (notably vlc 4) place a subsurface below the main surface (which is kinda cursed) but we have to accomodate for that

src/helpers/WLClasses.hpp

@@ -51,6 +51,9 @@ struct SRenderData {
     PHLWINDOW pWindow;
 
     bool      popup = false;
+
+    // counts how many surfaces this pass has rendered
+    int surfaceCounter = 0;
 };
 
 struct SSwipeGesture {

src/protocols/core/Compositor.cpp

@@ -277,8 +277,26 @@ void CWLSurfaceResource::resetRole() {
 }
 
 void CWLSurfaceResource::bfHelper(std::vector<SP<CWLSurfaceResource>> nodes, std::function<void(SP<CWLSurfaceResource>, const Vector2D&, void*)> fn, void* data) {
-    for (auto& n : nodes) {
 
+    std::vector<SP<CWLSurfaceResource>> nodes2;
+
+    // first, gather all nodes below
+    for (auto& n : nodes) {
+        std::erase_if(n->subsurfaces, [](const auto& e) { return e.expired(); });
+        // subsurfaces is sorted lowest -> highest
+        for (auto& c : n->subsurfaces) {
+            if (c->zIndex >= 0)
+                break;
+            nodes2.push_back(c->surface.lock());
+        }
+    }
+
+    if (!nodes2.empty())
+        bfHelper(nodes2, fn, data);
+
+    nodes2.clear();
+
+    for (auto& n : nodes) {
         Vector2D offset = {};
         if (n->role->role() == SURFACE_ROLE_SUBSURFACE) {
             auto subsurface = (CWLSubsurfaceResource*)n->role.get();
@@ -288,11 +306,10 @@ void CWLSurfaceResource::bfHelper(std::vector<SP<CWLSurfaceResource>> nodes, std
         fn(n, offset, data);
     }
 
-    std::vector<SP<CWLSurfaceResource>> nodes2;
-
     for (auto& n : nodes) {
-        std::erase_if(n->subsurfaces, [](const auto& e) { return e.expired(); });
         for (auto& c : n->subsurfaces) {
+            if (c->zIndex < 0)
+                continue;
             nodes2.push_back(c->surface.lock());
         }
     }

src/protocols/core/Subcompositor.cpp

@@ -23,15 +23,29 @@ CWLSubsurfaceResource::CWLSubsurfaceResource(SP<CWlSubsurface> resource_, SP<CWL
         if (!parent)
             return;
 
-        std::erase(parent->subsurfaces, self.lock());
+        auto pushAboveIndex = [this](int idx) -> void {
+            for (auto& c : parent->subsurfaces) {
+                if (c->zIndex >= idx)
+                    c->zIndex++;
+            }
+        };
+
+        std::erase_if(parent->subsurfaces, [this](const auto& e) { return e == self || !e; });
 
         auto it = std::find(parent->subsurfaces.begin(), parent->subsurfaces.end(), SURF);
 
         if (it == parent->subsurfaces.end()) {
-            LOGM(ERR, "Invalid surface reference in placeAbove");
-            parent->subsurfaces.emplace_back(self.lock());
-        } else
-            parent->subsurfaces.insert(it, self.lock());
+            LOGM(ERR, "Invalid surface reference in placeAbove, likely parent");
+            pushAboveIndex(1);
+            parent->subsurfaces.emplace_back(self);
+            zIndex = 1;
+        } else {
+            pushAboveIndex((*it)->zIndex);
+            zIndex = (*it)->zIndex;
+            parent->subsurfaces.emplace_back(self);
+        }
+
+        std::sort(parent->subsurfaces.begin(), parent->subsurfaces.end(), [](const auto& a, const auto& b) { return a->zIndex < b->zIndex; });
     });
 
     resource->setPlaceBelow([this](CWlSubsurface* r, wl_resource* surf) {
@@ -40,15 +54,29 @@ CWLSubsurfaceResource::CWLSubsurfaceResource(SP<CWlSubsurface> resource_, SP<CWL
         if (!parent)
             return;
 
-        std::erase(parent->subsurfaces, self.lock());
+        auto pushBelowIndex = [this](int idx) -> void {
+            for (auto& c : parent->subsurfaces) {
+                if (c->zIndex <= idx)
+                    c->zIndex--;
+            }
+        };
+
+        std::erase_if(parent->subsurfaces, [this](const auto& e) { return e == self || !e; });
 
         auto it = std::find(parent->subsurfaces.begin(), parent->subsurfaces.end(), SURF);
 
         if (it == parent->subsurfaces.end()) {
-            LOGM(ERR, "Invalid surface reference in placeBelow");
-            parent->subsurfaces.emplace_back(self.lock());
-        } else
-            parent->subsurfaces.insert(it--, self.lock());
+            LOGM(ERR, "Invalid surface reference in placeBelow, likely parent");
+            pushBelowIndex(-1);
+            parent->subsurfaces.emplace_back(self);
+            zIndex = -1;
+        } else {
+            pushBelowIndex((*it)->zIndex);
+            zIndex = (*it)->zIndex;
+            parent->subsurfaces.emplace_back(self);
+        }
+
+        std::sort(parent->subsurfaces.begin(), parent->subsurfaces.end(), [](const auto& a, const auto& b) { return a->zIndex < b->zIndex; });
     });
 
     listeners.commitSurface = surface->events.commit.registerListener([this](std::any d) {

src/protocols/core/Subcompositor.hpp

@@ -35,6 +35,8 @@ class CWLSubsurfaceResource : public ISurfaceRole {
 
     WP<CWLSubsurfaceResource> self;
 
+    int                       zIndex = 1; // by default, it's above
+
     struct {
         CSignal destroy;
     } events;

src/render/Renderer.cpp

@@ -209,7 +209,10 @@ static void renderSurface(SP<CWLSurfaceResource> surface, int x, int y, void* da
     else
         g_pHyprOpenGL->blend(true);
 
-    if (RDATA->surface && surface == RDATA->surface) {
+    // FIXME: This is wrong and will bug the blur out as shit if the first surface
+    // is a subsurface that does NOT cover the entire frame. In such cases, we probably should fall back
+    // to what we do for misaligned surfaces (blur the entire thing and then render shit without blur)
+    if (RDATA->surfaceCounter == 0) {
         if (RDATA->blur)
             g_pHyprOpenGL->renderTextureWithBlur(TEXTURE, &windowBox, ALPHA, surface, rounding, RDATA->blockBlurOptimization, RDATA->fadeAlpha);
         else
@@ -235,6 +238,9 @@ static void renderSurface(SP<CWLSurfaceResource> surface, int x, int y, void* da
     g_pHyprOpenGL->m_RenderData.primarySurfaceUVTopLeft     = Vector2D(-1, -1);
     g_pHyprOpenGL->m_RenderData.primarySurfaceUVBottomRight = Vector2D(-1, -1);
     g_pHyprOpenGL->m_RenderData.useNearestNeighbor          = NEARESTNEIGHBORSET;
+
+    // up the counter so that we dont blur any surfaces above this one
+    RDATA->surfaceCounter++;
 }
 
 bool CHyprRenderer::shouldRenderWindow(PHLWINDOW pWindow, CMonitor* pMonitor) {
@@ -602,6 +608,7 @@ void CHyprRenderer::renderWindow(PHLWINDOW pWindow, CMonitor* pMonitor, timespec
             renderdata.blur = false;
         }
 
+        renderdata.surfaceCounter = 0;
         pWindow->m_pWLSurface->resource()->breadthfirst([](SP<CWLSurfaceResource> s, const Vector2D& offset, void* data) { renderSurface(s, offset.x, offset.y, data); },
                                                         &renderdata);
 
@@ -658,6 +665,8 @@ void CHyprRenderer::renderWindow(PHLWINDOW pWindow, CMonitor* pMonitor, timespec
             if (pWindow->m_sAdditionalConfigData.nearestNeighbor.toUnderlying())
                 g_pHyprOpenGL->m_RenderData.useNearestNeighbor = true;
 
+            renderdata.surfaceCounter = 0;
+
             pWindow->m_pPopupHead->breadthfirst(
                 [](CPopup* popup, void* data) {
                     if (!popup->m_pWLSurface || !popup->m_pWLSurface->resource())
@@ -743,6 +752,7 @@ void CHyprRenderer::renderLayer(PHLLS pLayer, CMonitor* pMonitor, timespec* time
     renderdata.dontRound       = true;
     renderdata.popup           = true;
     renderdata.blur            = pLayer->forceBlurPopups;
+    renderdata.surfaceCounter  = 0;
     if (popups) {
         pLayer->popupHead->breadthfirst(
             [](CPopup* popup, void* data) {