From 3b3e0f50b487a0d59fdbe552642364ce8d3918bc Mon Sep 17 00:00:00 2001 From: Vaxry Date: Fri, 3 Nov 2023 19:04:49 +0000 Subject: [PATCH] alpha mask stuff --- src/render/OpenGL.cpp | 53 ++++++++++++++----- src/render/OpenGL.hpp | 2 +- src/render/Shader.hpp | 13 +++-- .../decorations/CHyprDropShadowDecoration.cpp | 39 +++++--------- src/render/shaders/Shadow.hpp | 7 +++ src/render/shaders/Textures.hpp | 3 ++ 6 files changed, 72 insertions(+), 45 deletions(-) diff --git a/src/render/OpenGL.cpp b/src/render/OpenGL.cpp index d23a1d38..d8637a53 100644 --- a/src/render/OpenGL.cpp +++ b/src/render/OpenGL.cpp @@ -329,18 +329,21 @@ void CHyprOpenGLImpl::initShaders() { m_RenderData.pCurrentMonData->m_shBLURFINISH.contrast = glGetUniformLocation(prog, "contrast"); m_RenderData.pCurrentMonData->m_shBLURFINISH.brightness = glGetUniformLocation(prog, "brightness"); - prog = createProgram(QUADVERTSRC, FRAGSHADOW); - m_RenderData.pCurrentMonData->m_shSHADOW.program = prog; - m_RenderData.pCurrentMonData->m_shSHADOW.proj = glGetUniformLocation(prog, "proj"); - m_RenderData.pCurrentMonData->m_shSHADOW.posAttrib = glGetAttribLocation(prog, "pos"); - m_RenderData.pCurrentMonData->m_shSHADOW.texAttrib = glGetAttribLocation(prog, "texcoord"); - m_RenderData.pCurrentMonData->m_shSHADOW.topLeft = glGetUniformLocation(prog, "topLeft"); - m_RenderData.pCurrentMonData->m_shSHADOW.bottomRight = glGetUniformLocation(prog, "bottomRight"); - m_RenderData.pCurrentMonData->m_shSHADOW.fullSize = glGetUniformLocation(prog, "fullSize"); - m_RenderData.pCurrentMonData->m_shSHADOW.radius = glGetUniformLocation(prog, "radius"); - m_RenderData.pCurrentMonData->m_shSHADOW.range = glGetUniformLocation(prog, "range"); - m_RenderData.pCurrentMonData->m_shSHADOW.shadowPower = glGetUniformLocation(prog, "shadowPower"); - m_RenderData.pCurrentMonData->m_shSHADOW.color = glGetUniformLocation(prog, "color"); + prog = createProgram(QUADVERTSRC, FRAGSHADOW); + m_RenderData.pCurrentMonData->m_shSHADOW.program = prog; + m_RenderData.pCurrentMonData->m_shSHADOW.proj = glGetUniformLocation(prog, "proj"); + m_RenderData.pCurrentMonData->m_shSHADOW.posAttrib = glGetAttribLocation(prog, "pos"); + m_RenderData.pCurrentMonData->m_shSHADOW.texAttrib = glGetAttribLocation(prog, "texcoord"); + m_RenderData.pCurrentMonData->m_shSHADOW.matteTexAttrib = glGetAttribLocation(prog, "texcoordMatte"); + m_RenderData.pCurrentMonData->m_shSHADOW.alphaMatte = glGetUniformLocation(prog, "alphaMatte"); + m_RenderData.pCurrentMonData->m_shSHADOW.topLeft = glGetUniformLocation(prog, "topLeft"); + m_RenderData.pCurrentMonData->m_shSHADOW.bottomRight = glGetUniformLocation(prog, "bottomRight"); + m_RenderData.pCurrentMonData->m_shSHADOW.fullSize = glGetUniformLocation(prog, "fullSize"); + m_RenderData.pCurrentMonData->m_shSHADOW.radius = glGetUniformLocation(prog, "radius"); + m_RenderData.pCurrentMonData->m_shSHADOW.range = glGetUniformLocation(prog, "range"); + m_RenderData.pCurrentMonData->m_shSHADOW.shadowPower = glGetUniformLocation(prog, "shadowPower"); + m_RenderData.pCurrentMonData->m_shSHADOW.color = glGetUniformLocation(prog, "color"); + m_RenderData.pCurrentMonData->m_shSHADOW.useAlphaMatte = glGetUniformLocation(prog, "useAlphaMatte"); prog = createProgram(QUADVERTSRC, FRAGBORDER1); m_RenderData.pCurrentMonData->m_shBORDER1.program = prog; @@ -1637,7 +1640,7 @@ void CHyprOpenGLImpl::renderSnapshot(SLayerSurface** pLayer) { m_bEndFrame = false; } -void CHyprOpenGLImpl::renderRoundedShadow(wlr_box* box, int round, int range, float a) { +void CHyprOpenGLImpl::renderRoundedShadow(wlr_box* box, int round, int range, float a, CFramebuffer* matte) { RASSERT(m_RenderData.pMonitor, "Tried to render shadow without begin()!"); RASSERT((box->width > 0 && box->height > 0), "Tried to render shadow with width/height < 0!"); RASSERT(m_pCurrentWindow, "Tried to render shadow without a window!"); @@ -1657,6 +1660,7 @@ void CHyprOpenGLImpl::renderRoundedShadow(wlr_box* box, int round, int range, fl static auto* const PSHADOWPOWER = &g_pConfigManager->getConfigValuePtr("decoration:shadow_render_power")->intValue; const auto SHADOWPOWER = std::clamp((int)*PSHADOWPOWER, 1, 4); + const auto USEMATTE = matte; const auto col = m_pCurrentWindow->m_cRealShadowColor.col(); @@ -1691,11 +1695,33 @@ void CHyprOpenGLImpl::renderRoundedShadow(wlr_box* box, int round, int range, fl glUniform1f(m_RenderData.pCurrentMonData->m_shSHADOW.range, range); glUniform1f(m_RenderData.pCurrentMonData->m_shSHADOW.shadowPower, SHADOWPOWER); + if (USEMATTE) { + glUniform1i(m_RenderData.pCurrentMonData->m_shSHADOW.useAlphaMatte, 1); + glUniform1i(m_RenderData.pCurrentMonData->m_shSHADOW.alphaMatte, 0); + glActiveTexture(GL_TEXTURE0); + glBindTexture(matte->m_cTex.m_iTarget, matte->m_cTex.m_iTexID); + } else { + glUniform1i(m_RenderData.pCurrentMonData->m_shSHADOW.useAlphaMatte, 0); + } + + const float texVerts[] = { + ((float)(box->x + box->width) / m_RenderData.pMonitor->vecPixelSize.x), + ((float)box->y / m_RenderData.pMonitor->vecPixelSize.y), // top right + ((float)box->x / m_RenderData.pMonitor->vecPixelSize.x), + ((float)box->y / m_RenderData.pMonitor->vecPixelSize.y), // top left + ((float)(box->x + box->width) / m_RenderData.pMonitor->vecPixelSize.x), + ((float)(box->y + box->height) / m_RenderData.pMonitor->vecPixelSize.y), // bottom right + ((float)box->x / m_RenderData.pMonitor->vecPixelSize.x), + ((float)(box->y + box->height) / m_RenderData.pMonitor->vecPixelSize.y), // bottom left + }; + glVertexAttribPointer(m_RenderData.pCurrentMonData->m_shSHADOW.posAttrib, 2, GL_FLOAT, GL_FALSE, 0, fullVerts); glVertexAttribPointer(m_RenderData.pCurrentMonData->m_shSHADOW.texAttrib, 2, GL_FLOAT, GL_FALSE, 0, fullVerts); + glVertexAttribPointer(m_RenderData.pCurrentMonData->m_shSHADOW.matteTexAttrib, 2, GL_FLOAT, GL_FALSE, 0, texVerts); glEnableVertexAttribArray(m_RenderData.pCurrentMonData->m_shSHADOW.posAttrib); glEnableVertexAttribArray(m_RenderData.pCurrentMonData->m_shSHADOW.texAttrib); + glEnableVertexAttribArray(m_RenderData.pCurrentMonData->m_shSHADOW.matteTexAttrib); if (m_RenderData.clipBox.width != 0 && m_RenderData.clipBox.height != 0) { CRegion damageClip{m_RenderData.clipBox.x, m_RenderData.clipBox.y, m_RenderData.clipBox.width, m_RenderData.clipBox.height}; @@ -1714,6 +1740,7 @@ void CHyprOpenGLImpl::renderRoundedShadow(wlr_box* box, int round, int range, fl } } + glDisableVertexAttribArray(m_RenderData.pCurrentMonData->m_shSHADOW.matteTexAttrib); glDisableVertexAttribArray(m_RenderData.pCurrentMonData->m_shSHADOW.posAttrib); glDisableVertexAttribArray(m_RenderData.pCurrentMonData->m_shSHADOW.texAttrib); } diff --git a/src/render/OpenGL.hpp b/src/render/OpenGL.hpp index 0d4b7852..ab3eb7cf 100644 --- a/src/render/OpenGL.hpp +++ b/src/render/OpenGL.hpp @@ -111,7 +111,7 @@ class CHyprOpenGLImpl { void renderTexture(wlr_texture*, wlr_box*, float a, int round = 0, bool allowCustomUV = false); void renderTexture(const CTexture&, wlr_box*, float a, int round = 0, bool discardActive = false, bool allowCustomUV = false); void renderTextureWithBlur(const CTexture&, wlr_box*, float a, wlr_surface* pSurface, int round = 0, bool blockBlurOptimization = false, float blurA = 1.f); - void renderRoundedShadow(wlr_box*, int round, int range, float a = 1.0); + void renderRoundedShadow(wlr_box*, int round, int range, float a = 1.0, CFramebuffer* matte = nullptr); void renderBorder(wlr_box*, const CGradientValueData&, int round, int borderSize, float a = 1.0, int outerRound = -1 /* use round */); void saveMatrix(); diff --git a/src/render/Shader.hpp b/src/render/Shader.hpp index e972eeb9..b67f61ef 100644 --- a/src/render/Shader.hpp +++ b/src/render/Shader.hpp @@ -10,10 +10,12 @@ class CShader { GLuint program = 0; GLint proj = -1; GLint color = -1; + GLint alphaMatte = -1; GLint tex = -1; GLint alpha = -1; GLint posAttrib = -1; GLint texAttrib = -1; + GLint matteTexAttrib = -1; GLint discardOpaque = -1; GLint discardAlpha = -1; GLfloat discardAlphaValue = -1; @@ -29,8 +31,9 @@ class CShader { GLint halfpixel = -1; - GLint range = -1; - GLint shadowPower = -1; + GLint range = -1; + GLint shadowPower = -1; + GLint useAlphaMatte = -1; // always inverted GLint applyTint = -1; GLint tint = -1; @@ -43,9 +46,9 @@ class CShader { GLint distort = -1; GLint output = -1; - GLint noise = -1; - GLint contrast = -1; - GLint brightness = -1; + GLint noise = -1; + GLint contrast = -1; + GLint brightness = -1; GLint getUniformLocation(const std::string&); diff --git a/src/render/decorations/CHyprDropShadowDecoration.cpp b/src/render/decorations/CHyprDropShadowDecoration.cpp index 367d03f0..55aa69ac 100644 --- a/src/render/decorations/CHyprDropShadowDecoration.cpp +++ b/src/render/decorations/CHyprDropShadowDecoration.cpp @@ -68,10 +68,9 @@ void CHyprDropShadowDecoration::updateWindow(CWindow* pWindow) { maxExtents.bottomRight.y = EXTENTS.bottomRight.y; } - // +1 +1 -2 -2 is to avoid artifacts with AA. TODO: figure out a better method. Alpha blending? This same shit will happen on hyprbars. - m_bLastWindowBox = {(int)(m_vLastWindowPos.x - maxExtents.topLeft.x - BORDER + 1), (int)(m_vLastWindowPos.y - maxExtents.topLeft.y - BORDER + 1), - (int)(m_vLastWindowSize.x + maxExtents.topLeft.x + maxExtents.bottomRight.x + 2 * BORDER - 2), - (int)(m_vLastWindowSize.y + maxExtents.topLeft.y + maxExtents.bottomRight.y + 2 * BORDER - 2)}; + m_bLastWindowBox = {(int)(m_vLastWindowPos.x - maxExtents.topLeft.x - BORDER), (int)(m_vLastWindowPos.y - maxExtents.topLeft.y - BORDER), + (int)(m_vLastWindowSize.x + maxExtents.topLeft.x + maxExtents.bottomRight.x + 2 * BORDER), + (int)(m_vLastWindowSize.y + maxExtents.topLeft.y + maxExtents.bottomRight.y + 2 * BORDER)}; } } @@ -145,41 +144,29 @@ void CHyprDropShadowDecoration::draw(CMonitor* pMonitor, float a, const Vector2D g_pHyprOpenGL->scissor((wlr_box*)nullptr); + // we'll take the liberty of using this as it should not be used rn + CFramebuffer& alphaFB = g_pHyprOpenGL->m_RenderData.pCurrentMonData->mirrorFB; + auto* LASTFB = g_pHyprOpenGL->m_RenderData.currentFB; + if (*PSHADOWIGNOREWINDOW) { - glEnable(GL_STENCIL_TEST); - - glClearStencil(0); - glClear(GL_STENCIL_BUFFER_BIT); - - glStencilFunc(GL_ALWAYS, 1, -1); - glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE); - wlr_box windowBox = {m_bLastWindowBox.x - pMonitor->vecPosition.x, m_bLastWindowBox.y - pMonitor->vecPosition.y, m_bLastWindowBox.width, m_bLastWindowBox.height}; scaleBox(&windowBox, pMonitor->scale); if (windowBox.width < 1 || windowBox.height < 1) { - glClearStencil(0); - glClear(GL_STENCIL_BUFFER_BIT); - glDisable(GL_STENCIL_TEST); return; // prevent assert failed } - g_pHyprOpenGL->renderRect(&windowBox, CColor(0, 0, 0, 0), ROUNDING * pMonitor->scale); + alphaFB.bind(); + g_pHyprOpenGL->clear(CColor(0, 0, 0, 0)); - glStencilFunc(GL_NOTEQUAL, 1, -1); - glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE); + g_pHyprOpenGL->renderRect(&windowBox, CColor(1.0, 1.0, 1.0, 1.0), ROUNDING * pMonitor->scale); + + LASTFB->bind(); } scaleBox(&fullBox, pMonitor->scale); - g_pHyprOpenGL->renderRoundedShadow(&fullBox, ROUNDING * pMonitor->scale, *PSHADOWSIZE * pMonitor->scale, a); - - if (*PSHADOWIGNOREWINDOW) { - // cleanup - glClearStencil(0); - glClear(GL_STENCIL_BUFFER_BIT); - glDisable(GL_STENCIL_TEST); - } + g_pHyprOpenGL->renderRoundedShadow(&fullBox, ROUNDING * pMonitor->scale, *PSHADOWSIZE * pMonitor->scale, a, &alphaFB); } eDecorationLayer CHyprDropShadowDecoration::getDecorationLayer() { diff --git a/src/render/shaders/Shadow.hpp b/src/render/shaders/Shadow.hpp index 2c7b07b2..f9bdeb8d 100644 --- a/src/render/shaders/Shadow.hpp +++ b/src/render/shaders/Shadow.hpp @@ -5,7 +5,9 @@ inline const std::string FRAGSHADOW = R"#( precision mediump float; varying vec4 v_color; +uniform sampler2D alphaMatte; varying vec2 v_texcoord; +varying vec2 v_texcoordMatte; uniform vec2 topLeft; uniform vec2 bottomRight; @@ -13,6 +15,7 @@ uniform vec2 fullSize; uniform float radius; uniform float range; uniform float shadowPower; +uniform int useAlphaMatte; float pixAlphaRoundedDistance(float distanceToCorner) { if (distanceToCorner > radius) { @@ -74,6 +77,10 @@ void main() { } } + if (useAlphaMatte == 1) { + pixColor[3] *= 1.0 - texture2D(alphaMatte, v_texcoordMatte)[3]; + } + if (pixColor[3] == 0.0) { discard; return; } diff --git a/src/render/shaders/Textures.hpp b/src/render/shaders/Textures.hpp index 109840ca..6f19ac9f 100644 --- a/src/render/shaders/Textures.hpp +++ b/src/render/shaders/Textures.hpp @@ -37,13 +37,16 @@ uniform mat3 proj; uniform vec4 color; attribute vec2 pos; attribute vec2 texcoord; +attribute vec2 texcoordMatte; varying vec4 v_color; varying vec2 v_texcoord; +varying vec2 v_texcoordMatte; void main() { gl_Position = vec4(proj * vec3(pos, 1.0), 1.0); v_color = color; v_texcoord = texcoord; + v_texcoordMatte = texcoordMatte; })#"; inline const std::string QUADFRAGSRC = R"#(