From 7da3e52c862a9f555e229a59a97e3a798be688bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Camilla=20L=C3=B6wy?= <elmindreda@elmindreda.org>
Date: Mon, 6 Jul 2020 23:17:41 +0200
Subject: [PATCH] Win32: Fix GLFW_MOUSE_PASSTHROUGH dropping events

Returning HTTRANSPARENT from WM_NCHITTEST does cause the window to be
transparent for some hit-testing APIs but does not make it pass mouse
input through to whatever window is below it.

For that to work on modern Windows, the window needs to be both layered
and extended-window-style-transparent.

Additional logic changes to ensure mouse input passthrough, framebuffer
transparency and window opacity are mindful of one another when
modifying WS_EX_LAYERED.

Related to #1568.
---
 src/win32_window.c | 64 +++++++++++++++++++++++++++++++++-------------
 1 file changed, 46 insertions(+), 18 deletions(-)

diff --git a/src/win32_window.c b/src/win32_window.c
index 0b5ccb5a..c7434f9a 100644
--- a/src/win32_window.c
+++ b/src/win32_window.c
@@ -417,10 +417,15 @@ static void updateFramebufferTransparency(const _GLFWwindow* window)
     else
     {
         LONG exStyle = GetWindowLongW(window->win32.handle, GWL_EXSTYLE);
-        exStyle &= ~WS_EX_LAYERED;
-        SetWindowLongW(window->win32.handle, GWL_EXSTYLE, exStyle);
-        RedrawWindow(window->win32.handle, NULL, NULL,
-                     RDW_ERASE | RDW_INVALIDATE | RDW_FRAME);
+        if (exStyle & WS_EX_TRANSPARENT)
+            SetLayeredWindowAttributes(window->win32.handle, 0, 0, 0);
+        else
+        {
+            exStyle &= ~WS_EX_LAYERED;
+            SetWindowLongW(window->win32.handle, GWL_EXSTYLE, exStyle);
+            RedrawWindow(window->win32.handle, NULL, NULL,
+                         RDW_ERASE | RDW_INVALIDATE | RDW_FRAME);
+        }
     }
 }
 
@@ -1201,13 +1206,6 @@ static LRESULT CALLBACK windowProc(HWND hWnd, UINT uMsg,
             DragFinish(drop);
             return 0;
         }
-
-        case WM_NCHITTEST:
-        {
-            if (window->mousePassthrough)
-                return HTTRANSPARENT;
-            break;
-        }
     }
 
     return DefWindowProcW(hWnd, uMsg, wParam, lParam);
@@ -1863,6 +1861,33 @@ void _glfwPlatformSetWindowFloating(_GLFWwindow* window, GLFWbool enabled)
 
 void _glfwPlatformSetWindowMousePassthrough(_GLFWwindow* window, GLFWbool enabled)
 {
+    COLORREF key = 0;
+    BYTE alpha = 0;
+    DWORD flags = 0;
+    DWORD exStyle = GetWindowLongW(window->win32.handle, GWL_EXSTYLE);
+
+    if (exStyle & WS_EX_LAYERED)
+        GetLayeredWindowAttributes(window->win32.handle, &key, &alpha, &flags);
+
+    if (enabled)
+        exStyle |= (WS_EX_TRANSPARENT | WS_EX_LAYERED);
+    else
+    {
+        exStyle &= ~WS_EX_TRANSPARENT;
+        // NOTE: Window opacity and framebuffer transparency also need to
+        //       control the layered style so avoid stepping on their feet
+        if (exStyle & WS_EX_LAYERED)
+        {
+            if (!(flags & (LWA_ALPHA | LWA_COLORKEY)))
+                exStyle &= ~WS_EX_LAYERED;
+        }
+    }
+
+    SetWindowLongW(window->win32.handle, GWL_EXSTYLE, exStyle);
+
+    if (enabled)
+        SetLayeredWindowAttributes(window->win32.handle, key, alpha, flags);
+
     window->mousePassthrough = enabled;
 }
 
@@ -1883,19 +1908,22 @@ float _glfwPlatformGetWindowOpacity(_GLFWwindow* window)
 
 void _glfwPlatformSetWindowOpacity(_GLFWwindow* window, float opacity)
 {
-    if (opacity < 1.f)
+    LONG exStyle = GetWindowLongW(window->win32.handle, GWL_EXSTYLE);
+    if (opacity < 1.f || (exStyle & WS_EX_TRANSPARENT))
     {
         const BYTE alpha = (BYTE) (255 * opacity);
-        DWORD style = GetWindowLongW(window->win32.handle, GWL_EXSTYLE);
-        style |= WS_EX_LAYERED;
-        SetWindowLongW(window->win32.handle, GWL_EXSTYLE, style);
+        exStyle |= WS_EX_LAYERED;
+        SetWindowLongW(window->win32.handle, GWL_EXSTYLE, exStyle);
         SetLayeredWindowAttributes(window->win32.handle, 0, alpha, LWA_ALPHA);
     }
+    else if (exStyle & WS_EX_TRANSPARENT)
+    {
+        SetLayeredWindowAttributes(window->win32.handle, 0, 0, 0);
+    }
     else
     {
-        DWORD style = GetWindowLongW(window->win32.handle, GWL_EXSTYLE);
-        style &= ~WS_EX_LAYERED;
-        SetWindowLongW(window->win32.handle, GWL_EXSTYLE, style);
+        exStyle &= ~WS_EX_LAYERED;
+        SetWindowLongW(window->win32.handle, GWL_EXSTYLE, exStyle);
     }
 }