Add X11/EGL transparent window support

Seems like Mesa EGL fbconfigs don't match with transparent picture
formats.

src/egl_context.c

@@ -86,13 +86,21 @@ static int getConfigAttrib(EGLConfig config, int attrib)
 //
 static GLFWbool chooseFBConfigs(const _GLFWctxconfig* ctxconfig,
                                 const _GLFWfbconfig* desired,
-                                EGLConfig* result)
+                                EGLConfig* result,
+				GLFWbool findTransparent)
 {
     EGLConfig* nativeConfigs;
     _GLFWfbconfig* usableConfigs;
     const _GLFWfbconfig* closest;
     int i, nativeCount, usableCount;
 
+#if defined(_GLFW_X11)
+    XVisualInfo visualTemplate = {0};
+    if ( !(_glfw.xrender.major || _glfw.xrender.minor) ) {
+        findTransparent = GLFW_FALSE;
+    }
+#endif // _GLFW_X11
+
     eglGetConfigs(_glfw.egl.display, NULL, 0, &nativeCount);
     if (!nativeCount)
     {
@@ -106,6 +114,7 @@ static GLFWbool chooseFBConfigs(const _GLFWctxconfig* ctxconfig,
     usableConfigs = calloc(nativeCount, sizeof(_GLFWfbconfig));
     usableCount = 0;
 
+selectionloop:
     for (i = 0;  i < nativeCount;  i++)
     {
         const EGLConfig n = nativeConfigs[i];
@@ -121,8 +130,31 @@ static GLFWbool chooseFBConfigs(const _GLFWctxconfig* ctxconfig,
 
 #if defined(_GLFW_X11)
         // Only consider EGLConfigs with associated Visuals
-        if (!getConfigAttrib(n, EGL_NATIVE_VISUAL_ID))
+	visualTemplate.visualid = getConfigAttrib(n, EGL_NATIVE_VISUAL_ID);
+        if (!visualTemplate.visualid)
             continue;
+
+	if( findTransparent ) {
+	    int n_vi;
+            XVisualInfo *visualinfo;
+            XRenderPictFormat *pictFormat;
+	    
+	    visualinfo = XGetVisualInfo(_glfw.x11.display, VisualIDMask, &visualTemplate, &n_vi);
+	    if (!visualinfo)
+	        continue;
+
+            pictFormat = XRenderFindVisualFormat(_glfw.x11.display, visualinfo->visual);
+            if( !pictFormat ) {
+	        XFree( visualinfo );
+	        continue;
+	    }
+
+            if( !pictFormat->direct.alphaMask ) {
+	        XFree( visualinfo );
+	        continue;
+	    }
+	    XFree( visualinfo );
+        }
 #endif // _GLFW_X11
 
         if (ctxconfig->api == GLFW_OPENGL_ES_API)
@@ -158,6 +190,12 @@ static GLFWbool chooseFBConfigs(const _GLFWctxconfig* ctxconfig,
         u->egl = n;
         usableCount++;
     }
+    // reiterate the selection loop without looking for transparency supporting
+    // formats if no matchig FB configs for a transparent window were found. 
+    if( findTransparent && !usableCount ) {
+        findTransparent = GLFW_FALSE;
+	goto selectionloop;
+    }
 
     closest = _glfwChooseFBConfig(desired, usableConfigs, usableCount);
     if (closest)
@@ -297,7 +335,7 @@ GLFWbool _glfwCreateContextEGL(_GLFWwindow* window,
     if (ctxconfig->share)
         share = ctxconfig->share->context.egl.handle;
 
-    if (!chooseFBConfigs(ctxconfig, fbconfig, &config))
+    if (!chooseFBConfigs(ctxconfig, fbconfig, &config, window->transparent))
     {
         _glfwInputError(GLFW_FORMAT_UNAVAILABLE,
                         "EGL: Failed to find a suitable EGLConfig");
@@ -535,7 +573,8 @@ void _glfwDestroyContextEGL(_GLFWwindow* window)
 // Returns the Visual and depth of the chosen EGLConfig
 //
 #if defined(_GLFW_X11)
-GLFWbool _glfwChooseVisualEGL(const _GLFWctxconfig* ctxconfig,
+GLFWbool _glfwChooseVisualEGL(const _GLFWwndconfig* wndconfig,
+                              const _GLFWctxconfig* ctxconfig,
                               const _GLFWfbconfig* fbconfig,
                               Visual** visual, int* depth)
 {
@@ -545,7 +584,7 @@ GLFWbool _glfwChooseVisualEGL(const _GLFWctxconfig* ctxconfig,
     EGLint visualID = 0, count = 0;
     const long vimask = VisualScreenMask | VisualIDMask;
 
-    if (!chooseFBConfigs(ctxconfig, fbconfig, &native))
+    if (!chooseFBConfigs(ctxconfig, fbconfig, &native, wndconfig->transparent))
     {
         _glfwInputError(GLFW_FORMAT_UNAVAILABLE,
                         "EGL: Failed to find a suitable EGLConfig");

src/egl_context.h

@@ -207,7 +207,8 @@ GLFWbool _glfwCreateContextEGL(_GLFWwindow* window,
                                const _GLFWfbconfig* fbconfig);
 void _glfwDestroyContextEGL(_GLFWwindow* window);
 #if defined(_GLFW_X11)
-GLFWbool _glfwChooseVisualEGL(const _GLFWctxconfig* ctxconfig,
+GLFWbool _glfwChooseVisualEGL(const _GLFWwndconfig* wndconfig,
+                              const _GLFWctxconfig* ctxconfig,
                               const _GLFWfbconfig* fbconfig,
                               Visual** visual, int* depth);
 #endif /*_GLFW_X11*/

src/glx_context.c

@@ -80,8 +80,6 @@ static GLFWbool chooseFBConfig(
     usableConfigs = calloc(nativeCount, sizeof(_GLFWfbconfig));
     usableCount = 0;
     
-    XVisualInfo *visualinfo = NULL;
-    XRenderPictFormat *pictFormat = NULL;
 selectionloop:
     for (i = 0;  i < nativeCount;  i++)
     {
@@ -100,27 +98,24 @@ selectionloop:
         }
 
 	if( findTransparent ) {
-	    
+            XVisualInfo *visualinfo;
-	    if( visualinfo ) {
+            XRenderPictFormat *pictFormat;
-	        XFree( visualinfo );
-		visualinfo = NULL;
-	    }
 
 	    visualinfo = glXGetVisualFromFBConfig(_glfw.x11.display, n);
 	    if (!visualinfo)
 	        continue;
 
-	    if( pictFormat ) {
+            pictFormat = XRenderFindVisualFormat(_glfw.x11.display, visualinfo->visual);
-	        XFree( pictFormat );
+            if( !pictFormat ) {
-		pictFormat = NULL;
+	        XFree( visualinfo );
+	        continue;
 	    }
 
-            pictFormat = XRenderFindVisualFormat(_glfw.x11.display, visualinfo->visual);
+            if( !pictFormat->direct.alphaMask ) {
-            if( !pictFormat )
+	        XFree( visualinfo );
-	        continue;
-
-            if( !pictFormat->direct.alphaMask ) 
 	        continue;
+	    }
+	    XFree( visualinfo );
         }
 
         u->redBits = getFBConfigAttrib(n, GLX_RED_SIZE);
@@ -151,15 +146,6 @@ selectionloop:
 
         u->glx = n;
         usableCount++;
-	    
-        if( visualinfo ) {
-            XFree( visualinfo );
-	    visualinfo = NULL;
-        }
-	if( pictFormat ) {
-	    XFree( pictFormat );
-	    pictFormat = NULL;
-	}
     }
     // reiterate the selection loop without looking for transparency supporting
     // formats if no matchig FB configs for a transparent window were found. 
@@ -212,17 +198,6 @@ GLFWbool _glfwInitGLX(void)
         NULL
     };
 
-    const char* sonames_xrender[] =
-    {
-#if defined(__CYGWIN__)
-        "libXrender-1.so",
-#else
-        "libXrender.so.1",
-        "libXrender.so",
-#endif
-        NULL
-    };
-
 
     for (i = 0;  sonames_glx[i];  i++)
     {
@@ -231,13 +206,6 @@ GLFWbool _glfwInitGLX(void)
             break;
     }
 
-    for (i = 0;  sonames_xrender[i];  i++)
-    {
-        _glfw.xrender.handle = dlopen(sonames_xrender[i], RTLD_LAZY | RTLD_GLOBAL);
-        if (_glfw.xrender.handle)
-            break;
-    }
-
     if (!_glfw.glx.handle)
     {
         _glfwInputError(GLFW_API_UNAVAILABLE, "GLX: Failed to load GLX");
@@ -297,37 +265,6 @@ GLFWbool _glfwInitGLX(void)
         return GLFW_FALSE;
     }
 
-    // Xrender support is optional and not a requirement for GLX
-    // to work. Xrender is required for selecting a FB config that
-    // supports a picture format with an alpha mask, which in turn
-    // is required for transparent windows. I Xrender is not supported
-    // the GLFW_TRANSPARENT window hint is ignored.
-    _glfw.xrender.errorBase = 0;
-    _glfw.xrender.eventBase = 0;
-    _glfw.xrender.major = 0;
-    _glfw.xrender.minor = 0;
-    if (_glfw.xrender.handle) do {
-    	int errorBase, eventBase, major, minor;
-        _glfw.xrender.QueryExtension =
-            dlsym(_glfw.xrender.handle, "XRenderQueryExtension");
-        _glfw.xrender.QueryVersion =
-            dlsym(_glfw.xrender.handle, "XRenderQueryVersion");
-        _glfw.xrender.FindVisualFormat =
-            dlsym(_glfw.xrender.handle, "XRenderFindVisualFormat");
-
-        if ( !XRenderQueryExtension(_glfw.x11.display, &errorBase, &eventBase)) {
-	    break;
-	}
-        if ( !XRenderQueryVersion(_glfw.x11.display, &major, &minor)) {
-	    break;
-	}
-
-        _glfw.xrender.errorBase = errorBase;
-        _glfw.xrender.eventBase = eventBase;
-        _glfw.xrender.major = major;
-        _glfw.xrender.minor = minor;
-    } while(0);
-
     if (_glfwPlatformExtensionSupported("GLX_EXT_swap_control"))
     {
         _glfw.glx.SwapIntervalEXT = (PFNGLXSWAPINTERVALEXTPROC)

src/glx_context.h

@@ -109,19 +109,9 @@ typedef void (*PFNGLXDESTROYWINDOWPROC)(Display*,GLXWindow);
 #define glXCreateWindow _glfw.glx.CreateWindow
 #define glXDestroyWindow _glfw.glx.DestroyWindow
 
-// libXrender.so function pointer typedefs
-typedef Bool (*PFNXRENDERQUERYEXTENSIONPROC)(Display*,int*,int*);
-typedef Status (*PFNXRENDERQUERYVERSIONPROC)(Display*dpy,int*,int*);
-typedef XRenderPictFormat* (*PFNXRENDERFINDVISUALFORMATPROC)(Display*,Visual const *);
-
-// libXrender.so function identifier overlays
-#define XRenderQueryExtension   _glfw.xrender.QueryExtension
-#define XRenderQueryVersion     _glfw.xrender.QueryVersion
-#define XRenderFindVisualFormat _glfw.xrender.FindVisualFormat
-
 #define _GLFW_PLATFORM_FBCONFIG                 GLXFBConfig     glx
 #define _GLFW_PLATFORM_CONTEXT_STATE            _GLFWcontextGLX glx
-#define _GLFW_PLATFORM_LIBRARY_CONTEXT_STATE    _GLFWlibraryGLX glx ; _GLFWlibraryXrender xrender
+#define _GLFW_PLATFORM_LIBRARY_CONTEXT_STATE    _GLFWlibraryGLX glx
 
 
 // GLX-specific per-context data
@@ -181,24 +171,6 @@ typedef struct _GLFWlibraryGLX
 
 } _GLFWlibraryGLX;
 
-// Xrender-specific global data
-//
-typedef struct _GLFWlibraryXrender
-{
-    int             major, minor;
-    int             eventBase;
-    int             errorBase;
-
-    // dlopen handle for libGL.so.1
-    void*           handle;
-
-    // Xrender functions (subset required for transparent window)
-    PFNXRENDERQUERYEXTENSIONPROC    QueryExtension;
-    PFNXRENDERQUERYVERSIONPROC      QueryVersion;
-    PFNXRENDERFINDVISUALFORMATPROC  FindVisualFormat;
-} _GLFWlibraryXrender;
-
-
 GLFWbool _glfwInitGLX(void);
 void _glfwTerminateGLX(void);
 GLFWbool _glfwCreateContextGLX(_GLFWwindow* window,

src/x11_init.c

@@ -474,6 +474,18 @@ static void detectEWMH(void)
 //
 static GLFWbool initExtensions(void)
 {
+    int i;
+    const char* sonames_xrender[] =
+    {
+#if defined(__CYGWIN__)
+        "libXrender-1.so",
+#else
+        "libXrender.so.1",
+        "libXrender.so",
+#endif
+        NULL
+    };
+
     // Find or create window manager atoms
     _glfw.x11.WM_PROTOCOLS = XInternAtom(_glfw.x11.display,
                                          "WM_PROTOCOLS",
@@ -615,6 +627,43 @@ static GLFWbool initExtensions(void)
     _glfw.x11.XdndFinished = XInternAtom(_glfw.x11.display, "XdndFinished", True);
     _glfw.x11.XdndSelection = XInternAtom(_glfw.x11.display, "XdndSelection", True);
 
+    // Xrender support is optional and not a requirement for GLX/EGL
+    // to work. Xrender is required for selecting a FB config that
+    // supports a picture format with an alpha mask, which in turn
+    // is required for transparent windows. I Xrender is not supported
+    // the GLFW_TRANSPARENT window hint is ignored.
+    for (i = 0;  sonames_xrender[i];  i++)
+    {
+        _glfw.xrender.handle = dlopen(sonames_xrender[i], RTLD_LAZY | RTLD_GLOBAL);
+        if (_glfw.xrender.handle)
+            break;
+    }
+    _glfw.xrender.errorBase = 0;
+    _glfw.xrender.eventBase = 0;
+    _glfw.xrender.major = 0;
+    _glfw.xrender.minor = 0;
+    if (_glfw.xrender.handle) do {
+    	int errorBase, eventBase, major, minor;
+        _glfw.xrender.QueryExtension =
+            dlsym(_glfw.xrender.handle, "XRenderQueryExtension");
+        _glfw.xrender.QueryVersion =
+            dlsym(_glfw.xrender.handle, "XRenderQueryVersion");
+        _glfw.xrender.FindVisualFormat =
+            dlsym(_glfw.xrender.handle, "XRenderFindVisualFormat");
+
+        if ( !XRenderQueryExtension(_glfw.x11.display, &errorBase, &eventBase)) {
+	    break;
+	}
+        if ( !XRenderQueryVersion(_glfw.x11.display, &major, &minor)) {
+	    break;
+	}
+
+        _glfw.xrender.errorBase = errorBase;
+        _glfw.xrender.eventBase = eventBase;
+        _glfw.xrender.major = major;
+        _glfw.xrender.minor = minor;
+    } while(0);
+
     return GLFW_TRUE;
 }
 

src/x11_platform.h

@@ -103,10 +103,20 @@ typedef VkBool32 (APIENTRY *PFN_vkGetPhysicalDeviceXcbPresentationSupportKHR)(Vk
 #define _glfw_dlsym(handle, name) dlsym(handle, name)
 
 #define _GLFW_PLATFORM_WINDOW_STATE         _GLFWwindowX11  x11
-#define _GLFW_PLATFORM_LIBRARY_WINDOW_STATE _GLFWlibraryX11 x11
+#define _GLFW_PLATFORM_LIBRARY_WINDOW_STATE _GLFWlibraryX11 x11 ; _GLFWlibraryXrender xrender
 #define _GLFW_PLATFORM_MONITOR_STATE        _GLFWmonitorX11 x11
 #define _GLFW_PLATFORM_CURSOR_STATE         _GLFWcursorX11  x11
 
+// libXrender.so function pointer typedefs
+typedef Bool (*PFNXRENDERQUERYEXTENSIONPROC)(Display*,int*,int*);
+typedef Status (*PFNXRENDERQUERYVERSIONPROC)(Display*dpy,int*,int*);
+typedef XRenderPictFormat* (*PFNXRENDERFINDVISUALFORMATPROC)(Display*,Visual const *);
+
+// libXrender.so function identifier overlays
+#define XRenderQueryExtension   _glfw.xrender.QueryExtension
+#define XRenderQueryVersion     _glfw.xrender.QueryVersion
+#define XRenderFindVisualFormat _glfw.xrender.FindVisualFormat
+
 
 // X11-specific per-window data
 //
@@ -255,6 +265,21 @@ typedef struct _GLFWlibraryX11
 
 } _GLFWlibraryX11;
 
+// Xrender-specific global data
+typedef struct _GLFWlibraryXrender
+{
+    int             major, minor;
+    int             eventBase;
+    int             errorBase;
+
+    // dlopen handle for libGL.so.1
+    void*           handle;
+
+    // Xrender functions (subset required for transparent window)
+    PFNXRENDERQUERYEXTENSIONPROC    QueryExtension;
+    PFNXRENDERQUERYVERSIONPROC      QueryVersion;
+    PFNXRENDERFINDVISUALFORMATPROC  FindVisualFormat;
+} _GLFWlibraryXrender;
 
 // X11-specific per-monitor data
 //

src/x11_window.c

@@ -1434,7 +1434,7 @@ int _glfwPlatformCreateWindow(_GLFWwindow* window,
         if (!_glfwChooseVisualGLX(wndconfig, ctxconfig, fbconfig, &visual, &depth))
             return GLFW_FALSE;
 #elif defined(_GLFW_EGL)
-        if (!_glfwChooseVisualEGL(ctxconfig, fbconfig, &visual, &depth))
+        if (!_glfwChooseVisualEGL(wndconfig, ctxconfig, fbconfig, &visual, &depth))
             return GLFW_FALSE;
 #endif
     }