added support for transparent X11/GLX window

examples/gears.c

@@ -172,6 +172,7 @@ static GLfloat angle = 0.f;
 /* OpenGL draw function & timing */
 static void draw(void)
 {
+  glClearColor(0., 0., 0., 0.);
   glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
 
   glPushMatrix();
@@ -311,6 +312,8 @@ int main(int argc, char *argv[])
     }
 
     glfwWindowHint(GLFW_DEPTH_BITS, 16);
+    glfwWindowHint(GLFW_ALPHA_BITS, 8);
+    glfwWindowHint(GLFW_TRANSPARENT, GLFW_TRUE);
 
     window = glfwCreateWindow( 300, 300, "Gears", NULL, NULL );
     if (!window)

include/GLFW/glfw3.h

@@ -627,6 +627,7 @@ extern "C" {
 #define GLFW_AUTO_ICONIFY           0x00020006
 #define GLFW_FLOATING               0x00020007
 #define GLFW_MAXIMIZED              0x00020008
+#define GLFW_TRANSPARENT            0x00020009
 
 #define GLFW_RED_BITS               0x00021001
 #define GLFW_GREEN_BITS             0x00021002

src/glx_context.c

@@ -47,7 +47,10 @@ static int getFBConfigAttrib(GLXFBConfig fbconfig, int attrib)
 
 // Return a list of available and usable framebuffer configs
 //
-static GLFWbool chooseFBConfig(const _GLFWfbconfig* desired, GLXFBConfig* result)
+static GLFWbool chooseFBConfig(
+	const _GLFWfbconfig* desired,
+	GLXFBConfig* result,
+	GLFWbool findTransparent)
 {
     GLXFBConfig* nativeConfigs;
     _GLFWfbconfig* usableConfigs;
@@ -56,6 +59,10 @@ static GLFWbool chooseFBConfig(const _GLFWfbconfig* desired, GLXFBConfig* result
     const char* vendor;
     GLFWbool trustWindowBit = GLFW_TRUE;
 
+    if (findTransparent && !(_glfw.xrender.major || _glfw.xrender.minor)) {
+        findTransparent = GLFW_FALSE;
+    }
+
     // HACK: This is a (hopefully temporary) workaround for Chromium
     //       (VirtualBox GL) not setting the window bit on any GLXFBConfigs
     vendor = glXGetClientString(_glfw.x11.display, GLX_VENDOR);
@@ -73,6 +80,7 @@ static GLFWbool chooseFBConfig(const _GLFWfbconfig* desired, GLXFBConfig* result
     usableConfigs = calloc(nativeCount, sizeof(_GLFWfbconfig));
     usableCount = 0;
 
+selectionloop:
     for (i = 0;  i < nativeCount;  i++)
     {
         const GLXFBConfig n = nativeConfigs[i];
@@ -89,6 +97,22 @@ static GLFWbool chooseFBConfig(const _GLFWfbconfig* desired, GLXFBConfig* result
                 continue;
         }
 
+	if( findTransparent ) {
+	    XVisualInfo *visualinfo;
+	    XRenderPictFormat *pictFormat;
+	    
+	    visualinfo = glXGetVisualFromFBConfig(_glfw.x11.display, n);
+	    if (!visualinfo)
+	        continue;
+
+            pictFormat = XRenderFindVisualFormat(_glfw.x11.display, visualinfo->visual);
+            if( !pictFormat )
+	        continue;
+
+            if( !pictFormat->direct.alphaMask ) 
+	        continue;
+        }
+
         u->redBits = getFBConfigAttrib(n, GLX_RED_SIZE);
         u->greenBits = getFBConfigAttrib(n, GLX_GREEN_SIZE);
         u->blueBits = getFBConfigAttrib(n, GLX_BLUE_SIZE);
@@ -118,6 +142,12 @@ static GLFWbool chooseFBConfig(const _GLFWfbconfig* desired, GLXFBConfig* result
         u->glx = n;
         usableCount++;
     }
+    // reiterate the selection loop without looking for transparency supporting
+    // formats if no matchig FB configs for a transparent window were found. 
+    if( findTransparent && !usableCount ) {
+        findTransparent = GLFW_FALSE;
+	goto selectionloop;
+    }
 
     closest = _glfwChooseFBConfig(desired, usableConfigs, usableCount);
     if (closest)
@@ -152,7 +182,7 @@ static GLXContext createLegacyContext(_GLFWwindow* window,
 GLFWbool _glfwInitGLX(void)
 {
     int i;
-    const char* sonames[] =
+    const char* sonames_glx[] =
     {
 #if defined(__CYGWIN__)
         "libGL-1.so",
@@ -163,14 +193,32 @@ GLFWbool _glfwInitGLX(void)
         NULL
     };
 
-
+    const char* sonames_xrender[] =
-    for (i = 0;  sonames[i];  i++)
     {
-        _glfw.glx.handle = dlopen(sonames[i], RTLD_LAZY | RTLD_GLOBAL);
+#if defined(__CYGWIN__)
+        "libXrender-1.so",
+#else
+        "libXrender.so.1",
+        "libXrender.so",
+#endif
+        NULL
+    };
+
+
+    for (i = 0;  sonames_glx[i];  i++)
+    {
+        _glfw.glx.handle = dlopen(sonames_glx[i], RTLD_LAZY | RTLD_GLOBAL);
         if (_glfw.glx.handle)
             break;
     }
 
+    for (i = 0;  sonames_xrender[i];  i++)
+    {
+        _glfw.xrender.handle = dlopen(sonames_xrender[i], RTLD_LAZY | RTLD_GLOBAL);
+        if (_glfw.xrender.handle)
+            break;
+    }
+
     if (!_glfw.glx.handle)
     {
         _glfwInputError(GLFW_API_UNAVAILABLE, "GLX: Failed to load GLX");
@@ -230,6 +278,37 @@ GLFWbool _glfwInitGLX(void)
         return GLFW_FALSE;
     }
 
+    // Xrender support is optional and not a requirement for GLX
+    // to work. Xrender is required for selecting a FB config that
+    // supports a picture format with an alpha mask, which in turn
+    // is required for transparent windows. I Xrender is not supported
+    // the GLFW_TRANSPARENT window hint is ignored.
+    _glfw.xrender.errorBase = 0;
+    _glfw.xrender.eventBase = 0;
+    _glfw.xrender.major = 0;
+    _glfw.xrender.minor = 0;
+    if (_glfw.xrender.handle) do {
+    	int errorBase, eventBase, major, minor;
+        _glfw.xrender.QueryExtension =
+            dlsym(_glfw.xrender.handle, "XRenderQueryExtension");
+        _glfw.xrender.QueryVersion =
+            dlsym(_glfw.xrender.handle, "XRenderQueryVersion");
+        _glfw.xrender.FindVisualFormat =
+            dlsym(_glfw.xrender.handle, "XRenderFindVisualFormat");
+
+        if ( !XRenderQueryExtension(_glfw.x11.display, &errorBase, &eventBase)) {
+	    break;
+	}
+        if ( !XRenderQueryVersion(_glfw.x11.display, &major, &minor)) {
+	    break;
+	}
+
+        _glfw.xrender.errorBase = errorBase;
+        _glfw.xrender.eventBase = eventBase;
+        _glfw.xrender.major = major;
+        _glfw.xrender.minor = minor;
+    } while(0);
+
     if (_glfwPlatformExtensionSupported("GLX_EXT_swap_control"))
     {
         _glfw.glx.SwapIntervalEXT = (PFNGLXSWAPINTERVALEXTPROC)
@@ -324,7 +403,7 @@ GLFWbool _glfwCreateContextGLX(_GLFWwindow* window,
     if (ctxconfig->share)
         share = ctxconfig->share->context.glx.handle;
 
-    if (!chooseFBConfig(fbconfig, &native))
+    if (!chooseFBConfig(fbconfig, &native, window->transparent))
     {
         _glfwInputError(GLFW_FORMAT_UNAVAILABLE,
                         "GLX: Failed to find a suitable GLXFBConfig");
@@ -507,14 +586,15 @@ void _glfwDestroyContextGLX(_GLFWwindow* window)
 
 // Returns the Visual and depth of the chosen GLXFBConfig
 //
-GLFWbool _glfwChooseVisualGLX(const _GLFWctxconfig* ctxconfig,
+GLFWbool _glfwChooseVisualGLX(const _GLFWwndconfig* wndconfig,
+			      const _GLFWctxconfig* ctxconfig,
                               const _GLFWfbconfig* fbconfig,
                               Visual** visual, int* depth)
 {
     GLXFBConfig native;
     XVisualInfo* result;
 
-    if (!chooseFBConfig(fbconfig, &native))
+    if (!chooseFBConfig(fbconfig, &native, wndconfig->transparent))
     {
         _glfwInputError(GLFW_FORMAT_UNAVAILABLE,
                         "GLX: Failed to find a suitable GLXFBConfig");

src/glx_context.h

@@ -74,6 +74,7 @@ typedef struct __GLXFBConfig* GLXFBConfig;
 typedef struct __GLXcontext* GLXContext;
 typedef void (*__GLXextproc)(void);
 
+// libGL.so function pointer typedefs
 typedef int (*PFNGLXGETFBCONFIGATTRIBPROC)(Display*,GLXFBConfig,int,int*);
 typedef const char* (*PFNGLXGETCLIENTSTRINGPROC)(Display*,int);
 typedef Bool (*PFNGLXQUERYEXTENSIONPROC)(Display*,int*,int*);
@@ -93,7 +94,7 @@ typedef XVisualInfo* (*PFNGLXGETVISUALFROMFBCONFIGPROC)(Display*,GLXFBConfig);
 typedef GLXWindow (*PFNGLXCREATEWINDOWPROC)(Display*,GLXFBConfig,Window,const int*);
 typedef void (*PFNGLXDESTROYWINDOWPROC)(Display*,GLXWindow);
 
-// libGL.so function pointer typedefs
+// libGL.so function identifier overlays 
 #define glXGetFBConfigs _glfw.glx.GetFBConfigs
 #define glXGetFBConfigAttrib _glfw.glx.GetFBConfigAttrib
 #define glXGetClientString _glfw.glx.GetClientString
@@ -108,9 +109,19 @@ typedef void (*PFNGLXDESTROYWINDOWPROC)(Display*,GLXWindow);
 #define glXCreateWindow _glfw.glx.CreateWindow
 #define glXDestroyWindow _glfw.glx.DestroyWindow
 
+// libXrender.so function pointer typedefs
+typedef Bool (*PFNXRENDERQUERYEXTENSIONPROC)(Display*,int*,int*);
+typedef Status (*PFNXRENDERQUERYVERSIONPROC)(Display*dpy,int*,int*);
+typedef XRenderPictFormat* (*PFNXRENDERFINDVISUALFORMATPROC)(Display*,Visual const *);
+
+// libXrender.so function identifier overlays
+#define XRenderQueryExtension   _glfw.xrender.QueryExtension
+#define XRenderQueryVersion     _glfw.xrender.QueryVersion
+#define XRenderFindVisualFormat _glfw.xrender.FindVisualFormat
+
 #define _GLFW_PLATFORM_FBCONFIG                 GLXFBConfig     glx
 #define _GLFW_PLATFORM_CONTEXT_STATE            _GLFWcontextGLX glx
-#define _GLFW_PLATFORM_LIBRARY_CONTEXT_STATE    _GLFWlibraryGLX glx
+#define _GLFW_PLATFORM_LIBRARY_CONTEXT_STATE    _GLFWlibraryGLX glx ; _GLFWlibraryXrender xrender
 
 
 // GLX-specific per-context data
@@ -170,6 +181,23 @@ typedef struct _GLFWlibraryGLX
 
 } _GLFWlibraryGLX;
 
+// Xrender-specific global data
+//
+typedef struct _GLFWlibraryXrender
+{
+    int             major, minor;
+    int             eventBase;
+    int             errorBase;
+
+    // dlopen handle for libGL.so.1
+    void*           handle;
+
+    // Xrender functions (subset required for transparent window)
+    PFNXRENDERQUERYEXTENSIONPROC    QueryExtension;
+    PFNXRENDERQUERYVERSIONPROC      QueryVersion;
+    PFNXRENDERFINDVISUALFORMATPROC  FindVisualFormat;
+} _GLFWlibraryXrender;
+
 
 GLFWbool _glfwInitGLX(void);
 void _glfwTerminateGLX(void);
@@ -177,7 +205,8 @@ GLFWbool _glfwCreateContextGLX(_GLFWwindow* window,
                                const _GLFWctxconfig* ctxconfig,
                                const _GLFWfbconfig* fbconfig);
 void _glfwDestroyContextGLX(_GLFWwindow* window);
-GLFWbool _glfwChooseVisualGLX(const _GLFWctxconfig* ctxconfig,
+GLFWbool _glfwChooseVisualGLX(const _GLFWwndconfig* wndconfig,
+                              const _GLFWctxconfig* ctxconfig,
                               const _GLFWfbconfig* fbconfig,
                               Visual** visual, int* depth);
 

src/internal.h

@@ -259,6 +259,7 @@ struct _GLFWwndconfig
     GLFWbool      resizable;
     GLFWbool      visible;
     GLFWbool      decorated;
+    GLFWbool      transparent;
     GLFWbool      focused;
     GLFWbool      autoIconify;
     GLFWbool      floating;
@@ -348,6 +349,7 @@ struct _GLFWwindow
     // Window settings and state
     GLFWbool            resizable;
     GLFWbool            decorated;
+    GLFWbool            transparent;
     GLFWbool            autoIconify;
     GLFWbool            floating;
     GLFWbool            closed;

src/window.c

@@ -178,6 +178,7 @@ GLFWAPI GLFWwindow* glfwCreateWindow(int width, int height,
     window->monitor     = wndconfig.monitor;
     window->resizable   = wndconfig.resizable;
     window->decorated   = wndconfig.decorated;
+    window->transparent = wndconfig.transparent;
     window->autoIconify = wndconfig.autoIconify;
     window->floating    = wndconfig.floating;
     window->cursorMode  = GLFW_CURSOR_NORMAL;
@@ -256,6 +257,7 @@ void glfwDefaultWindowHints(void)
     _glfw.hints.window.resizable   = GLFW_TRUE;
     _glfw.hints.window.visible     = GLFW_TRUE;
     _glfw.hints.window.decorated   = GLFW_TRUE;
+    _glfw.hints.window.transparent = GLFW_FALSE;
     _glfw.hints.window.focused     = GLFW_TRUE;
     _glfw.hints.window.autoIconify = GLFW_TRUE;
 
@@ -330,6 +332,9 @@ GLFWAPI void glfwWindowHint(int hint, int value)
         case GLFW_DECORATED:
             _glfw.hints.window.decorated = value ? GLFW_TRUE : GLFW_FALSE;
             break;
+        case GLFW_TRANSPARENT:
+            _glfw.hints.window.transparent = value ? GLFW_TRUE : GLFW_FALSE;
+            break;
         case GLFW_FOCUSED:
             _glfw.hints.window.focused = value ? GLFW_TRUE : GLFW_FALSE;
             break;
@@ -650,6 +655,8 @@ GLFWAPI int glfwGetWindowAttrib(GLFWwindow* handle, int attrib)
             return window->resizable;
         case GLFW_DECORATED:
             return window->decorated;
+        case GLFW_TRANSPARENT:
+            return window->transparent;
         case GLFW_FLOATING:
             return window->floating;
         case GLFW_CLIENT_API:

src/x11_window.c

@@ -1431,7 +1431,7 @@ int _glfwPlatformCreateWindow(_GLFWwindow* window,
     else
     {
 #if defined(_GLFW_GLX)
-        if (!_glfwChooseVisualGLX(ctxconfig, fbconfig, &visual, &depth))
+        if (!_glfwChooseVisualGLX(wndconfig, ctxconfig, fbconfig, &visual, &depth))
             return GLFW_FALSE;
 #elif defined(_GLFW_EGL)
         if (!_glfwChooseVisualEGL(ctxconfig, fbconfig, &visual, &depth))