wayland: Add window hint to render as shell layer (for example, desktop background)

This adds a wayland specific hint `GLFW_WAYLAND_SHELL_LAYER` to make it
possible to render to one of wayland's full screen layers using the
[wlr-layer-shell-unstable-v1](https://wayland.app/protocols/wlr-layer-shell-unstable-v1) wayland
interface. This can be used to, for example, render to the desktop
background, or as an overlay.

It can take any of the values from `zwlr_layer_shell_v1::layer`, or `-1`
(default) to create an application window.

deps/wayland-protocol/wlr-layer-shell-unstable-v1.xml

@@ -0,0 +1,285 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<protocol name="wlr_layer_shell_v1_unstable_v1">
+  <copyright>
+    Copyright © 2017 Drew DeVault
+
+    Permission to use, copy, modify, distribute, and sell this
+    software and its documentation for any purpose is hereby granted
+    without fee, provided that the above copyright notice appear in
+    all copies and that both that copyright notice and this permission
+    notice appear in supporting documentation, and that the name of
+    the copyright holders not be used in advertising or publicity
+    pertaining to distribution of the software without specific,
+    written prior permission.  The copyright holders make no
+    representations about the suitability of this software for any
+    purpose.  It is provided "as is" without express or implied
+    warranty.
+
+    THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
+    SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+    FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
+    SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
+    AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION,
+    ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
+    THIS SOFTWARE.
+  </copyright>
+
+  <interface name="zwlr_layer_shell_v1" version="1">
+    <description summary="create surfaces that are layers of the desktop">
+      Clients can use this interface to assign the surface_layer role to
+      wl_surfaces. Such surfaces are assigned to a "layer" of the output and
+      rendered with a defined z-depth respective to each other. They may also be
+      anchored to the edges and corners of a screen and specify input handling
+      semantics. This interface should be suitable for the implementation of
+      many desktop shell components, and a broad number of other applications
+      that interact with the desktop.
+    </description>
+
+    <request name="get_layer_surface">
+      <description summary="create a layer_surface from a surface">
+        Create a layer surface for an existing surface. This assigns the role of
+        layer_surface, or raises a protocol error if another role is already
+        assigned.
+
+        Creating a layer surface from a wl_surface which has a buffer attached
+        or committed is a client error, and any attempts by a client to attach
+        or manipulate a buffer prior to the first layer_surface.configure call
+        must also be treated as errors.
+
+        You may pass NULL for output to allow the compositor to decide which
+        output to use. Generally this will be the one that the user most
+        recently interacted with.
+
+        Clients can specify a namespace that defines the purpose of the layer
+        surface.
+      </description>
+      <arg name="id" type="new_id" interface="zwlr_layer_surface_v1"/>
+      <arg name="surface" type="object" interface="wl_surface"/>
+      <arg name="output" type="object" interface="wl_output" allow-null="true"/>
+      <arg name="layer" type="uint" enum="layer" summary="layer to add this surface to"/>
+      <arg name="namespace" type="string" summary="namespace for the layer surface"/>
+    </request>
+
+    <enum name="error">
+      <entry name="role" value="0" summary="wl_surface has another role"/>
+      <entry name="invalid_layer" value="1" summary="layer value is invalid"/>
+      <entry name="already_constructed" value="2" summary="wl_surface has a buffer attached or committed"/>
+    </enum>
+
+    <enum name="layer">
+      <description summary="available layers for surfaces">
+        These values indicate which layers a surface can be rendered in. They
+        are ordered by z depth, bottom-most first. Traditional shell surfaces
+        will typically be rendered between the bottom and top layers.
+        Fullscreen shell surfaces are typically rendered at the top layer.
+        Multiple surfaces can share a single layer, and ordering within a
+        single layer is undefined.
+      </description>
+
+      <entry name="background" value="0"/>
+      <entry name="bottom" value="1"/>
+      <entry name="top" value="2"/>
+      <entry name="overlay" value="3"/>
+    </enum>
+  </interface>
+
+  <interface name="zwlr_layer_surface_v1" version="1">
+    <description summary="layer metadata interface">
+      An interface that may be implemented by a wl_surface, for surfaces that
+      are designed to be rendered as a layer of a stacked desktop-like
+      environment.
+
+      Layer surface state (size, anchor, exclusive zone, margin, interactivity)
+      is double-buffered, and will be applied at the time wl_surface.commit of
+      the corresponding wl_surface is called.
+    </description>
+
+    <request name="set_size">
+      <description summary="sets the size of the surface">
+        Sets the size of the surface in surface-local coordinates. The
+        compositor will display the surface centered with respect to its
+        anchors.
+
+        If you pass 0 for either value, the compositor will assign it and
+        inform you of the assignment in the configure event. You must set your
+        anchor to opposite edges in the dimensions you omit; not doing so is a
+        protocol error. Both values are 0 by default.
+
+        Size is double-buffered, see wl_surface.commit.
+      </description>
+      <arg name="width" type="uint"/>
+      <arg name="height" type="uint"/>
+    </request>
+
+    <request name="set_anchor">
+      <description summary="configures the anchor point of the surface">
+        Requests that the compositor anchor the surface to the specified edges
+        and corners. If two orthoginal edges are specified (e.g. 'top' and
+        'left'), then the anchor point will be the intersection of the edges
+        (e.g. the top left corner of the output); otherwise the anchor point
+        will be centered on that edge, or in the center if none is specified.
+
+        Anchor is double-buffered, see wl_surface.commit.
+      </description>
+      <arg name="anchor" type="uint" enum="anchor"/>
+    </request>
+
+    <request name="set_exclusive_zone">
+      <description summary="configures the exclusive geometry of this surface">
+        Requests that the compositor avoids occluding an area of the surface
+        with other surfaces. The compositor's use of this information is
+        implementation-dependent - do not assume that this region will not
+        actually be occluded.
+
+        A positive value is only meaningful if the surface is anchored to an
+        edge, rather than a corner. The zone is the number of surface-local
+        coordinates from the edge that are considered exclusive.
+
+        Surfaces that do not wish to have an exclusive zone may instead specify
+        how they should interact with surfaces that do. If set to zero, the
+        surface indicates that it would like to be moved to avoid occluding
+        surfaces with a positive excluzive zone. If set to -1, the surface
+        indicates that it would not like to be moved to accommodate for other
+        surfaces, and the compositor should extend it all the way to the edges
+        it is anchored to.
+
+        For example, a panel might set its exclusive zone to 10, so that
+        maximized shell surfaces are not shown on top of it. A notification
+        might set its exclusive zone to 0, so that it is moved to avoid
+        occluding the panel, but shell surfaces are shown underneath it. A
+        wallpaper or lock screen might set their exclusive zone to -1, so that
+        they stretch below or over the panel.
+
+        The default value is 0.
+
+        Exclusive zone is double-buffered, see wl_surface.commit.
+      </description>
+      <arg name="zone" type="int"/>
+    </request>
+
+    <request name="set_margin">
+      <description summary="sets a margin from the anchor point">
+        Requests that the surface be placed some distance away from the anchor
+        point on the output, in surface-local coordinates. Setting this value
+        for edges you are not anchored to has no effect.
+
+        The exclusive zone includes the margin.
+
+        Margin is double-buffered, see wl_surface.commit.
+      </description>
+      <arg name="top" type="int"/>
+      <arg name="right" type="int"/>
+      <arg name="bottom" type="int"/>
+      <arg name="left" type="int"/>
+    </request>
+
+    <request name="set_keyboard_interactivity">
+      <description summary="requests keyboard events">
+        Set to 1 to request that the seat send keyboard events to this layer
+        surface. For layers below the shell surface layer, the seat will use
+        normal focus semantics. For layers above the shell surface layers, the
+        seat will always give exclusive keyboard focus to the top-most layer
+        which has keyboard interactivity set to true.
+
+        Layer surfaces receive pointer, touch, and tablet events normally. If
+        you do not want to receive them, set the input region on your surface
+        to an empty region.
+
+        Events is double-buffered, see wl_surface.commit.
+      </description>
+      <arg name="keyboard_interactivity" type="uint"/>
+    </request>
+
+    <request name="get_popup">
+      <description summary="assign this layer_surface as an xdg_popup parent">
+        This assigns an xdg_popup's parent to this layer_surface.  This popup
+        should have been created via xdg_surface::get_popup with the parent set
+        to NULL, and this request must be invoked before committing the popup's
+        initial state.
+
+        See the documentation of xdg_popup for more details about what an
+        xdg_popup is and how it is used.
+      </description>
+      <arg name="popup" type="object" interface="xdg_popup"/>
+    </request>
+
+    <request name="ack_configure">
+      <description summary="ack a configure event">
+        When a configure event is received, if a client commits the
+        surface in response to the configure event, then the client
+        must make an ack_configure request sometime before the commit
+        request, passing along the serial of the configure event.
+
+        If the client receives multiple configure events before it
+        can respond to one, it only has to ack the last configure event.
+
+        A client is not required to commit immediately after sending
+        an ack_configure request - it may even ack_configure several times
+        before its next surface commit.
+
+        A client may send multiple ack_configure requests before committing, but
+        only the last request sent before a commit indicates which configure
+        event the client really is responding to.
+      </description>
+      <arg name="serial" type="uint" summary="the serial from the configure event"/>
+    </request>
+
+    <request name="destroy" type="destructor">
+      <description summary="destroy the layer_surface">
+        This request destroys the layer surface.
+      </description>
+    </request>
+
+    <event name="configure">
+      <description summary="suggest a surface change">
+        The configure event asks the client to resize its surface.
+
+        Clients should arrange their surface for the new states, and then send
+        an ack_configure request with the serial sent in this configure event at
+        some point before committing the new surface.
+
+        The client is free to dismiss all but the last configure event it
+        received.
+
+        The width and height arguments specify the size of the window in
+        surface-local coordinates.
+
+        The size is a hint, in the sense that the client is free to ignore it if
+        it doesn't resize, pick a smaller size (to satisfy aspect ratio or
+        resize in steps of NxM pixels). If the client picks a smaller size and
+        is anchored to two opposite anchors (e.g. 'top' and 'bottom'), the
+        surface will be centered on this axis.
+
+        If the width or height arguments are zero, it means the client should
+        decide its own window dimension.
+      </description>
+      <arg name="serial" type="uint"/>
+      <arg name="width" type="uint"/>
+      <arg name="height" type="uint"/>
+    </event>
+
+    <event name="closed">
+      <description summary="surface should be closed">
+        The closed event is sent by the compositor when the surface will no
+        longer be shown. The output may have been destroyed or the user may
+        have asked for it to be removed. Further changes to the surface will be
+        ignored. The client should destroy the resource after receiving this
+        event, and create a new surface if they so choose.
+      </description>
+    </event>
+
+    <enum name="error">
+      <entry name="invalid_surface_state" value="0" summary="provided surface state is invalid"/>
+      <entry name="invalid_size" value="1" summary="size is invalid"/>
+      <entry name="invalid_anchor" value="2" summary="anchor bitfield is invalid"/>
+    </enum>
+
+    <enum name="anchor" bitfield="true">
+      <entry name="top" value="1" summary="the top edge of the anchor rectangle"/>
+      <entry name="bottom" value="2" summary="the bottom edge of the anchor rectangle"/>
+      <entry name="left" value="4" summary="the left edge of the anchor rectangle"/>
+      <entry name="right" value="8" summary="the right edge of the anchor rectangle"/>
+    </enum>
+  </interface>
+</protocol>

include/GLFW/glfw3.h

@@ -1106,6 +1106,10 @@ extern "C" {
  */
 #define GLFW_X11_INSTANCE_NAME      0x00024002
 #define GLFW_WIN32_KEYBOARD_MENU    0x00025001
+/*! @brief Wayland specific
+ *  [window hint](@ref GLFW_WAYLAND_SHELL_LAYER_hint).
+ */
+#define GLFW_WAYLAND_SHELL_LAYER    0x00026001
 /*! @} */
 
 #define GLFW_NO_API                          0

src/CMakeLists.txt

@@ -113,6 +113,9 @@ if (GLFW_BUILD_WAYLAND)
     wayland_generate(
         "${WAYLAND_PROTOCOLS_BASE}/unstable/idle-inhibit/idle-inhibit-unstable-v1.xml"
         "${GLFW_BINARY_DIR}/src/wayland-idle-inhibit-unstable-v1-client-protocol")
+    wayland_generate(
+        "${GLFW_SOURCE_DIR}/deps/wayland-protocol/wlr-layer-shell-unstable-v1.xml"
+        "${GLFW_BINARY_DIR}/src/wayland-wlr-layer-shell-unstable-v1-client-protocol")
 endif()
 
 if (WIN32 AND GLFW_BUILD_SHARED_LIBRARY)

src/internal.h

@@ -421,6 +421,9 @@ struct _GLFWwndconfig
     struct {
         GLFWbool  keymenu;
     } win32;
+    struct {
+        int       shellLayer;
+    } wl;
 };
 
 // Context configuration

src/window.c

@@ -289,6 +289,9 @@ void glfwDefaultWindowHints(void)
 
     // The default is to use full Retina resolution framebuffers
     _glfw.hints.window.ns.retina = GLFW_TRUE;
+
+    // Default for wayland is to not create a shell layer but a application window
+    _glfw.hints.window.wl.shellLayer = -1;
 }
 
 GLFWAPI void glfwWindowHint(int hint, int value)
@@ -420,6 +423,9 @@ GLFWAPI void glfwWindowHint(int hint, int value)
         case GLFW_REFRESH_RATE:
             _glfw.hints.refreshRate = value;
             return;
+        case GLFW_WAYLAND_SHELL_LAYER:
+            _glfw.hints.window.wl.shellLayer = value;
+            return;
     }
 
     _glfwInputError(GLFW_INVALID_ENUM, "Invalid window hint 0x%08X", hint);

src/wl_init.c

@@ -47,6 +47,7 @@
 #include "wayland-relative-pointer-unstable-v1-client-protocol.h"
 #include "wayland-pointer-constraints-unstable-v1-client-protocol.h"
 #include "wayland-idle-inhibit-unstable-v1-client-protocol.h"
+#include "wayland-wlr-layer-shell-unstable-v1-client-protocol.h"
 
 #include "wayland-client-protocol-code.h"
 #include "wayland-xdg-shell-client-protocol-code.h"
@@ -55,6 +56,7 @@
 #include "wayland-relative-pointer-unstable-v1-client-protocol-code.h"
 #include "wayland-pointer-constraints-unstable-v1-client-protocol-code.h"
 #include "wayland-idle-inhibit-unstable-v1-client-protocol-code.h"
+#include "wayland-wlr-layer-shell-unstable-v1-client-protocol-code.h"
 
 
 static _GLFWwindow* findWindowFromDecorationSurface(struct wl_surface* surface,
@@ -861,6 +863,13 @@ static void registryHandleGlobal(void* data,
                              &zwp_idle_inhibit_manager_v1_interface,
                              1);
     }
+    else if (strcmp(interface, "zwlr_layer_shell_v1") == 0)
+    {
+        _glfw.wl.layerShell =
+            wl_registry_bind(registry, name,
+                             &zwlr_layer_shell_v1_interface,
+                             1);
+    }
 }
 
 static void registryHandleGlobalRemove(void *data,
@@ -1443,6 +1452,8 @@ void _glfwTerminateWayland(void)
         zwp_pointer_constraints_v1_destroy(_glfw.wl.pointerConstraints);
     if (_glfw.wl.idleInhibitManager)
         zwp_idle_inhibit_manager_v1_destroy(_glfw.wl.idleInhibitManager);
+    if (_glfw.wl.layerShell)
+        zwlr_layer_shell_v1_destroy(_glfw.wl.layerShell);
     if (_glfw.wl.registry)
         wl_registry_destroy(_glfw.wl.registry);
     if (_glfw.wl.display)

src/wl_platform.h

@@ -122,6 +122,7 @@ struct wl_shm;
 #define xdg_surface_interface _glfw_xdg_surface_interface
 #define xdg_toplevel_interface _glfw_xdg_toplevel_interface
 #define xdg_wm_base_interface _glfw_xdg_wm_base_interface
+#define zwlr_layer_surface_v1_interface _glfw_zwlr_layer_surface_v1_interface
 
 #define GLFW_WAYLAND_WINDOW_STATE         _GLFWwindowWayland  wl;
 #define GLFW_WAYLAND_LIBRARY_WINDOW_STATE _GLFWlibraryWayland wl;
@@ -229,6 +230,7 @@ typedef struct _GLFWwindowWayland
     GLFWbool                    hovered;
     GLFWbool                    transparent;
     struct wl_surface*          surface;
+    struct zwlr_layer_surface_v1 *layer_surface;
     struct wl_egl_window*       native;
     struct wl_callback*         callback;
 
@@ -289,6 +291,7 @@ typedef struct _GLFWlibraryWayland
     struct zwp_relative_pointer_manager_v1* relativePointerManager;
     struct zwp_pointer_constraints_v1*      pointerConstraints;
     struct zwp_idle_inhibit_manager_v1*     idleInhibitManager;
+    struct zwlr_layer_shell_v1* layerShell;
 
     int                         compositorVersion;
     int                         seatVersion;

src/wl_window.c

@@ -47,6 +47,7 @@
 #include "wayland-relative-pointer-unstable-v1-client-protocol.h"
 #include "wayland-pointer-constraints-unstable-v1-client-protocol.h"
 #include "wayland-idle-inhibit-unstable-v1-client-protocol.h"
+#include "wayland-wlr-layer-shell-unstable-v1-client-protocol.h"
 
 
 static int createTmpfileCloexec(char* tmpname)
@@ -610,6 +611,31 @@ static GLFWbool createXdgSurface(_GLFWwindow* window)
     return GLFW_TRUE;
 }
 
+static void zwlrLayerSurfaceConfigure(void* data,
+            struct zwlr_layer_surface_v1* surface,
+            uint32_t serial, uint32_t width, uint32_t height) {
+    // Acknowledge configure request.
+    zwlr_layer_surface_v1_ack_configure(surface, serial);
+
+    // Handle layer surface resize as window resize.
+    _GLFWwindow* window = data;
+    _glfwInputWindowSize(window, width, height);
+    _glfwSetWindowSizeWayland(window, width, height);
+    _glfwInputWindowDamage(window);
+}
+
+static void zwlrLayerSurfaceClosed(void* data,
+            struct zwlr_layer_surface_v1* surface) {
+    // Handle layer surface close as window close.
+    _GLFWwindow* window = data;
+    _glfwInputWindowCloseRequest(window);
+}
+
+static const struct zwlr_layer_surface_v1_listener layer_surface_listener = {
+    zwlrLayerSurfaceConfigure,
+    zwlrLayerSurfaceClosed,
+};
+
 static GLFWbool createSurface(_GLFWwindow* window,
                               const _GLFWwndconfig* wndconfig,
                               const _GLFWfbconfig* fbconfig)
@@ -639,7 +665,48 @@ static GLFWbool createSurface(_GLFWwindow* window,
     if (!window->wl.transparent)
         setOpaqueRegion(window);
 
-    if (window->monitor || wndconfig->visible)
+    if (wndconfig->wl.shellLayer != -1)
+    {
+        if (!_glfw.wl.layerShell)
+        {
+            _glfwInputError(GLFW_FEATURE_UNAVAILABLE,
+                            "Wayland: Layer shell extension not available");
+            return GLFW_FALSE;
+        }
+
+        // Empty input region (background shouldn't capture events).
+        struct wl_region *input_region =
+                wl_compositor_create_region(_glfw.wl.compositor);
+        wl_surface_set_input_region(window->wl.surface, input_region);
+        wl_region_destroy(input_region);
+
+        // Setup layer surface for background.
+        window->wl.layer_surface = zwlr_layer_shell_v1_get_layer_surface(
+                        _glfw.wl.layerShell, window->wl.surface, window->monitor ? window->monitor->wl.output : NULL,
+                        wndconfig->wl.shellLayer, "wallpaper");
+        if (!window->wl.layer_surface)
+        {
+            _glfwInputError(GLFW_PLATFORM_ERROR,
+                            "Wayland: Could not create layer surface");
+            return GLFW_FALSE;
+        }
+
+        zwlr_layer_surface_v1_set_size(window->wl.layer_surface, 0, 0);
+        zwlr_layer_surface_v1_set_anchor(window->wl.layer_surface,
+                        ZWLR_LAYER_SURFACE_V1_ANCHOR_TOP |
+                        ZWLR_LAYER_SURFACE_V1_ANCHOR_RIGHT |
+                        ZWLR_LAYER_SURFACE_V1_ANCHOR_BOTTOM |
+                        ZWLR_LAYER_SURFACE_V1_ANCHOR_LEFT);
+        zwlr_layer_surface_v1_set_exclusive_zone(window->wl.layer_surface, -1);
+        zwlr_layer_surface_v1_add_listener(window->wl.layer_surface,
+                        &layer_surface_listener, window);
+        wl_surface_commit(window->wl.surface);
+        wl_display_roundtrip(_glfw.wl.display);
+
+        // For layers, make the assumption that they're always visible.
+        window->wl.visible = GLFW_TRUE;
+    }
+    else if (window->monitor || wndconfig->visible)
     {
         if (!createXdgSurface(window))
             return GLFW_FALSE;
@@ -880,6 +947,9 @@ void _glfwDestroyWindowWayland(_GLFWwindow* window)
     if (window->wl.xdg.surface)
         xdg_surface_destroy(window->wl.xdg.surface);
 
+    if (window->wl.layer_surface)
+        zwlr_layer_surface_v1_destroy(window->wl.layer_surface);
+
     if (window->wl.surface)
         wl_surface_destroy(window->wl.surface);
 
@@ -1030,7 +1100,7 @@ void _glfwShowWindowWayland(_GLFWwindow* window)
     {
         // NOTE: The XDG surface and role are created here so command-line applications
         //       with off-screen windows do not appear in for example the Unity dock
-        if (!window->wl.xdg.toplevel)
+        if (!window->wl.layer_surface && !window->wl.xdg.toplevel)
             createXdgSurface(window);
 
         window->wl.visible = GLFW_TRUE;