bring modal or floating windows to front

README.md

@@ -191,8 +191,19 @@ layouts, monocle and tabbed alike: with one window filling the area there is
 nothing for a border to separate it from. Floating windows keep theirs.
 
 In every layout a newly mapped window goes to the front of the arrangement
-order — the first master slot — and takes focus, as in dwm. A window spawned
-onto a tag nobody is viewing leaves both the order and focus alone.
+order — the first master slot — and takes focus, as in dwm. A floating or
+fullscreen window is not part of the arrangement, so it stays where it is, but
+it takes focus like any other new window. A window spawned onto a tag nobody is
+viewing leaves both the order and focus alone.
+
+**Dialogs.** A window that names a parent — a dialog, a file picker — starts
+floating, which `float_children` turns off. Floating windows are stacked by
+focus, but a window and its parents and children are stacked as one family: the
+family rises together on the most recent focus any of its members has had, and
+within it a child is always drawn directly above its parent. Without that, a
+click on the parent of a *modal* dialog would bury the dialog behind the one
+window that will not answer, with no way to raise it again. A dialog opened by
+a fullscreen window is drawn above the fullscreen layer for the same reason.
 
 **tabbed** — same geometry as monocle, minus a strip at the top where att_wm
 draws one tab per window, the focused one highlighted. Each tab is clickable and

src/Window.zig

@@ -75,6 +75,11 @@ pending_close: bool = false,
 /// Bumped whenever the window takes focus, giving a cheap "most recently
 /// focused" ordering without maintaining dwm's second linked list.
 focus_serial: u64 = 0,
+/// Where this window's family of parents and children sits in the floating
+/// stack, and how far down that family the window is. Recomputed each render
+/// sequence; see `Wm.computeStacking`.
+stack_serial: u64 = 0,
+stack_depth: u32 = 0,
 /// Cleared once the window has had its one chance at taking focus as it maps.
 wants_initial_focus: bool = true,
 
@@ -105,6 +110,38 @@ pub fn getNode(self: *Window) ?*river.NodeV1 {
     return self.node;
 }
 
+/// The window at the top of this window's parent chain, and how many parents
+/// away it is. A window with no parent is its own root, at depth zero.
+///
+/// The protocol promises the parent links form a tree, but a client that
+/// breaks that promise must not hang the compositor, so the walk is bounded.
+pub fn ancestry(self: *Window) struct { root: *Window, depth: u32 } {
+    var root = self;
+    var depth: u32 = 0;
+    while (root.parent) |p| {
+        if (depth >= max_parent_depth) break;
+        root = p;
+        depth += 1;
+    }
+    return .{ .root = root, .depth = depth };
+}
+
+/// Whether this window belongs to a window that is currently fullscreen. Such
+/// a dialog has to be drawn above the fullscreen layer: its parent covers the
+/// whole output, and everything below it with it.
+pub fn hasFullscreenAncestor(self: *Window) bool {
+    var next = self.parent;
+    var depth: u32 = 0;
+    while (next) |p| : (depth += 1) {
+        if (depth >= max_parent_depth) break;
+        if (p.fullscreen and p.visible and !p.closed) return true;
+        next = p.parent;
+    }
+    return false;
+}
+
+const max_parent_depth = 32;
+
 /// Clamp a proposed size to the window's advertised limits. These are hints,
 /// but respecting them avoids pointless configure round-trips with windows
 /// that will refuse the size anyway.
@@ -215,6 +252,11 @@ fn onEvent(_: *river.WindowV1, event: river.WindowV1.Event, self: *Window) void
             if (config.float_children and self.parent != null and !self.floating_forced) {
                 self.floating = true;
             }
+            // A parent may be set after the window has already been mapped and
+            // laid out — Vivado's dialogs do exactly that — so the change of
+            // both float state and stacking has to be arranged for.
+            self.wm.needsManage();
+            self.wm.ipcDirty();
         },
 
         .fullscreen_requested => |ev| {

src/Wm.zig

@@ -740,6 +740,13 @@ fn assignOutputs(self: *Wm) void {
 /// pointer, and promoting it without moving focus would leave the keyboard
 /// talking to a window that is no longer on screen.
 ///
+/// Floating and fullscreen windows are not part of the tiled arrangement, so
+/// they are not attached to master, but they are focused like any other new
+/// window. dwm focuses new clients whether they float or not, and a dialog
+/// that appeared without being focused would also have kept a focus serial of
+/// zero — putting it at the bottom of the floating stack, behind the very
+/// window it belongs to.
+///
 /// Runs before `arrangeAll` so the serial set here is the one the layout picks
 /// its top window with, and the order set here is the one it arranges.
 fn focusNewWindows(self: *Wm) void {
@@ -754,15 +761,12 @@ fn focusNewWindows(self: *Wm) void {
         if (win.closed or !win.mapped) continue;
         win.wants_initial_focus = false;
 
-        // Floating and fullscreen windows are not part of the stack.
-        if (win.floating or win.fullscreen) continue;
-
         const out = win.output orelse continue;
         // Spawned onto a tag nobody is looking at: nothing to come to the
         // front of, and stealing focus would yank the user off their tag.
         if ((win.tags & out.tags) == 0) continue;
 
-        self.attachToMaster(win, out);
+        if (!win.floating and !win.fullscreen) self.attachToMaster(win, out);
 
         self.focus_serial += 1;
         win.focus_serial = self.focus_serial;
@@ -1036,10 +1040,40 @@ fn applyWindowState(self: *Wm) void {
 
 // ─── Render sequence ─────────────────────────────────────────────────────────
 
+/// Work out where each window's family — a window, its parents and its
+/// children — sits in the floating stack.
+///
+/// The floating layer is ordered by focus, which on its own puts a dialog
+/// behind its parent as soon as the parent is clicked. That is fatal for a
+/// modal dialog, the kind Vivado opens constantly: the parent ignores the
+/// click it just took the focus and the stacking order for, so the dialog is
+/// buried with no way to raise it again.
+///
+/// So a family rises and falls as a unit, on the most recent focus any of its
+/// members has had, and within the family the deeper window is the higher one.
+/// A child is therefore always drawn directly above its parent, which is what
+/// the protocol asks for.
+fn computeStacking(self: *Wm) void {
+    for (self.windows.items) |win| {
+        const a = win.ancestry();
+        win.stack_depth = a.depth;
+        // Staged on the root, so the loop below only ever reads focus serials.
+        win.stack_serial = win.focus_serial;
+    }
+    for (self.windows.items) |win| {
+        const root = win.ancestry().root;
+        root.stack_serial = @max(root.stack_serial, win.focus_serial);
+    }
+    for (self.windows.items) |win| {
+        win.stack_serial = win.ancestry().root.stack_serial;
+    }
+}
+
 fn render(self: *Wm) void {
     const gpa = self.gpa;
 
     self.scratch_order.clearRetainingCapacity();
+    self.computeStacking();
 
     for (self.outputs.items) |out| {
         // Tiled windows, bottom of the stack.
@@ -1081,16 +1115,19 @@ fn render(self: *Wm) void {
         }
 
         // Floating windows above, oldest focus first so the most recently
-        // focused ends up on top.
+        // focused family ends up on top, its dialogs above their parents.
         self.scratch_windows.clearRetainingCapacity();
         for (self.windows.items) |win| {
             if (win.output != out or win.closed) continue;
             if (!win.floating or win.fullscreen) continue;
             self.setHidden(win, !win.visible);
             if (!win.visible) continue;
+            // A dialog whose parent is fullscreen is drawn after the
+            // fullscreen layer instead; anything before it is covered.
+            if (win.hasFullscreenAncestor()) continue;
             self.scratch_windows.append(gpa, win) catch {};
         }
-        std.mem.sort(*Window, self.scratch_windows.items, {}, lessByFocus);
+        std.mem.sort(*Window, self.scratch_windows.items, {}, lessByStacking);
         for (self.scratch_windows.items) |win| {
             self.place(win, borderWidth(win));
             self.scratch_order.append(gpa, win) catch {};
@@ -1105,6 +1142,22 @@ fn render(self: *Wm) void {
             win.window.setBorders(.{}, 0, 0, 0, 0, 0);
             self.scratch_order.append(gpa, win) catch {};
         }
+
+        // Everything a fullscreen window opened, above it. A modal dialog is
+        // the only thing its parent will talk to, so hiding it under the
+        // parent leaves the user stuck.
+        self.scratch_windows.clearRetainingCapacity();
+        for (self.windows.items) |win| {
+            if (win.output != out or win.closed or win.fullscreen) continue;
+            if (!win.floating or !win.visible) continue;
+            if (!win.hasFullscreenAncestor()) continue;
+            self.scratch_windows.append(gpa, win) catch {};
+        }
+        std.mem.sort(*Window, self.scratch_windows.items, {}, lessByStacking);
+        for (self.scratch_windows.items) |win| {
+            self.place(win, borderWidth(win));
+            self.scratch_order.append(gpa, win) catch {};
+        }
     }
 
     // Applying place_top from the bottom up yields exactly the order above.
@@ -1120,7 +1173,9 @@ fn render(self: *Wm) void {
     self.window_manager.?.renderFinish();
 }
 
-fn lessByFocus(_: void, a: *Window, b: *Window) bool {
+fn lessByStacking(_: void, a: *Window, b: *Window) bool {
+    if (a.stack_serial != b.stack_serial) return a.stack_serial < b.stack_serial;
+    if (a.stack_depth != b.stack_depth) return a.stack_depth < b.stack_depth;
     return a.focus_serial < b.focus_serial;
 }