Wayland: Dropdown menu moves when navigated (#613) - cont'd

Menu windows containing sub-menus are now processed differently.

src/Fl_Menu.cxx

@@ -107,16 +107,22 @@ static const Fl_Menu_* button=0;
 
 ////////////////////////////////////////////////////////////////
 
-// tiny window for title of menu:
-class menutitle : public Fl_Menu_Window {
-  void draw();
+class window_with_items : public Fl_Menu_Window {
 public:
   const Fl_Menu_Item* menu;
+  window_with_items(int X, int Y, int W, int H, const Fl_Menu_Item *m) :
+    Fl_Menu_Window(X, Y, W, H, 0) { menu = m; }
+};
+
+// tiny window for title of menu:
+class menutitle : public window_with_items {
+  void draw();
+public:
   menutitle(int X, int Y, int W, int H, const Fl_Menu_Item*);
 };
 
 // each vertical menu has one of these:
-class menuwindow : public Fl_Menu_Window {
+class menuwindow : public window_with_items {
   friend class Fl_Window_Driver;
   friend struct Fl_Menu_Item;
   void draw();
@@ -132,7 +138,6 @@ public:
   int selected;
   int drawn_selected;   // last redraw has this selected
   int shortcutWidth;
-  const Fl_Menu_Item* menu;
   menuwindow(const Fl_Menu_Item* m, int X, int Y, int W, int H,
              const Fl_Menu_Item* picked, const Fl_Menu_Item* title,
              int menubar = 0, int menubar_title = 0, int right_edge = 0);
@@ -157,6 +162,10 @@ Fl_Window *Fl_Window_Driver::menu_parent() {
   return menuwindow::parent_;
 }
 
+const Fl_Menu_Item *Fl_Window_Driver::current_menu() {
+  if (!pWindow->menu_window()) return NULL;
+  return ((window_with_items*)pWindow)->menu;
+}
 /**
  \}
  \endcond
@@ -298,18 +307,17 @@ void Fl_Menu_Item::draw(int x, int y, int w, int h, const Fl_Menu_* m,
 }
 
 menutitle::menutitle(int X, int Y, int W, int H, const Fl_Menu_Item* L) :
-  Fl_Menu_Window(X, Y, W, H, 0) {
+  window_with_items(X, Y, W, H, L) {
   end();
   set_modal();
   clear_border();
   set_menu_window();
-  menu = L;
 }
 
 menuwindow::menuwindow(const Fl_Menu_Item* m, int X, int Y, int Wp, int Hp,
                        const Fl_Menu_Item* picked, const Fl_Menu_Item* t,
                        int menubar, int menubar_title, int right_edge)
-  : Fl_Menu_Window(X, Y, Wp, Hp, 0)
+  : window_with_items(X, Y, Wp, Hp, m)
 {
   int scr_x, scr_y, scr_w, scr_h;
   int tx = X, ty = Y;
@@ -321,7 +329,6 @@ menuwindow::menuwindow(const Fl_Menu_Item* m, int X, int Y, int Wp, int Hp,
   set_modal();
   clear_border();
   set_menu_window();
-  menu = m;
   if (m) m = m->first(); // find the first item that needs to be rendered
   drawn_selected = -1;
   if (button) {

src/Fl_Window_Driver.H

@@ -191,6 +191,7 @@ public:
   virtual void reposition_menu_window(int x, int y);
   virtual void menu_window_area(int &X, int &Y, int &W, int &H, int nscreen = -1);
   static Fl_Window *menu_parent();
+  const Fl_Menu_Item *current_menu();
 
   virtual fl_uintptr_t os_id() { return 0; }
 };

src/drivers/Wayland/Fl_Wayland_Window_Driver.cxx

@@ -30,6 +30,7 @@
 #include <FL/fl_ask.H>
 #include <FL/Fl.H>
 #include <FL/Fl_Image_Surface.H>
+#include <FL/Fl_Menu_Item.H>
 #include <string.h>
 #include <math.h> // for ceil()
 #include <sys/types.h> // for pid_t
@@ -950,6 +951,44 @@ static const char *get_prog_name() {
 }
 
 
+/* Implementation note about menu windows under Wayland.
+ Wayland offers a way to position popup windows such as menu windows using constraints
+ but hides the position of the window inside the display. Each popup is located relatively
+ to a parent window which can be a popup itself and MUST overlap or at least touch this parent.
+ FLTK computes the adequate position of a menu window in the display and then maps it
+ at that position.
+ These 2 logics are quite different.
+ The approach implemented here is two-fold.
+ 1) If a menu window is not taller than the display and contains no submenu, use Wayland
+ logic to position it. The benefit is that window menus become authorized to lay outside
+ the parent window but Wayland will not make them run beyond display limits.
+ We avoid submenu-containing popups because these could lead to
+ locate the future submenu outside its parent window, which Wayland forbids.
+ We avoid very tall menu windows because navigating with FLTK inside them would require to know
+ what part of them is visible which Wayland hides.
+ 2) Otherwise, have FLTK compute the menu position under the constraint that its active item
+ must be inside the menu-containing window. This constraint ensures Wayland will accept this
+ position because the required overlap is satisfied.
+ Function use_wayland_menu_positioning() below determines wether 1) or 2) is used for any
+ window menu. The result of this function is stored in the state member of the menu window's
+ struct wld_window for fast re-use.
+ */
+
+//returns true if win is a menuwindow without submenu and is not taller than display
+static bool use_wayland_menu_positioning(Fl_Window *win, Fl_Window *parent_win) {
+  if (!win->menu_window()) return true;
+  int XX, YY, WW, HH;
+  Fl::screen_xywh(XX, YY, WW, HH, parent_win->screen_num());
+  if (win->h() > HH) return false;
+  const Fl_Menu_Item *m = Fl_Window_Driver::driver(win)->current_menu();
+  while (m->label()) {
+    if (m->flags & (FL_SUBMENU | FL_SUBMENU_POINTER)) return false;
+    m = m->next();
+  }
+  return true;
+}
+
+
 Fl_X *Fl_Wayland_Window_Driver::makeWindow()
 {
   struct wld_window *new_window;
@@ -1005,10 +1044,9 @@ Fl_X *Fl_Wayland_Window_Driver::makeWindow()
     xdg_positioner_set_size(positioner, pWindow->w() * f , pWindow->h() * f );
     xdg_positioner_set_anchor(positioner, XDG_POSITIONER_ANCHOR_TOP_LEFT);
     xdg_positioner_set_gravity(positioner, XDG_POSITIONER_GRAVITY_BOTTOM_RIGHT);
-    int XX, YY, WW, HH;
-    Fl::screen_xywh(XX, YY, WW, HH, parent_win->screen_num());
-    if (pWindow->h() <= HH) {
-      // prevent menuwindow from expanding beyond display bottom
+    new_window->state = use_wayland_menu_positioning(pWindow, parent_win);
+    if (new_window->state) {
+      // prevent menuwindow from expanding beyond display limits
       xdg_positioner_set_constraint_adjustment(positioner,
                                              XDG_POSITIONER_CONSTRAINT_ADJUSTMENT_SLIDE_Y);
     }
@@ -1557,9 +1595,10 @@ void Fl_Wayland_Window_Driver::reposition_menu_window(int x, int y) {
 void Fl_Wayland_Window_Driver::menu_window_area(int &X, int &Y, int &W, int &H, int nscreen) {
   Fl_Window *parent = Fl_Window_Driver::menu_parent();
   if (parent) {
-    int XX,YY,WW,HH;
-    Fl::screen_xywh(XX, YY, WW, HH, parent->screen_num());
-    if (pWindow->h() > HH) { // the menu window is higher than the display
+    struct wld_window *xid = fl_wl_xid(pWindow);
+    bool condition = xid ? xid->state : use_wayland_menu_positioning(pWindow, parent);
+    if (!condition) {
+      // keep active menu part inside parent window
       X = parent->x();
       Y = parent->y();
       W = parent->w();