Add a test framework and stacking tests

Add a basic framework for tests of Mutter handling of client behavior;
mutter-test-runner is a Mutter-based compositor that forks off instances
of mutter-test-client and sends commands to them based on scripts.
The scripts also include assertions.

mutter-test-runner always runs in nested-Wayland mode since the separate
copy of Xwayland is helpful in giving a reliably clean X server to
test against.

Initially the commands and assertions are designed to test the stacking
behavior of Mutter, but the framework should be extensible to test other
parts of client behavior like focus.

The tests are installed according to:

https://wiki.gnome.org/Initiatives/GnomeGoals/InstalledTests

if --enable-installed-tests is passed to configure. You can run them
uninstalled with:

 cd src && make run-tests

(Not in 'make check' to avoid breaking 'make distcheck' if Mutter can't be
run nested.)

https://bugzilla.gnome.org/show_bug.cgi?id=736505

.gitignore

@@ -44,6 +44,8 @@ po/*.pot
 libmutter.pc
 mutter
 mutter-restart-helper
+mutter-test-client
+mutter-test-runner
 org.gnome.mutter.gschema.valid
 org.gnome.mutter.gschema.xml
 org.gnome.mutter.wayland.gschema.valid

configure.ac

@@ -127,6 +127,12 @@ AC_ARG_WITH([xwayland-path],
             [XWAYLAND_PATH="$withval"],
             [XWAYLAND_PATH="$bindir/Xwayland"])
 
+AC_ARG_ENABLE(installed_tests,
+              AS_HELP_STRING([--enable-installed-tests],
+                             [Install test programs (default: no)]),,
+              [enable_installed_tests=no])
+AM_CONDITIONAL(BUILDOPT_INSTALL_TESTS, test x$enable_installed_tests = xyes)
+
 ## here we get the flags we'll actually use
 
 # Unconditionally use this dir to avoid a circular dep with gnomecc

src/Makefile-tests.am

@@ -0,0 +1,46 @@
+# A framework for running scripted tests
+
+if BUILDOPT_INSTALL_TESTS
+stackingdir = $(pkgdatadir)/tests/stacking
+dist_stacking_DATA =				\
+	tests/stacking/basic-x11.metatest	\
+	tests/stacking/basic-wayland.metatest	\
+	tests/stacking/mixed-windows.metatest   \
+	tests/stacking/override-redirect.metatest
+
+mutter-all.test: tests/mutter-all.test.in
+	$(AM_V_GEN) sed  -e "s|@libexecdir[@]|$(libexecdir)|g"  $< > $@.tmp && mv $@.tmp $@
+
+installedtestsdir = $(datadir)/installed-tests/mutter
+installedtests_DATA = mutter-all.test
+
+installedtestsbindir = $(libexecdir)/installed-tests/mutter
+installedtestsbin_PROGRAMS = mutter-test-client mutter-test-runner
+else
+noinst_PROGRAMS += mutter-test-client mutter-test-runner
+endif
+
+EXTRA_DIST += tests/mutter-all.test.in
+
+mutter_test_client_SOURCES = tests/test-client.c
+mutter_test_client_LDADD = $(MUTTER_LIBS) libmutter.la
+
+mutter_test_runner_SOURCES = tests/test-runner.c
+mutter_test_runner_LDADD = $(MUTTER_LIBS) libmutter.la
+
+.PHONY: run-tests
+
+run-tests: mutter-test-client mutter-test-runner
+	./mutter-test-runner $(dist_stacking_DATA)
+
+# Some random test programs for bits of the code
+
+testboxes_SOURCES = core/testboxes.c
+testgradient_SOURCES = ui/testgradient.c
+testasyncgetprop_SOURCES = x11/testasyncgetprop.c
+
+noinst_PROGRAMS+=testboxes testgradient testasyncgetprop
+
+testboxes_LDADD = $(MUTTER_LIBS) libmutter.la
+testgradient_LDADD = $(MUTTER_LIBS) libmutter.la
+testasyncgetprop_LDADD = $(MUTTER_LIBS) libmutter.la

src/Makefile.am

@@ -5,6 +5,8 @@ lib_LTLIBRARIES = libmutter.la
 
 SUBDIRS=compositor/plugins
 
+EXTRA_DIST =
+
 AM_CPPFLAGS = \
 	-DCLUTTER_ENABLE_COMPOSITOR_API					\
 	-DCLUTTER_ENABLE_EXPERIMENTAL_API				\
@@ -325,6 +327,7 @@ nodist_libmutterinclude_HEADERS =		\
 	$(libmutterinclude_built_headers)
 
 bin_PROGRAMS=mutter
+noinst_PROGRAMS=
 
 mutter_SOURCES = core/mutter.c
 mutter_LDADD = $(MUTTER_LIBS) libmutter.la
@@ -333,6 +336,8 @@ libexec_PROGRAMS = mutter-restart-helper
 mutter_restart_helper_SOURCES = core/restart-helper.c
 mutter_restart_helper_LDADD = $(MUTTER_LIBS)
 
+include Makefile-tests.am
+
 if HAVE_INTROSPECTION
 include $(INTROSPECTION_MAKEFILE)
 
@@ -366,16 +371,6 @@ Meta-$(api_version).gir: libmutter.la
 
 endif
 
-testboxes_SOURCES = core/testboxes.c
-testgradient_SOURCES = ui/testgradient.c
-testasyncgetprop_SOURCES = x11/testasyncgetprop.c
-
-noinst_PROGRAMS=testboxes testgradient testasyncgetprop
-
-testboxes_LDADD = $(MUTTER_LIBS) libmutter.la
-testgradient_LDADD = $(MUTTER_LIBS) libmutter.la
-testasyncgetprop_LDADD = $(MUTTER_LIBS) libmutter.la
-
 dbus_idle_built_sources = meta-dbus-idle-monitor.c meta-dbus-idle-monitor.h
 
 CLEANFILES =					\
@@ -389,7 +384,7 @@ DISTCLEANFILES = 				\
 pkgconfigdir = $(libdir)/pkgconfig
 pkgconfig_DATA = libmutter.pc
 
-EXTRA_DIST =				\
+EXTRA_DIST +=				\
 	$(wayland_protocols)		\
 	libmutter.pc.in \
 	mutter-enum-types.h.in \

src/tests/README

@@ -0,0 +1,85 @@
+This directory implements a framework for automated tests of Mutter. The basic
+idea is that mutter-test-runner acts as the window manager and compositor, and
+forks off instances of mutter-test-client to act as clients.
+
+There's a simple scripting language for tests. A very small test would look like:
+
+---
+# Start up a new X11 client with the client id 1 (doesn't have to be an integer)
+# Windows for this client will be referred to as 1/<window-id>
+new_client 1 x11
+
+# Create and show two windows - again the IDs don't have to be integers
+create 1/1
+show 1/1
+create 1/2
+show 1/2
+
+# Wait for the commands we've executed in the clients to reach Mutter
+wait
+
+# Check that the windows are in the order we expect
+assert_stacking 1/1 1/2
+---
+
+Running
+=======
+
+The tests are installed according to:
+
+https://wiki.gnome.org/Initiatives/GnomeGoals/InstalledTests
+
+if --enable-installed-tests is passed to configure. You can run them
+uninstalled with:
+
+ cd src && make run-tests
+
+Command reference
+=================
+
+The following commands are supported. Quoting and comments follow shell rules.
+
+new_client <client-id> [wayland|x11]
+ Starts a client, connecting by either Wayland or X11. The client
+ will subsequently be known with the given client-id (an arbitrary
+ string)
+
+quit_client <client-id>
+ Destroys all windows for the client, waits for that to be processed,
+ then instructs the client to exit.
+
+create <client-id>/<window-id> [override]
+ Creates a new window. For the X11 backend, the keyword 'override'
+ can be given to create an override-redirect
+
+show <client-id>/<window-id>
+hide <client-id>/<window-id>
+ Ask the client to show (map) or hide (unmap) the given window
+
+activate <client-id>/<window-id>
+ Ask the client to raise and focus the given window. This is currently a no-op
+ for Wayland, where this capability is not supported in the protocol.
+
+local_activate <client-id>-<window-id>
+  The same as 'activate', but the operation is done directly inside Mutter
+  and works for both backends
+
+raise <client-id>/<window-id>
+lower <client-id>/<window-id>
+  Ask the client to raise or lower the given window ID. This is a no-op
+  for Wayland clients. (It's also considered discouraged, but supported, for
+  non-override-redirect X11 clients.)
+
+destroy <client-id>/<window-id>
+  Destroy the given window
+
+wait
+  Wait until all requests sent by Mutter to clients have been received by Mutter,
+  and then wait until all requests by Mutter have been processed by the X server.
+
+assert_stacking <client-id>/<window-id> <client-id>/<window-id> ...
+  Assert that the list of client windows known to Mutter is as given and in
+  the given order, bottom to top.
+
+  This function also queries the X server stack and verifies that Mutter's
+  expectation of the X server stack matches reality.

src/tests/stacking/basic-wayland.metatest

@@ -0,0 +1,22 @@
+new_client 1 wayland
+create 1/1
+show 1/1
+create 1/2
+show 1/2
+wait
+assert_stacking 1/1 1/2
+
+# Currently Wayland clients have no wait to bring themselves to the user's
+# attention; gtk_window_present() is a no-op with the X11 backend of GTK+
+
+# activate 1/1
+# wait
+# assert_stacking 1/2 1/1
+# activate 1/2
+# wait
+# assert_stacking 1/1 1/2
+
+local_activate 1/1
+assert_stacking 1/2 1/1
+local_activate 1/2
+assert_stacking 1/1 1/2

src/tests/stacking/basic-x11.metatest

@@ -0,0 +1,19 @@
+new_client 1 x11
+create 1/1
+show 1/1
+create 1/2
+show 1/2
+wait
+assert_stacking 1/1 1/2
+
+activate 1/1
+wait
+assert_stacking 1/2 1/1
+activate 1/2
+wait
+assert_stacking 1/1 1/2
+
+local_activate 1/1
+assert_stacking 1/2 1/1
+local_activate 1/2
+assert_stacking 1/1 1/2

src/tests/stacking/mixed-windows.metatest

@@ -0,0 +1,26 @@
+new_client w wayland
+new_client x x11
+
+create w/1
+show w/1
+create w/2
+show w/2
+wait
+
+create x/1
+show x/1
+create x/2
+show x/2
+wait
+
+assert_stacking w/1 w/2 x/1 x/2
+
+local_activate w/1
+assert_stacking w/2 x/1 x/2 w/1
+
+local_activate x/1
+assert_stacking w/2 x/2 w/1 x/1
+
+lower x/1
+wait
+assert_stacking x/1 w/2 x/2 w/1

src/tests/stacking/override-redirect.metatest

@@ -0,0 +1,19 @@
+new_client 1 x11
+create 1/1
+show 1/1
+create 1/2 override
+show 1/2
+wait
+assert_stacking 1/1 1/2
+
+activate 1/1
+wait
+assert_stacking 1/1 1/2
+
+lower 1/2
+wait
+assert_stacking 1/2 1/1
+
+raise 1/2
+wait
+assert_stacking 1/1 1/2

src/tests/test-client.c

@@ -0,0 +1,339 @@
+/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- */
+
+/*
+ * Copyright (C) 2014 Red Hat, Inc.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <gio/gunixinputstream.h>
+#include <gtk/gtk.h>
+#include <gdk/gdkx.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <string.h>
+#include <X11/extensions/sync.h>
+
+char *client_id = "0";
+static gboolean wayland;
+GHashTable *windows;
+
+static void read_next_line (GDataInputStream *in);
+
+static GtkWidget *
+lookup_window (const char *window_id)
+{
+  GtkWidget *window = g_hash_table_lookup (windows, window_id);
+  if (!window)
+    g_print ("Window %s doesn't exist", window_id);
+
+  return window;
+}
+
+static void
+process_line (const char *line)
+{
+  GError *error = NULL;
+  int argc;
+  char **argv;
+
+  if (!g_shell_parse_argv (line, &argc, &argv, &error))
+    {
+      g_print ("error parsing command: %s", error->message);
+      g_error_free (error);
+      return;
+    }
+
+  if (argc < 1)
+    {
+      g_print ("Empty command");
+      goto out;
+    }
+
+  if (strcmp (argv[0], "create") == 0)
+    {
+      int i;
+
+      if (argc  < 2)
+        {
+          g_print ("usage: create <id> [override]");
+          goto out;
+        }
+
+      if (g_hash_table_lookup (windows, argv[1]))
+        {
+          g_print ("window %s already exists", argv[1]);
+          goto out;
+        }
+
+      gboolean override = FALSE;
+      for (i = 2; i < argc; i++)
+        if (strcmp (argv[i], "override") == 0)
+          override = TRUE;
+
+      GtkWidget *window = gtk_window_new (override ? GTK_WINDOW_POPUP : GTK_WINDOW_TOPLEVEL);
+      g_hash_table_insert (windows, g_strdup (argv[1]), window);
+
+      gtk_window_set_default_size (GTK_WINDOW (window), 100, 100);
+
+      gchar *title = g_strdup_printf ("test/%s/%s", client_id, argv[1]);
+      gtk_window_set_title (GTK_WINDOW (window), title);
+      g_free (title);
+
+      gtk_widget_realize (window);
+
+      if (!wayland)
+        {
+          /* The cairo xlib backend creates a window when initialized, which
+           * confuses our testing if it happens asynchronously the first
+           * time a window is painted. By creating an Xlib surface and
+           * destroying it, we force initialization at a more predictable time.
+           */
+          GdkWindow *window_gdk = gtk_widget_get_window (window);
+          cairo_surface_t *surface = gdk_window_create_similar_surface (window_gdk,
+                                                                        CAIRO_CONTENT_COLOR,
+                                                                        1, 1);
+          cairo_surface_destroy (surface);
+        }
+
+    }
+  else if (strcmp (argv[0], "show") == 0)
+    {
+      if (argc != 2)
+        {
+          g_print ("usage: show <id>");
+          goto out;
+        }
+
+      GtkWidget *window = lookup_window (argv[1]);
+      if (!window)
+        goto out;
+
+      gtk_widget_show (window);
+    }
+  else if (strcmp (argv[0], "hide") == 0)
+    {
+      if (argc != 2)
+        {
+          g_print ("usage: hide <id>");
+          goto out;
+        }
+
+      GtkWidget *window = lookup_window (argv[1]);
+      if (!window)
+        goto out;
+
+      gtk_widget_hide (window);
+    }
+  else if (strcmp (argv[0], "activate") == 0)
+    {
+      if (argc != 2)
+        {
+          g_print ("usage: activate <id>");
+          goto out;
+        }
+
+      GtkWidget *window = lookup_window (argv[1]);
+      if (!window)
+        goto out;
+
+      gtk_window_present (GTK_WINDOW (window));
+    }
+  else if (strcmp (argv[0], "raise") == 0)
+    {
+      if (argc != 2)
+        {
+          g_print ("usage: raise <id>");
+          goto out;
+        }
+
+      GtkWidget *window = lookup_window (argv[1]);
+      if (!window)
+        goto out;
+
+      gdk_window_raise (gtk_widget_get_window (window));
+    }
+  else if (strcmp (argv[0], "lower") == 0)
+    {
+      if (argc != 2)
+        {
+          g_print ("usage: lower <id>");
+          goto out;
+        }
+
+      GtkWidget *window = lookup_window (argv[1]);
+      if (!window)
+        goto out;
+
+      gdk_window_lower (gtk_widget_get_window (window));
+    }
+  else if (strcmp (argv[0], "destroy") == 0)
+    {
+      if (argc != 2)
+        {
+          g_print ("usage: destroy <id>");
+          goto out;
+        }
+
+      GtkWidget *window = lookup_window (argv[1]);
+      if (!window)
+        goto out;
+
+      g_hash_table_remove (windows, argv[1]);
+      gtk_widget_destroy (window);
+    }
+  else if (strcmp (argv[0], "destroy_all") == 0)
+    {
+      if (argc != 1)
+        {
+          g_print ("usage: destroy_all");
+          goto out;
+        }
+
+      GHashTableIter iter;
+      gpointer key, value;
+
+      g_hash_table_iter_init (&iter, windows);
+      while (g_hash_table_iter_next (&iter, &key, &value))
+        gtk_widget_destroy (value);
+
+      g_hash_table_remove_all (windows);
+    }
+  else if (strcmp (argv[0], "sync") == 0)
+    {
+      if (argc != 1)
+        {
+          g_print ("usage: sync");
+          goto out;
+        }
+
+      gdk_display_sync (gdk_display_get_default ());
+    }
+  else if (strcmp (argv[0], "set_counter") == 0)
+    {
+      XSyncCounter counter;
+      int value;
+
+      if (argc != 3)
+        {
+          g_print ("usage: set_counter <counter> <value>");
+          goto out;
+        }
+
+      if (wayland)
+        {
+          g_print ("usage: set_counter can only be used for X11");
+          goto out;
+        }
+
+      counter = strtoul(argv[1], NULL, 10);
+      value = atoi(argv[2]);
+      XSyncValue sync_value;
+      XSyncIntToValue (&sync_value, value);
+
+      XSyncSetCounter (gdk_x11_display_get_xdisplay (gdk_display_get_default ()),
+                       counter, sync_value);
+    }
+  else
+    {
+      g_print ("Unknown command %s", argv[0]);
+      goto out;
+    }
+
+  g_print ("OK\n");
+
+ out:
+  g_strfreev (argv);
+}
+
+static void
+on_line_received (GObject      *source,
+                  GAsyncResult *result,
+                  gpointer      user_data)
+{
+  GDataInputStream *in = G_DATA_INPUT_STREAM (source);
+  GError *error = NULL;
+  gsize length;
+  char *line = g_data_input_stream_read_line_finish_utf8 (in, result, &length, &error);
+
+  if (line == NULL)
+    {
+      if (error != NULL)
+        g_printerr ("Error reading from stdin: %s\n", error->message);
+      gtk_main_quit ();
+      return;
+    }
+
+  process_line (line);
+  g_free (line);
+  read_next_line (in);
+}
+
+static void
+read_next_line (GDataInputStream *in)
+{
+  g_data_input_stream_read_line_async (in, G_PRIORITY_DEFAULT, NULL,
+                                       on_line_received, NULL);
+}
+
+const GOptionEntry options[] = {
+  {
+    "wayland", 0, 0, G_OPTION_ARG_NONE,
+    &wayland,
+    "Create a wayland client, not an X11 one",
+    NULL
+  },
+  {
+    "client-id", 0, 0, G_OPTION_ARG_STRING,
+    &client_id,
+    "Identifier used in Window titles for this client",
+    "CLIENT_ID",
+  },
+  { NULL }
+};
+
+int
+main(int argc, char **argv)
+{
+  GOptionContext *context = g_option_context_new (NULL);
+  GError *error = NULL;
+
+  g_option_context_add_main_entries (context, options, NULL);
+
+  if (!g_option_context_parse (context,
+                               &argc, &argv, &error))
+    {
+      g_printerr ("%s", error->message);
+      return 1;
+    }
+
+  if (wayland)
+    gdk_set_allowed_backends ("wayland");
+  else
+    gdk_set_allowed_backends ("x11");
+
+  gtk_init (NULL, NULL);
+
+  windows = g_hash_table_new_full (g_str_hash, g_str_equal,
+                                   g_free, NULL);
+
+  GInputStream *raw_in = g_unix_input_stream_new (0, FALSE);
+  GDataInputStream *in = g_data_input_stream_new (raw_in);
+
+  read_next_line (in);
+
+  gtk_main ();
+
+  return 0;
+}