diff --git a/clutter/meson.build b/clutter/meson.build
index 94f1eb0f1..ac6db2fd2 100644
--- a/clutter/meson.build
+++ b/clutter/meson.build
@@ -59,7 +59,7 @@ if have_wayland
   ]
 endif
 
-if have_x11
+if have_x11_client
   clutter_pkg_deps += [
     x11_dep,
   ]
diff --git a/cogl/cogl/meson.build b/cogl/cogl/meson.build
index 531cf8f43..d5da0151e 100644
--- a/cogl/cogl/meson.build
+++ b/cogl/cogl/meson.build
@@ -9,8 +9,8 @@ cdata.set('COGL_HAS_EGL_PLATFORM_XLIB_SUPPORT', have_egl_xlib)
 cdata.set('COGL_HAS_EGL_SUPPORT', have_egl)
 cdata.set('COGL_HAS_X11', have_x11)
 cdata.set('COGL_HAS_X11_SUPPORT', have_x11)
-cdata.set('COGL_HAS_XLIB', have_x11)
-cdata.set('COGL_HAS_XLIB_SUPPORT', have_x11)
+cdata.set('COGL_HAS_XLIB', have_x11_client)
+cdata.set('COGL_HAS_XLIB_SUPPORT', have_x11_client)
 cdata.set('COGL_HAS_TRACING', have_profiler)
 
 cogl_defines_h = configure_file(
@@ -360,7 +360,7 @@ cogl_sources = [
   'cogl-graphene.c',
 ]
 
-if have_x11
+if have_x11_client
   cogl_nonintrospected_headers += [
     'winsys/cogl-texture-pixmap-x11.h',
     'cogl-xlib.h',
diff --git a/cogl/meson.build b/cogl/meson.build
index a3c61cfb7..70d22a9ba 100644
--- a/cogl/meson.build
+++ b/cogl/meson.build
@@ -47,7 +47,7 @@ if have_egl
   ]
 endif
 
-if have_x11
+if have_x11_client
   cogl_pkg_deps += [
     x11_dep,
   ]
diff --git a/meson.build b/meson.build
index b4df4a30e..4768a8d14 100644
--- a/meson.build
+++ b/meson.build
@@ -101,7 +101,6 @@ mutter_installed_tests_libexecdir = join_paths(
   libexecdir, 'installed-tests', libmutter_name)
 
 m_dep = cc.find_library('m', required: true)
-x11_dep = dependency('x11')
 graphene_dep = dependency('graphene-gobject-1.0', version: graphene_req)
 gtk3_dep = dependency('gtk+-3.0', version: gtk3_req)
 gdk_pixbuf_dep = dependency('gdk-pixbuf-2.0')
@@ -120,31 +119,42 @@ gthread_dep = dependency('gobject-2.0', version: glib_req)
 gmodule_no_export_dep = dependency('gmodule-no-export-2.0', version: glib_req)
 gnome_settings_daemon_dep = dependency('gnome-settings-daemon', required: false)
 json_glib_dep = dependency('json-glib-1.0', version: json_glib_req)
-xcomposite_dep = dependency('xcomposite', version: xcomposite_req)
-xcursor_dep = dependency('xcursor')
-xdamage_dep = dependency('xdamage')
-xext_dep = dependency('xext')
-xfixes_dep = dependency('xfixes', version: xfixes_req)
-xi_dep = dependency('xi', version: xi_req)
-xtst_dep = dependency('xtst')
-xkbfile_dep = dependency('xkbfile')
-xkeyboard_config_dep = dependency('xkeyboard-config')
 xkbcommon_dep = dependency('xkbcommon', version: xkbcommon_req)
-xkbcommon_x11_dep = dependency('xkbcommon-x11')
-xrender_dep = dependency('xrender')
-x11_xcb_dep = dependency('x11-xcb')
-xrandr_dep = dependency('xrandr', version: xrandr_req)
-xcb_randr_dep = dependency('xcb-randr')
-xcb_res_dep = dependency('xcb-res')
-xinerama_dep = dependency('xinerama')
-xau_dep = dependency('xau')
 ice_dep = dependency('ice')
 atk_dep = dependency('atk', version: atk_req)
 libcanberra_dep = dependency('libcanberra', version: libcanberra_req)
 dbus_dep = dependency('dbus-1')
 
+have_wayland = get_option('wayland')
 # For now always require X11 support
 have_x11 = true
+have_xwayland = have_wayland # for now default to have_wayland
+have_x11_client = have_x11 or have_xwayland
+
+if have_xwayland and not have_wayland
+  error('XWayland support requires Wayland support enabled')
+endif
+
+if have_x11_client
+  x11_dep = dependency('x11')
+  xcomposite_dep = dependency('xcomposite', version: xcomposite_req)
+  xcursor_dep = dependency('xcursor')
+  xdamage_dep = dependency('xdamage')
+  xext_dep = dependency('xext')
+  xfixes_dep = dependency('xfixes', version: xfixes_req)
+  xi_dep = dependency('xi', version: xi_req)
+  xtst_dep = dependency('xtst')
+  xkbfile_dep = dependency('xkbfile')
+  xkeyboard_config_dep = dependency('xkeyboard-config')
+  xkbcommon_x11_dep = dependency('xkbcommon-x11')
+  xrender_dep = dependency('xrender')
+  x11_xcb_dep = dependency('x11-xcb')
+  xrandr_dep = dependency('xrandr', version: xrandr_req)
+  xcb_randr_dep = dependency('xcb-randr')
+  xcb_res_dep = dependency('xcb-res')
+  xinerama_dep = dependency('xinerama')
+  xau_dep = dependency('xau')
+endif
 
 
 have_gnome_desktop = get_option('libgnome_desktop')
@@ -170,7 +180,7 @@ if have_glx
   endif
 endif
 
-have_egl_xlib = have_egl and have_x11
+have_egl_xlib = have_egl and have_x11_client
 
 have_gles2 = get_option('gles2')
 if have_gles2
@@ -182,7 +192,6 @@ if have_gles2
   endif
 endif
 
-have_wayland = get_option('wayland')
 if have_wayland
   wayland_server_dep = dependency('wayland-server', version: wayland_server_req)
   wayland_client_dep = dependency('wayland-client', version: wayland_server_req)
@@ -304,6 +313,9 @@ if have_tests
     if not have_wayland
       error('Tests require Wayland to be enabled')
     endif
+    if not have_x11_client
+      error('Tests requires an X11 client')
+    endif
   endif
   have_native_tests = get_option('native_tests')
   if have_native_tests
@@ -469,8 +481,10 @@ cdata.set('HAVE_STARTUP_NOTIFICATION', have_startup_notification)
 cdata.set('HAVE_INTROSPECTION', have_introspection)
 cdata.set('HAVE_PROFILER', have_profiler)
 
-xkb_base = xkeyboard_config_dep.get_pkgconfig_variable('xkb_base')
-cdata.set_quoted('XKB_BASE', xkb_base)
+if have_x11_client
+  xkb_base = xkeyboard_config_dep.get_pkgconfig_variable('xkb_base')
+  cdata.set_quoted('XKB_BASE', xkb_base)
+endif
 
 if cc.has_header_symbol('sys/prctl.h', 'prctl')
   cdata.set('HAVE_SYS_PRCTL', 1)
@@ -479,7 +493,7 @@ endif
 have_xwayland_initfd = false
 have_xwayland_listenfd = false
 have_xwayland_terminate_delay = false
-if have_wayland
+if have_xwayland
   xwayland_dep = dependency('xwayland', required: false)
 
   xwayland_path = get_option('xwayland_path')
@@ -543,10 +557,12 @@ if have_wayland
 endif
 
 have_xsetioerrorexithandler = false
-if cc.has_function('XSetIOErrorExitHandler', dependencies: x11_dep,
-                   prefix: '''#include <X11/Xlib.h>''')
-  have_xsetioerrorexithandler = true
-  cdata.set('HAVE_XSETIOERROREXITHANDLER', 1)
+if have_x11_client
+  if cc.has_function('XSetIOErrorExitHandler', dependencies: x11_dep,
+                    prefix: '''#include <X11/Xlib.h>''')
+    have_xsetioerrorexithandler = true
+    cdata.set('HAVE_XSETIOERROREXITHANDLER', 1)
+  endif
 endif
 
 optional_functions = [
@@ -607,6 +623,8 @@ summary('GLX', have_glx, section: 'Rendering APIs')
 
 summary('Wayland', have_wayland, section: 'Options')
 summary('Wayland EGLStream', have_wayland_eglstream, section: 'Options')
+summary('X11', have_x11, section: 'Options')
+summary('XWayland', have_xwayland, section: 'Options')
 summary('Native Backend', have_native_backend, section: 'Options')
 summary('EGL Device', have_egl_device, section: 'Options')
 summary('Remote desktop', have_remote_desktop, section: 'Options')
diff --git a/src/meson.build b/src/meson.build
index 14a8a986a..42d7a5a55 100644
--- a/src/meson.build
+++ b/src/meson.build
@@ -252,58 +252,6 @@ mutter_sources = [
   'backends/meta-viewport-info.h',
   'backends/meta-virtual-monitor.c',
   'backends/meta-virtual-monitor.h',
-  'backends/x11/cm/meta-backend-x11-cm.c',
-  'backends/x11/cm/meta-backend-x11-cm.h',
-  'backends/x11/cm/meta-cursor-sprite-xfixes.c',
-  'backends/x11/cm/meta-cursor-sprite-xfixes.h',
-  'backends/x11/cm/meta-renderer-x11-cm.c',
-  'backends/x11/cm/meta-renderer-x11-cm.h',
-  'backends/x11/meta-backend-x11.c',
-  'backends/x11/meta-backend-x11.h',
-  'backends/x11/meta-barrier-x11.c',
-  'backends/x11/meta-barrier-x11.h',
-  'backends/x11/meta-clutter-backend-x11.c',
-  'backends/x11/meta-clutter-backend-x11.h',
-  'backends/x11/meta-crtc-xrandr.c',
-  'backends/x11/meta-crtc-xrandr.h',
-  'backends/x11/meta-cursor-renderer-x11.c',
-  'backends/x11/meta-cursor-renderer-x11.h',
-  'backends/x11/meta-cursor-tracker-x11.c',
-  'backends/x11/meta-cursor-tracker-x11.h',
-  'backends/x11/meta-event-x11.c',
-  'backends/x11/meta-event-x11.h',
-  'backends/x11/meta-gpu-xrandr.c',
-  'backends/x11/meta-gpu-xrandr.h',
-  'backends/x11/meta-input-device-x11.c',
-  'backends/x11/meta-input-device-x11.h',
-  'backends/x11/meta-input-device-tool-x11.c',
-  'backends/x11/meta-input-device-tool-x11.h',
-  'backends/x11/meta-input-settings-x11.c',
-  'backends/x11/meta-input-settings-x11.h',
-  'backends/x11/meta-seat-x11.c',
-  'backends/x11/meta-seat-x11.h',
-  'backends/x11/meta-keymap-x11.c',
-  'backends/x11/meta-keymap-x11.h',
-  'backends/x11/meta-monitor-manager-xrandr.c',
-  'backends/x11/meta-monitor-manager-xrandr.h',
-  'backends/x11/meta-output-xrandr.c',
-  'backends/x11/meta-output-xrandr.h',
-  'backends/x11/meta-renderer-x11.c',
-  'backends/x11/meta-renderer-x11.h',
-  'backends/x11/meta-stage-x11.c',
-  'backends/x11/meta-stage-x11.h',
-  'backends/x11/meta-virtual-input-device-x11.c',
-  'backends/x11/meta-virtual-input-device-x11.h',
-  'backends/x11/meta-xkb-a11y-x11.c',
-  'backends/x11/meta-xkb-a11y-x11.h',
-  'backends/x11/nested/meta-backend-x11-nested.c',
-  'backends/x11/nested/meta-backend-x11-nested.h',
-  'backends/x11/nested/meta-cursor-renderer-x11-nested.c',
-  'backends/x11/nested/meta-cursor-renderer-x11-nested.h',
-  'backends/x11/nested/meta-stage-x11-nested.c',
-  'backends/x11/nested/meta-stage-x11-nested.h',
-  'backends/x11/nested/meta-renderer-x11-nested.c',
-  'backends/x11/nested/meta-renderer-x11-nested.h',
   'compositor/clutter-utils.c',
   'compositor/clutter-utils.h',
   'compositor/cogl-utils.c',
@@ -320,8 +268,6 @@ mutter_sources = [
   'compositor/meta-background-private.h',
   'compositor/meta-compositor-server.c',
   'compositor/meta-compositor-server.h',
-  'compositor/meta-compositor-x11.c',
-  'compositor/meta-compositor-x11.h',
   'compositor/meta-cullable.c',
   'compositor/meta-cullable.h',
   'compositor/meta-dnd-actor.c',
@@ -340,16 +286,10 @@ mutter_sources = [
   'compositor/meta-shaped-texture-private.h',
   'compositor/meta-surface-actor.c',
   'compositor/meta-surface-actor.h',
-  'compositor/meta-surface-actor-x11.c',
-  'compositor/meta-surface-actor-x11.h',
-  'compositor/meta-sync-ring.c',
-  'compositor/meta-sync-ring.h',
   'compositor/meta-texture-tower.c',
   'compositor/meta-texture-tower.h',
   'compositor/meta-window-actor.c',
   'compositor/meta-window-actor-private.h',
-  'compositor/meta-window-actor-x11.c',
-  'compositor/meta-window-actor-x11.h',
   'compositor/meta-window-group.c',
   'compositor/meta-window-group-private.h',
   'compositor/meta-window-shape.c',
@@ -418,50 +358,121 @@ mutter_sources = [
   'core/window-private.h',
   'core/workspace.c',
   'core/workspace-private.h',
-  'ui/frames.c',
-  'ui/frames.h',
-  'ui/theme.c',
-  'ui/theme-private.h',
-  'ui/ui.c',
-  'ui/ui.h',
-  'x11/atomnames.h',
-  'x11/events.c',
-  'x11/events.h',
-  'x11/group.c',
-  'x11/group-private.h',
-  'x11/group-props.c',
-  'x11/group-props.h',
-  'x11/iconcache.c',
-  'x11/iconcache.h',
-  'x11/meta-selection-source-x11.c',
-  'x11/meta-selection-source-x11-private.h',
-  'x11/meta-startup-notification-x11.c',
-  'x11/meta-startup-notification-x11.h',
-  'x11/meta-x11-display.c',
-  'x11/meta-x11-display-private.h',
-  'x11/meta-x11-errors.c',
-  'x11/meta-x11-selection.c',
-  'x11/meta-x11-selection-private.h',
-  'x11/meta-x11-selection-input-stream.c',
-  'x11/meta-x11-selection-input-stream-private.h',
-  'x11/meta-x11-selection-output-stream.c',
-  'x11/meta-x11-selection-output-stream-private.h',
-  'x11/meta-x11-stack.c',
-  'x11/meta-x11-stack-private.h',
-  'x11/meta-x11-window-control.c',
-  'x11/meta-x11-window-control.h',
-  'x11/mutter-Xatomtype.h',
-  'x11/session.c',
-  'x11/session.h',
-  'x11/window-props.c',
-  'x11/window-props.h',
-  'x11/window-x11.c',
-  'x11/window-x11.h',
-  'x11/window-x11-private.h',
-  'x11/xprops.c',
-  'x11/xprops.h',
 ]
 
+if have_x11
+  mutter_sources += [
+    'backends/x11/cm/meta-backend-x11-cm.c',
+    'backends/x11/cm/meta-backend-x11-cm.h',
+    'backends/x11/cm/meta-cursor-sprite-xfixes.c',
+    'backends/x11/cm/meta-cursor-sprite-xfixes.h',
+    'backends/x11/cm/meta-renderer-x11-cm.c',
+    'backends/x11/cm/meta-renderer-x11-cm.h',
+    'backends/x11/meta-backend-x11.c',
+    'backends/x11/meta-backend-x11.h',
+    'backends/x11/meta-barrier-x11.c',
+    'backends/x11/meta-barrier-x11.h',
+    'backends/x11/meta-clutter-backend-x11.c',
+    'backends/x11/meta-clutter-backend-x11.h',
+    'backends/x11/meta-crtc-xrandr.c',
+    'backends/x11/meta-crtc-xrandr.h',
+    'backends/x11/meta-cursor-renderer-x11.c',
+    'backends/x11/meta-cursor-renderer-x11.h',
+    'backends/x11/meta-cursor-tracker-x11.c',
+    'backends/x11/meta-cursor-tracker-x11.h',
+    'backends/x11/meta-event-x11.c',
+    'backends/x11/meta-event-x11.h',
+    'backends/x11/meta-gpu-xrandr.c',
+    'backends/x11/meta-gpu-xrandr.h',
+    'backends/x11/meta-input-device-x11.c',
+    'backends/x11/meta-input-device-x11.h',
+    'backends/x11/meta-input-device-tool-x11.c',
+    'backends/x11/meta-input-device-tool-x11.h',
+    'backends/x11/meta-input-settings-x11.c',
+    'backends/x11/meta-input-settings-x11.h',
+    'backends/x11/meta-seat-x11.c',
+    'backends/x11/meta-seat-x11.h',
+    'backends/x11/meta-keymap-x11.c',
+    'backends/x11/meta-keymap-x11.h',
+    'backends/x11/meta-monitor-manager-xrandr.c',
+    'backends/x11/meta-monitor-manager-xrandr.h',
+    'backends/x11/meta-output-xrandr.c',
+    'backends/x11/meta-output-xrandr.h',
+    'backends/x11/meta-renderer-x11.c',
+    'backends/x11/meta-renderer-x11.h',
+    'backends/x11/meta-stage-x11.c',
+    'backends/x11/meta-stage-x11.h',
+    'backends/x11/meta-virtual-input-device-x11.c',
+    'backends/x11/meta-virtual-input-device-x11.h',
+    'backends/x11/meta-xkb-a11y-x11.c',
+    'backends/x11/meta-xkb-a11y-x11.h',
+    'backends/x11/nested/meta-backend-x11-nested.c',
+    'backends/x11/nested/meta-backend-x11-nested.h',
+    'backends/x11/nested/meta-cursor-renderer-x11-nested.c',
+    'backends/x11/nested/meta-cursor-renderer-x11-nested.h',
+    'backends/x11/nested/meta-stage-x11-nested.c',
+    'backends/x11/nested/meta-stage-x11-nested.h',
+    'backends/x11/nested/meta-renderer-x11-nested.c',
+    'backends/x11/nested/meta-renderer-x11-nested.h',
+  ]
+endif
+
+if have_x11_client
+  mutter_sources += [
+    'compositor/meta-compositor-x11.c',
+    'compositor/meta-compositor-x11.h',
+    'compositor/meta-surface-actor-x11.c',
+    'compositor/meta-surface-actor-x11.h',
+    'compositor/meta-sync-ring.c',
+    'compositor/meta-sync-ring.h',
+    'compositor/meta-window-actor-x11.c',
+    'compositor/meta-window-actor-x11.h',
+    'ui/frames.c',
+    'ui/frames.h',
+    'ui/theme.c',
+    'ui/theme-private.h',
+    'ui/ui.c',
+    'ui/ui.h',
+    'x11/atomnames.h',
+    'x11/events.c',
+    'x11/events.h',
+    'x11/group.c',
+    'x11/group-private.h',
+    'x11/group-props.c',
+    'x11/group-props.h',
+    'x11/iconcache.c',
+    'x11/iconcache.h',
+    'x11/meta-selection-source-x11.c',
+    'x11/meta-selection-source-x11-private.h',
+    'x11/meta-startup-notification-x11.c',
+    'x11/meta-startup-notification-x11.h',
+    'x11/meta-x11-display.c',
+    'x11/meta-x11-display-private.h',
+    'x11/meta-x11-errors.c',
+    'x11/meta-x11-selection.c',
+    'x11/meta-x11-selection-private.h',
+    'x11/meta-x11-selection-input-stream.c',
+    'x11/meta-x11-selection-input-stream-private.h',
+    'x11/meta-x11-selection-output-stream.c',
+    'x11/meta-x11-selection-output-stream-private.h',
+    'x11/meta-x11-stack.c',
+    'x11/meta-x11-stack-private.h',
+    'x11/meta-x11-window-control.c',
+    'x11/meta-x11-window-control.h',
+    'x11/mutter-Xatomtype.h',
+    'x11/session.c',
+    'x11/session.h',
+    'x11/window-props.c',
+    'x11/window-props.h',
+    'x11/window-x11.c',
+    'x11/window-x11.h',
+    'x11/window-x11-private.h',
+    'x11/xprops.c',
+    'x11/xprops.h',
+  ]
+endif
+
+
 if have_egl_device
   mutter_sources += [
     'backends/native/meta-render-device-egl-stream.c',
@@ -636,18 +647,23 @@ if have_wayland
     'wayland/meta-wayland-xdg-shell.h',
     'wayland/meta-window-wayland.c',
     'wayland/meta-window-wayland.h',
-    'wayland/meta-window-xwayland.c',
-    'wayland/meta-window-xwayland.h',
-    'wayland/meta-xwayland.c',
-    'wayland/meta-xwayland-grab-keyboard.c',
-    'wayland/meta-xwayland-grab-keyboard.h',
-    'wayland/meta-xwayland.h',
-    'wayland/meta-xwayland-private.h',
-    'wayland/meta-xwayland-dnd.c',
-    'wayland/meta-xwayland-dnd-private.h',
-    'wayland/meta-xwayland-surface.c',
-    'wayland/meta-xwayland-surface.h',
   ]
+
+  if have_xwayland
+    mutter_sources += [
+      'wayland/meta-window-xwayland.c',
+      'wayland/meta-window-xwayland.h',
+      'wayland/meta-xwayland.c',
+      'wayland/meta-xwayland-grab-keyboard.c',
+      'wayland/meta-xwayland-grab-keyboard.h',
+      'wayland/meta-xwayland.h',
+      'wayland/meta-xwayland-private.h',
+      'wayland/meta-xwayland-dnd.c',
+      'wayland/meta-xwayland-dnd-private.h',
+      'wayland/meta-xwayland-surface.c',
+      'wayland/meta-xwayland-surface.h',
+    ]
+  endif
 endif
 
 if have_native_backend
@@ -1047,25 +1063,26 @@ executable('mutter',
   install_dir: bindir,
   install: true,
 )
-
-executable('mutter-restart-helper',
-  sources: [
-    files('core/restart-helper.c'),
-  ],
-  include_directories: [
-    top_includepath,
-  ],
-  c_args: [
-    mutter_c_args,
-    '-DG_LOG_DOMAIN="mutter-restart-helper"',
-  ],
-  dependencies: [
-    x11_dep,
-    xcomposite_dep,
-  ],
-  install_dir: libexecdir,
-  install: true,
-)
+if have_x11
+  executable('mutter-restart-helper',
+    sources: [
+      files('core/restart-helper.c'),
+    ],
+    include_directories: [
+      top_includepath,
+    ],
+    c_args: [
+      mutter_c_args,
+      '-DG_LOG_DOMAIN="mutter-restart-helper"',
+    ],
+    dependencies: [
+      x11_dep,
+      xcomposite_dep,
+    ],
+    install_dir: libexecdir,
+    install: true,
+  )
+endif
 
 if have_introspection
   mutter_introspected_sources = []