onscreen/native: Use EGLSyncs instead of cogl_framebuffer_finish

cogl_framebuffer_finish can result in a CPU-side stall because it waits for
the primary GPU to flush and execute all commands that were queued before
that. By using a GPU-side EGLSync we can let the primary GPU inform us when
it is done with the queued commands instead. We then create another EGLSync
on the secondary GPU using the same fd so the primary GPU effectively
signals the secondary GPU when it is done rendering, causing the latter
to wait for the former before copying part of the frames it needs for
monitors attached to it directly.

This solves the corruption that cogl_framebuffer_finish also solved, but
without needing a CPU-side stall.

Signed-off-by: Mingi Sung <sungmg@saltyming.net>

src/backends/meta-egl.c

@@ -44,6 +44,11 @@ struct _MetaEgl
   PFNEGLCREATEIMAGEKHRPROC eglCreateImageKHR;
   PFNEGLDESTROYIMAGEKHRPROC eglDestroyImageKHR;
 
+  PFNEGLCREATESYNCPROC eglCreateSync;
+  PFNEGLDESTROYSYNCPROC eglDestroySync;
+  PFNEGLWAITSYNCPROC eglWaitSync;
+  PFNEGLDUPNATIVEFENCEFDANDROIDPROC eglDupNativeFenceFDANDROID;
+
   PFNEGLBINDWAYLANDDISPLAYWL eglBindWaylandDisplayWL;
   PFNEGLQUERYWAYLANDBUFFERWL eglQueryWaylandBufferWL;
 
@@ -1162,6 +1167,90 @@ meta_egl_query_display_attrib (MetaEgl     *egl,
   return TRUE;
 }
 
+gboolean
+meta_egl_create_sync (MetaEgl           *egl,
+                      EGLDisplay         display,
+                      EGLenum            type,
+                      const EGLAttrib   *attrib_list,
+                      EGLSync           *egl_sync,
+                      GError           **error)
+{
+  if (!is_egl_proc_valid (egl->eglCreateSync, error))
+    return FALSE;
+
+  EGLSync sync;
+
+  sync = egl->eglCreateSync (display, type, attrib_list);
+
+  if (sync == EGL_NO_SYNC)
+    {
+      set_egl_error (error);
+      return FALSE;
+    }
+
+  *egl_sync = sync;
+
+  return TRUE;
+}
+
+gboolean
+meta_egl_destroy_sync (MetaEgl     *egl,
+                       EGLDisplay   display,
+                       EGLSync      sync,
+                       GError     **error)
+{
+  if (!is_egl_proc_valid (egl->eglDestroySync, error))
+    return FALSE;
+
+  if (!egl->eglDestroySync (display, sync))
+    {
+      set_egl_error (error);
+      return FALSE;
+    }
+
+  return TRUE;
+}
+
+gboolean
+meta_egl_wait_sync (MetaEgl     *egl,
+                    EGLDisplay   display,
+                    EGLSync      sync,
+                    EGLint       flags,
+                    GError     **error)
+{
+  if (!is_egl_proc_valid (egl->eglWaitSync, error))
+    return FALSE;
+
+  if (!egl->eglWaitSync (display, sync, flags))
+    {
+      set_egl_error (error);
+      return FALSE;
+    }
+
+  return TRUE;
+}
+
+EGLint
+meta_egl_duplicate_native_fence_fd (MetaEgl     *egl,
+                                    EGLDisplay   display,
+                                    EGLSync      sync,
+                                    GError     **error)
+{
+  if (!is_egl_proc_valid (egl->eglDupNativeFenceFDANDROID, error))
+    return EGL_NO_NATIVE_FENCE_FD_ANDROID;
+
+  EGLint fd = EGL_NO_NATIVE_FENCE_FD_ANDROID;
+
+  fd = egl->eglDupNativeFenceFDANDROID (display, sync);
+
+  if (fd == EGL_NO_NATIVE_FENCE_FD_ANDROID)
+    {
+      set_egl_error (error);
+    }
+
+  return fd;
+}
+
 #define GET_EGL_PROC_ADDR(proc) \
   egl->proc = (void *) eglGetProcAddress (#proc);
 
@@ -1175,6 +1264,11 @@ meta_egl_constructed (GObject *object)
   GET_EGL_PROC_ADDR (eglCreateImageKHR);
   GET_EGL_PROC_ADDR (eglDestroyImageKHR);
 
+  GET_EGL_PROC_ADDR (eglCreateSync);
+  GET_EGL_PROC_ADDR (eglDestroySync);
+  GET_EGL_PROC_ADDR (eglWaitSync);
+  GET_EGL_PROC_ADDR (eglDupNativeFenceFDANDROID);
+
   GET_EGL_PROC_ADDR (eglBindWaylandDisplayWL);
   GET_EGL_PROC_ADDR (eglQueryWaylandBufferWL);
 

src/backends/meta-egl.h

@@ -276,3 +276,26 @@ gboolean meta_egl_query_display_attrib (MetaEgl     *egl,
                                         EGLint       attribute,
                                         EGLAttrib   *value,
                                         GError     **error);
+
+gboolean meta_egl_create_sync (MetaEgl           *egl,
+                               EGLDisplay         display,
+                               EGLenum            type,
+                               const EGLAttrib   *attrib_list,
+                               EGLSync           *egl_sync,
+                               GError           **error);
+
+gboolean meta_egl_destroy_sync (MetaEgl     *egl,
+                                EGLDisplay   display,
+                                EGLSync      sync,
+                                GError     **error);
+
+gboolean meta_egl_wait_sync (MetaEgl     *egl,
+                             EGLDisplay   display,
+                             EGLSync      sync,
+                             EGLint       flags,
+                             GError     **error);
+
+EGLint meta_egl_duplicate_native_fence_fd (MetaEgl     *egl,
+                                           EGLDisplay   display,
+                                           EGLSync      sync,
+                                           GError     **error);

src/backends/native/meta-onscreen-native.c

@@ -29,6 +29,7 @@
 
 #include "backends/native/meta-onscreen-native.h"
 
+#include <glib/gstdio.h>
 #include <drm_fourcc.h>
 
 #include "backends/meta-egl-ext.h"
@@ -844,19 +845,51 @@ copy_shared_framebuffer_gpu (CoglOnscreen                         *onscreen,
   CoglFramebuffer *framebuffer = COGL_FRAMEBUFFER (onscreen);
   CoglContext *cogl_context = cogl_framebuffer_get_context (framebuffer);
   CoglDisplay *cogl_display = cogl_context_get_display (cogl_context);
+  CoglRendererEGL *cogl_renderer_egl = cogl_context->display->renderer->winsys;
   MetaRenderDevice *render_device;
-  EGLDisplay egl_display;
+  EGLDisplay egl_display = NULL;
   gboolean use_modifiers;
   MetaDeviceFile *device_file;
   MetaDrmBufferFlags flags;
   MetaDrmBufferGbm *buffer_gbm = NULL;
   struct gbm_bo *bo;
+  EGLSync primary_gpu_egl_sync = EGL_NO_SYNC;
+  EGLSync secondary_gpu_egl_sync = EGL_NO_SYNC;
+  g_autofd int primary_gpu_sync_fence = EGL_NO_NATIVE_FENCE_FD_ANDROID;
 
   COGL_TRACE_BEGIN_SCOPED (CopySharedFramebufferSecondaryGpu,
                            "copy_shared_framebuffer_gpu()");
 
   if (renderer_gpu_data->secondary.needs_explicit_sync)
-    cogl_framebuffer_finish (COGL_FRAMEBUFFER (onscreen));
+    {
+      if (!meta_egl_create_sync (egl,
+                                cogl_renderer_egl->edpy,
+                                EGL_SYNC_NATIVE_FENCE_ANDROID,
+                                NULL,
+                                &primary_gpu_egl_sync,
+                                error))
+       {
+         g_prefix_error (error, "Failed to create EGLSync on primary GPU: ");
+         return NULL;
+       }
+
+      // According to the EGL_KHR_fence_sync specification we must ensure
+      // the fence command is flushed in this context to be able to await it
+      // in another (secondary GPU context) or we risk waiting indefinitely.
+      cogl_framebuffer_flush (COGL_FRAMEBUFFER (onscreen));
+
+      primary_gpu_sync_fence =
+        meta_egl_duplicate_native_fence_fd (egl,
+                                            cogl_renderer_egl->edpy,
+                                            primary_gpu_egl_sync,
+                                            error);
+
+      if (primary_gpu_sync_fence == EGL_NO_NATIVE_FENCE_FD_ANDROID)
+        {
+          g_prefix_error (error, "Failed to duplicate EGLSync FD on primary GPU: ");
+          goto done;
+        }
+    }
 
   render_device = renderer_gpu_data->render_device;
   egl_display = meta_render_device_get_egl_display (render_device);
@@ -872,6 +905,40 @@ copy_shared_framebuffer_gpu (CoglOnscreen                         *onscreen,
       goto done;
     }
 
+  if (primary_gpu_sync_fence != EGL_NO_NATIVE_FENCE_FD_ANDROID)
+    {
+      EGLAttrib attribs[3];
+
+      attribs[0] = EGL_SYNC_NATIVE_FENCE_FD_ANDROID;
+      attribs[1] = primary_gpu_sync_fence;
+      attribs[2] = EGL_NONE;
+
+      if (!meta_egl_create_sync (egl,
+                                egl_display,
+                                EGL_SYNC_NATIVE_FENCE_ANDROID,
+                                attribs,
+                                &secondary_gpu_egl_sync,
+                                error))
+        {
+          g_prefix_error (error, "Failed to create EGLSync on secondary GPU: ");
+          goto done;
+        }
+
+      // eglCreateSync takes ownership of an existing fd that is passed, so
+      // don't try to clean it up twice.
+      primary_gpu_sync_fence = EGL_NO_NATIVE_FENCE_FD_ANDROID;
+
+      if (!meta_egl_wait_sync (egl,
+                               egl_display,
+                               secondary_gpu_egl_sync,
+                               0,
+                               error))
+        {
+          g_prefix_error (error, "Failed to wait for EGLSync on secondary GPU: ");
+          goto done;
+        }
+    }
+
   buffer_gbm = META_DRM_BUFFER_GBM (primary_gpu_fb);
   bo = meta_drm_buffer_gbm_get_bo (buffer_gbm);
   if (!meta_renderer_native_gles3_blit_shared_bo (egl,
@@ -921,6 +988,20 @@ copy_shared_framebuffer_gpu (CoglOnscreen                         *onscreen,
 done:
   _cogl_winsys_egl_ensure_current (cogl_display);
 
+  if (primary_gpu_egl_sync != EGL_NO_SYNC &&
+      !meta_egl_destroy_sync (egl,
+                              cogl_renderer_egl->edpy,
+                              primary_gpu_egl_sync,
+                              error))
+    g_prefix_error (error, "Failed to destroy primary GPU EGLSync: ");
+
+  if (secondary_gpu_egl_sync != EGL_NO_SYNC &&
+      !meta_egl_destroy_sync (egl,
+                              egl_display,
+                              secondary_gpu_egl_sync,
+                              error))
+    g_prefix_error (error, "Failed to destroy secondary GPU EGLSync: ");
+
   return buffer_gbm ? META_DRM_BUFFER (buffer_gbm) : NULL;
 }