hwdec_cuda: remove usage of deprecated libplacebo sync API

The libplacebo sync abstraction is deprecated and we should be using
the more explicit Vulkan semaphore helpers instead. This means that
more of the book keeping moves to our side, but it's not too bad.

There are two main things going on here:

1. After a lot of back and forth, I decided to write the new code with
   timeline semaphores to streamline things, and that also means all
   the variables are separate - which makes the #ifdefs easier to read.
   Which is important because:

2. While pl_sync owned the exported fd/handle, pl_vulkan_sem does not,
   so we are responsible for managing them. That means reversing the
   previous logic - we now can pass an original fd to CUDA and never
   think about it again, while we have to clean up a Win32 Handle
   because CUDA will not take ownership.

1 files changed, 105 insertions, 1 deletions

diff --git a/video/out/hwdec/hwdec_cuda_vk.c b/video/out/hwdec/hwdec_cuda_vk.c
index e5929b8be6..49f266f72e 100644
--- a/video/out/hwdec/hwdec_cuda_vk.c
+++ b/video/out/hwdec/hwdec_cuda_vk.c
@@ -23,6 +23,9 @@
 
 #include <libavutil/hwcontext.h>
 #include <libavutil/hwcontext_cuda.h>
+#if HAVE_LIBPLACEBO_NEXT
+#include <libplacebo/vulkan.h>
+#endif
 #include <unistd.h>
 
 #if HAVE_WIN32_DESKTOP
@@ -39,10 +42,16 @@ struct ext_vk {
     CUmipmappedArray mma;
 
     pl_tex pltex;
+#if HAVE_LIBPLACEBO_NEXT
+    pl_vulkan_sem vk_sem;
+    union pl_handle sem_handle;
+    CUexternalSemaphore cuda_sem;
+#else
     pl_sync sync;
 
     CUexternalSemaphore ss;
     CUexternalSemaphore ws;
+#endif
 };
 
 static bool cuda_ext_vk_init(struct ra_hwdec_mapper *mapper,
@@ -51,7 +60,10 @@ static bool cuda_ext_vk_init(struct ra_hwdec_mapper *mapper,
     struct cuda_hw_priv *p_owner = mapper->owner->priv;
     struct cuda_mapper_priv *p = mapper->priv;
     CudaFunctions *cu = p_owner->cu;
-    int mem_fd = -1, wait_fd = -1, signal_fd = -1;
+    int mem_fd = -1;
+#if !HAVE_LIBPLACEBO_NEXT
+    int wait_fd = -1, signal_fd = -1;
+#endif
     int ret = 0;
 
     struct ext_vk *evk = talloc_ptrtype(NULL, evk);
@@ -139,6 +151,32 @@ static bool cuda_ext_vk_init(struct ra_hwdec_mapper *mapper,
     if (ret < 0)
         goto error;
 
+#if HAVE_LIBPLACEBO_NEXT
+    evk->vk_sem.sem = pl_vulkan_sem_create(gpu, pl_vulkan_sem_params(
+        .type = VK_SEMAPHORE_TYPE_TIMELINE,
+        .export_handle = HANDLE_TYPE,
+        .out_handle = &(evk->sem_handle),
+    ));
+    if (evk->vk_sem.sem == VK_NULL_HANDLE) {
+         ret = -1;
+         goto error;
+     }
+     // The returned FD or Handle is owned by the caller (us).
+
+    CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC w_desc = {
+#if HAVE_WIN32_DESKTOP
+        .type = CU_EXTERNAL_SEMAPHORE_HANDLE_TYPE_TIMELINE_SEMAPHORE_WIN32,
+        .handle.win32.handle = evk->sem_handle.handle,
+#else
+        .type = CU_EXTERNAL_SEMAPHORE_HANDLE_TYPE_TIMELINE_SEMAPHORE_FD,
+        .handle.fd = evk->sem_handle.fd,
+#endif
+    };
+    ret = CHECK_CU(cu->cuImportExternalSemaphore(&evk->cuda_sem, &w_desc));
+    if (ret < 0)
+        goto error;
+    // CUDA takes ownership of an imported FD *but not* an imported Handle.
+#else
     evk->sync = pl_sync_create(gpu, HANDLE_TYPE);
     if (!evk->sync) {
         ret = -1;
@@ -178,6 +216,7 @@ static bool cuda_ext_vk_init(struct ra_hwdec_mapper *mapper,
     if (ret < 0)
         goto error;
     signal_fd = -1;
+#endif
 
     return true;
 
@@ -185,10 +224,20 @@ error:
     MP_ERR(mapper, "cuda_ext_vk_init failed\n");
     if (mem_fd > -1)
         close(mem_fd);
+#if HAVE_LIBPLACEBO_NEXT
+#if HAVE_WIN32_DESKTOP
+    if (evk->sem_handle.handle != NULL)
+        CloseHandle(evk->sem_handle.handle);
+#else
+    if (evk->sem_handle.fd > -1)
+        close(evk->sem_handle.fd);
+#endif
+#else
     if (wait_fd > -1)
         close(wait_fd);
     if (signal_fd > -1)
         close(signal_fd);
+#endif
     return false;
 }
 
@@ -208,6 +257,16 @@ static void cuda_ext_vk_uninit(const struct ra_hwdec_mapper *mapper, int n)
             CHECK_CU(cu->cuDestroyExternalMemory(evk->mem));
             evk->mem = 0;
         }
+#if HAVE_LIBPLACEBO_NEXT
+        if (evk->cuda_sem) {
+            CHECK_CU(cu->cuDestroyExternalSemaphore(evk->cuda_sem));
+            evk->cuda_sem = 0;
+        }
+        pl_vulkan_sem_destroy(ra_pl_get(mapper->ra), &evk->vk_sem.sem);
+#if HAVE_WIN32_DESKTOP
+        CloseHandle(evk->sem_handle.handle);
+#endif
+#else
         if (evk->ss) {
             CHECK_CU(cu->cuDestroyExternalSemaphore(evk->ss));
             evk->ss = 0;
@@ -217,6 +276,7 @@ static void cuda_ext_vk_uninit(const struct ra_hwdec_mapper *mapper, int n)
             evk->ws = 0;
         }
         pl_sync_destroy(ra_pl_get(mapper->ra), &evk->sync);
+#endif
     }
     talloc_free(evk);
 }
@@ -229,6 +289,27 @@ static bool cuda_ext_vk_wait(const struct ra_hwdec_mapper *mapper, int n)
     int ret;
     struct ext_vk *evk = p->ext[n];
 
+#if HAVE_LIBPLACEBO_NEXT
+    evk->vk_sem.value += 1;
+    ret = pl_vulkan_hold_ex(ra_pl_get(mapper->ra), pl_vulkan_hold_params(
+        .tex = evk->pltex,
+        .layout = VK_IMAGE_LAYOUT_GENERAL,
+        .qf = VK_QUEUE_FAMILY_EXTERNAL,
+        .semaphore = evk->vk_sem,
+    ));
+    if (!ret)
+        return false;
+
+    CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS wp = {
+        .params = {
+            .fence = {
+                .value = evk->vk_sem.value
+            }
+        }
+     };
+     ret = CHECK_CU(cu->cuWaitExternalSemaphoresAsync(&evk->cuda_sem,
+                                                     &wp, 1, 0));
+#else
     ret = pl_tex_export(ra_pl_get(mapper->ra),
                         evk->pltex, evk->sync);
     if (!ret)
@@ -237,6 +318,7 @@ static bool cuda_ext_vk_wait(const struct ra_hwdec_mapper *mapper, int n)
     CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS wp = { 0, };
     ret = CHECK_CU(cu->cuWaitExternalSemaphoresAsync(&evk->ws,
                                                      &wp, 1, 0));
+#endif
     return ret == 0;
 }
 
@@ -248,9 +330,31 @@ static bool cuda_ext_vk_signal(const struct ra_hwdec_mapper *mapper, int n)
     int ret;
     struct ext_vk *evk = p->ext[n];
 
+#if HAVE_LIBPLACEBO_NEXT
+    evk->vk_sem.value += 1;
+    CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS sp = {
+        .params = {
+            .fence = {
+                .value = evk->vk_sem.value
+            }
+        }
+    };
+    ret = CHECK_CU(cu->cuSignalExternalSemaphoresAsync(&evk->cuda_sem,
+                                                       &sp, 1, 0));
+    if (ret != 0)
+        return false;
+
+    pl_vulkan_release_ex(ra_pl_get(mapper->ra), pl_vulkan_release_params(
+        .tex = evk->pltex,
+        .layout = VK_IMAGE_LAYOUT_GENERAL,
+        .qf = VK_QUEUE_FAMILY_EXTERNAL,
+        .semaphore = evk->vk_sem,
+    ));
+#else
     CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS sp = { 0, };
     ret = CHECK_CU(cu->cuSignalExternalSemaphoresAsync(&evk->ss,
                                                        &sp, 1, 0));
+#endif
     return ret == 0;
 }