From ef83498aefb91168292439c88e01723880eb6707 Mon Sep 17 00:00:00 2001
From: Niklas Haas <git@haasn.dev>
Date: Fri, 11 Feb 2022 05:04:27 +0100
Subject: vo_gpu_next: don't crash on negative plane strides

This is an annoying special case only really needed because of the
`vflip` filter. mpv handles this by directly adjusting the plane
transform. The libplacebo API, unfortunately, does not allow passing the
required information for this to work smoothly.

Long-term I plan on adding support for plane flipping in libplacebo
directly, but for the meantime, we will have to work-around it by moving
the flipping to the whole-image `crop` instead. Not an ideal solution
but better than crashing.

Fixes #9855
---
 video/out/vo_gpu_next.c | 39 ++++++++++++++++++++++++++++++++++-----
 1 file changed, 34 insertions(+), 5 deletions(-)

(limited to 'video/out')

diff --git a/video/out/vo_gpu_next.c b/video/out/vo_gpu_next.c
index 599ea758d4..da613f498b 100644
--- a/video/out/vo_gpu_next.c
+++ b/video/out/vo_gpu_next.c
@@ -481,17 +481,36 @@ static bool map_frame(pl_gpu gpu, pl_tex *tex, const struct pl_source_frame *src
 
     enum pl_chroma_location chroma = mp_chroma_to_pl(par->chroma_location);
     int planes = plane_data_from_imgfmt(data, &frame->repr.bits, mpi->imgfmt);
+    bool img_vflipped = false;
     for (int n = 0; n < planes; n++) {
         data[n].width = mp_image_plane_w(mpi, n);
         data[n].height = mp_image_plane_h(mpi, n);
-        data[n].row_stride = mpi->stride[n];
-        data[n].pixels = mpi->planes[n];
+        bool vflipped = mpi->stride[n] < 0;
+        if (vflipped) {
+            int h = mp_image_plane_h(mpi, n);
+            data[n].pixels = mpi->planes[n] + (h - 1) * mpi->stride[n];
+            data[n].row_stride = -mpi->stride[n];
+        } else {
+            data[n].pixels = mpi->planes[n];
+            data[n].row_stride = mpi->stride[n];
+        }
+
+        // libplacebo can't deal with images that have partially flipped
+        // textures. We could work around this by blitting them to fresh
+        // (unflipped) textures, but it's an unlikely enough case to warrant
+        // just erroring out instead. (Usually, all planes will be flipped or
+        // unflipped simultaneously, e.g. by the `vflip` filter)
+        if (n > 0 && img_vflipped != vflipped) {
+            MP_ERR(vo, "Inconsistently flipped planes!\n");
+            return false;
+        }
+        img_vflipped = vflipped;
 
         pl_buf buf = get_dr_buf(mpi);
         if (buf) {
-            data[n].pixels = NULL;
             data[n].buf = buf;
-            data[n].buf_offset = mpi->planes[n] - buf->data;
+            data[n].buf_offset = (uint8_t *) data[n].pixels - buf->data;
+            data[n].pixels = NULL;
         } else if (gpu->limits.callbacks) {
             data[n].callback = talloc_free;
             data[n].priv = mp_image_new_ref(mpi);
@@ -504,8 +523,11 @@ static bool map_frame(pl_gpu gpu, pl_tex *tex, const struct pl_source_frame *src
             return false;
         }
 
-        if (mpi->fmt.xs[n] || mpi->fmt.ys[n])
+        if (mpi->fmt.xs[n] || mpi->fmt.ys[n]) {
             pl_chroma_location_offset(chroma, &plane->shift_x, &plane->shift_y);
+            if (vflipped)
+                plane->shift_y = -plane->shift_y;
+        }
     }
 
 #ifdef PL_HAVE_LAV_DOLBY_VISION
@@ -668,6 +690,13 @@ static void apply_crop(struct pl_frame *frame, struct mp_rect crop,
         frame->crop.y0 = height - frame->crop.y0;
         frame->crop.y1 = height - frame->crop.y1;
     }
+
+    struct mp_image *mpi = frame->user_data;
+    if (mpi && mpi->stride[0] < 0) {
+        // Adjust for vertically flipped planes
+        frame->crop.y0 = height - frame->crop.y0;
+        frame->crop.y1 = height - frame->crop.y1;
+    }
 }
 
 static void draw_frame(struct vo *vo, struct vo_frame *frame)
-- 
cgit v1.2.3