vo_gpu: allow invalidating FBO in renderpass_run

This is especially interesting for vulkan since it allows completely
skipping the layout transition as part of the renderpass. Unfortunately,
that also means it needs to be put into renderpass_params, as opposed to
renderpass_run_params (unlike #4777).

Closes #4777.

3 files changed, 22 insertions, 5 deletions

diff --git a/video/out/gpu/ra.h b/video/out/gpu/ra.h
index 934e5db844..ffb010960a 100644
--- a/video/out/gpu/ra.h
+++ b/video/out/gpu/ra.h
@@ -285,6 +285,9 @@ struct ra_renderpass_params {
     enum ra_blend blend_src_alpha;
     enum ra_blend blend_dst_alpha;
 
+    // If true, the contents of `target` not written to will become undefined
+    bool invalidate_target;
+
     // --- type==RA_RENDERPASS_TYPE_COMPUTE only
 
     // Shader text, like vertex_shader/frag_shader.
diff --git a/video/out/opengl/ra_gl.c b/video/out/opengl/ra_gl.c
index 5b03368436..37b7285235 100644
--- a/video/out/opengl/ra_gl.c
+++ b/video/out/opengl/ra_gl.c
@@ -996,6 +996,10 @@ static void gl_renderpass_run(struct ra *ra,
         assert(params->target->params.render_dst);
         assert(params->target->params.format == pass->params.target_format);
         gl->BindFramebuffer(GL_FRAMEBUFFER, target_gl->fbo);
+        if (pass->params.invalidate_target && gl->InvalidateFramebuffer) {
+            GLenum fb = target_gl->fbo ? GL_COLOR_ATTACHMENT0 : GL_COLOR;
+            gl->InvalidateFramebuffer(GL_FRAMEBUFFER, 1, &fb);
+        }
         gl->Viewport(params->viewport.x0, params->viewport.y0,
                      mp_rect_w(params->viewport),
                      mp_rect_h(params->viewport));
diff --git a/video/out/vulkan/ra_vk.c b/video/out/vulkan/ra_vk.c
index e3b1d5aaba..42e6c2f64c 100644
--- a/video/out/vulkan/ra_vk.c
+++ b/video/out/vulkan/ra_vk.c
@@ -328,10 +328,12 @@ static void tex_barrier(struct ra *ra, struct vk_cmd *cmd, struct ra_tex *tex,
     VkEvent event = NULL;
     vk_cmd_wait(vk, cmd, &tex_vk->sig, stage, &event);
 
-    // Image barriers are redundant if there's nothing to be done
-    if (imgBarrier.oldLayout != imgBarrier.newLayout ||
-        imgBarrier.srcAccessMask != imgBarrier.dstAccessMask)
-    {
+    bool need_trans = tex_vk->current_layout != newLayout ||
+                      tex_vk->current_access != newAccess;
+
+    // Transitioning to VK_IMAGE_LAYOUT_UNDEFINED is a pseudo-operation
+    // that for us means we don't need to perform the actual transition
+    if (need_trans && newLayout != VK_IMAGE_LAYOUT_UNDEFINED) {
         if (event) {
             vkCmdWaitEvents(cmd->buf, 1, &event, tex_vk->sig_stage,
                             stage, 0, NULL, 0, NULL, 1, &imgBarrier);
@@ -1206,6 +1208,13 @@ static struct ra_renderpass *vk_renderpass_create(struct ra *ra,
         if (pass->params.enable_blend)
             loadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
 
+        // If we're invalidating the target, we don't need to load or transition
+        if (pass->params.invalidate_target) {
+            pass_vk->initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
+            pass_vk->initialAccess = 0;
+            loadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
+        }
+
         VK(vk_create_render_pass(vk->dev, params->target_format, loadOp,
                                  pass_vk->initialLayout, pass_vk->finalLayout,
                                  &pass_vk->renderPass));
@@ -1536,7 +1545,8 @@ static void vk_renderpass_run(struct ra *ra,
 
         // The renderpass expects the images to be in a certain layout
         tex_barrier(ra, cmd, tex, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT,
-                    pass_vk->initialAccess, pass_vk->initialLayout, false);
+                    pass_vk->initialAccess, pass_vk->initialLayout,
+                    pass->params.invalidate_target);
 
         VkViewport viewport = {
             .x = params->viewport.x0,