From 274e71ee8b774d6c9c69929a548a83c343202be1 Mon Sep 17 00:00:00 2001 From: wm4 Date: Mon, 12 Sep 2016 15:08:38 +0200 Subject: vo_opengl: add hw overlay support and use it for RPI This overlay support specifically skips the OpenGL rendering chain, and uses GL rendering only for OSD/subtitles. This is for devices which don't have performant GL support. hwdec_rpi.c contains code ported from vo_rpi.c. vo_rpi.c is going to be deprecated. I left in the code for uploading sw surfaces (as it might be slightly more efficient for rendering sw decoded video), although it's dead code for now. --- libmpv/opengl_cb.h | 23 +++ video/out/opengl/common.c | 1 + video/out/opengl/common.h | 1 + video/out/opengl/context_rpi.c | 2 +- video/out/opengl/hwdec.c | 4 + video/out/opengl/hwdec.h | 17 ++ video/out/opengl/hwdec_rpi.c | 399 +++++++++++++++++++++++++++++++++++++++++ video/out/opengl/video.c | 31 +++- wscript_build.py | 3 +- 9 files changed, 478 insertions(+), 3 deletions(-) create mode 100644 video/out/opengl/hwdec_rpi.c diff --git a/libmpv/opengl_cb.h b/libmpv/opengl_cb.h index 8751ca01e0..6500b4ef65 100644 --- a/libmpv/opengl_cb.h +++ b/libmpv/opengl_cb.h @@ -190,6 +190,29 @@ extern "C" { * In previous libmpv releases, this used "GL_MP_D3D_interfaces" and * "glMPGetD3DInterface". This is deprecated; use glMPGetNativeDisplay instead * (the semantics are 100% compatible). + * + * Windowing system interop on RPI + * ------------------------------- + * + * The RPI uses no proper interop, but hardware overlays instead. To place the + * overlay correctly, you can communicate the window parameters as follows to + * libmpv. gl->MPGetNativeDisplay("MPV_RPI_WINDOW") return an array of type int + * with the following 4 elements: + * 0: display number (default 0) + * 1: layer number of the GL layer - video will be placed in the layer + * directly below (default: 0) + * 2: absolute x position of the GL context (default: 0) + * 3: absolute y position of the GL context (default: 0) + * The (x,y) position must be the absolute screen pixel position of the + * top/left pixel of the dispmanx layer used for the GL context. If you render + * to a FBO, the position must be that of the final position of the FBO + * contents on screen. You can't transform or scale the video other than what + * mpv will render to the video overlay. The defaults are suitable for + * rendering the video at fullscreen. + * The parameters are checked on every draw by calling MPGetNativeDisplay and + * checking the values in the returned array for changes. The returned array + * must remain valid until the libmpv render function returns; then it can be + * deallocated by the API user. */ /** diff --git a/video/out/opengl/common.c b/video/out/opengl/common.c index dd44165b81..5762f44440 100644 --- a/video/out/opengl/common.c +++ b/video/out/opengl/common.c @@ -125,6 +125,7 @@ static const struct gl_functions gl_functions[] = { DEF_FN(LinkProgram), DEF_FN(PixelStorei), DEF_FN(ReadPixels), + DEF_FN(Scissor), DEF_FN(ShaderSource), DEF_FN(TexImage2D), DEF_FN(TexParameteri), diff --git a/video/out/opengl/common.h b/video/out/opengl/common.h index e3ebd66c41..5abe839b8d 100644 --- a/video/out/opengl/common.h +++ b/video/out/opengl/common.h @@ -120,6 +120,7 @@ struct GL { void (GLAPIENTRY *DrawArrays)(GLenum, GLint, GLsizei); GLenum (GLAPIENTRY *GetError)(void); void (GLAPIENTRY *GetTexLevelParameteriv)(GLenum, GLint, GLenum, GLint *); + void (GLAPIENTRY *Scissor)(GLint, GLint, GLsizei, GLsizei); void (GLAPIENTRY *GenBuffers)(GLsizei, GLuint *); void (GLAPIENTRY *DeleteBuffers)(GLsizei, const GLuint *); diff --git a/video/out/opengl/context_rpi.c b/video/out/opengl/context_rpi.c index c0ca73335a..5a257a741c 100644 --- a/video/out/opengl/context_rpi.c +++ b/video/out/opengl/context_rpi.c @@ -181,7 +181,7 @@ static int rpi_init(struct MPGLContext *ctx, int flags) VC_RECT_T dst = {.width = w, .height = h}; VC_RECT_T src = {.width = w << 16, .height = h << 16}; VC_DISPMANX_ALPHA_T alpha = { - .flags = DISPMANX_FLAGS_ALPHA_FIXED_ALL_PIXELS, + .flags = DISPMANX_FLAGS_ALPHA_FROM_SOURCE, .opacity = 0xFF, }; p->window = vc_dispmanx_element_add(p->update, p->display, 1, &dst, 0, diff --git a/video/out/opengl/hwdec.c b/video/out/opengl/hwdec.c index b6c2ba1070..f5943aa61b 100644 --- a/video/out/opengl/hwdec.c +++ b/video/out/opengl/hwdec.c @@ -34,6 +34,7 @@ extern const struct gl_hwdec_driver gl_hwdec_d3d11eglrgb; extern const struct gl_hwdec_driver gl_hwdec_dxva2gldx; extern const struct gl_hwdec_driver gl_hwdec_dxva2; extern const struct gl_hwdec_driver gl_hwdec_cuda; +extern const struct gl_hwdec_driver gl_hwdec_rpi_overlay; static const struct gl_hwdec_driver *const mpgl_hwdec_drivers[] = { #if HAVE_VAAPI_EGL @@ -61,6 +62,9 @@ static const struct gl_hwdec_driver *const mpgl_hwdec_drivers[] = { #endif #if HAVE_CUDA_GL &gl_hwdec_cuda, +#endif +#if HAVE_RPI + &gl_hwdec_rpi_overlay, #endif NULL }; diff --git a/video/out/opengl/hwdec.h b/video/out/opengl/hwdec.h index 29ccd18a42..ce59b67e58 100644 --- a/video/out/opengl/hwdec.h +++ b/video/out/opengl/hwdec.h @@ -16,6 +16,8 @@ struct gl_hwdec { void *priv; // For working around the vdpau vs. vaapi mess. bool probing; + // Used in overlay mode only. + float overlay_colorkey[4]; }; struct gl_hwdec_plane { @@ -53,6 +55,21 @@ struct gl_hwdec_driver { void (*unmap)(struct gl_hwdec *hw); void (*destroy)(struct gl_hwdec *hw); + + // The following functions provide an alternative API. Each gl_hwdec_driver + // must have either map_frame or overlay_frame set (not both or none), and + // if overlay_frame is set, it operates in overlay mode. In this mode, + // OSD etc. is rendered via OpenGL, but the video is rendered as a separate + // layer below it. + // Non-overlay mode is strictly preferred, so try not to use overlay mode. + + // Set the given frame as overlay, replacing the previous one. + // hw_image==NULL is passed to clear the overlay. + int (*overlay_frame)(struct gl_hwdec *hw, struct mp_image *hw_image); + + // Move overlay position within the "window". + void (*overlay_adjust)(struct gl_hwdec *hw, int w, int h, + struct mp_rect *src, struct mp_rect *dst); }; struct gl_hwdec *gl_hwdec_load_api(struct mp_log *log, GL *gl, diff --git a/video/out/opengl/hwdec_rpi.c b/video/out/opengl/hwdec_rpi.c new file mode 100644 index 0000000000..4cde8c5fbb --- /dev/null +++ b/video/out/opengl/hwdec_rpi.c @@ -0,0 +1,399 @@ +/* + * This file is part of mpv. + * + * mpv is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * mpv is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with mpv. If not, see . + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +#include "common/common.h" +#include "common/msg.h" +#include "video/mp_image.h" + +#include "hwdec.h" +#include "common.h" + +struct priv { + struct mp_log *log; + struct mp_vaapi_ctx *ctx; + + struct mp_image_params params; + + MMAL_COMPONENT_T *renderer; + bool renderer_enabled; + + // for RAM input + MMAL_POOL_T *swpool; + + struct mp_image *current_frame; + + int w, h; + struct mp_rect src, dst; + int cur_window[4]; // raw user params +}; + +// Magic alignments (in pixels) expected by the MMAL internals. +#define ALIGN_W 32 +#define ALIGN_H 16 + +// Make mpi point to buffer, assuming MMAL_ENCODING_I420. +// buffer can be NULL. +// Return the required buffer space. +static size_t layout_buffer(struct mp_image *mpi, MMAL_BUFFER_HEADER_T *buffer, + struct mp_image_params *params) +{ + assert(params->imgfmt == IMGFMT_420P); + mp_image_set_params(mpi, params); + int w = MP_ALIGN_UP(params->w, ALIGN_W); + int h = MP_ALIGN_UP(params->h, ALIGN_H); + uint8_t *cur = buffer ? buffer->data : NULL; + size_t size = 0; + for (int i = 0; i < 3; i++) { + int div = i ? 2 : 1; + mpi->planes[i] = cur; + mpi->stride[i] = w / div; + size_t plane_size = h / div * mpi->stride[i]; + if (cur) + cur += plane_size; + size += plane_size; + } + return size; +} + +static MMAL_FOURCC_T map_csp(enum mp_csp csp) +{ + switch (csp) { + case MP_CSP_BT_601: return MMAL_COLOR_SPACE_ITUR_BT601; + case MP_CSP_BT_709: return MMAL_COLOR_SPACE_ITUR_BT709; + case MP_CSP_SMPTE_240M: return MMAL_COLOR_SPACE_SMPTE240M; + default: return MMAL_COLOR_SPACE_UNKNOWN; + } +} + +static void control_port_cb(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer) +{ + mmal_buffer_header_release(buffer); +} + +static void input_port_cb(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer) +{ + struct mp_image *mpi = buffer->user_data; + talloc_free(mpi); +} + +static void disable_renderer(struct gl_hwdec *hw) +{ + struct priv *p = hw->priv; + + if (p->renderer_enabled) { + mmal_port_disable(p->renderer->control); + mmal_port_disable(p->renderer->input[0]); + + mmal_port_flush(p->renderer->control); + mmal_port_flush(p->renderer->input[0]); + + mmal_component_disable(p->renderer); + } + mmal_pool_destroy(p->swpool); + p->swpool = NULL; + p->renderer_enabled = false; +} + +// check_window_only: assume params and dst/src rc are unchanged +static void update_overlay(struct gl_hwdec *hw, bool check_window_only) +{ + struct priv *p = hw->priv; + GL *gl = hw->gl; + MMAL_PORT_T *input = p->renderer->input[0]; + struct mp_rect src = p->src; + struct mp_rect dst = p->dst; + + if (!p->w || !p->h) + return; + + int defs[4] = {0, 0, 0, 0}; + int *z = + gl->MPGetNativeDisplay ? gl->MPGetNativeDisplay("MPV_RPI_WINDOW") : defs; + + // As documented in the libmpv openglcb headers. + int display = z[0]; + int layer = z[1]; + int x = z[2]; + int y = z[3]; + + if (check_window_only && memcmp(z, p->cur_window, sizeof(p->cur_window)) == 0) + return; + + memcpy(p->cur_window, z, sizeof(p->cur_window)); + + int rotate[] = {MMAL_DISPLAY_ROT0, + MMAL_DISPLAY_ROT90, + MMAL_DISPLAY_ROT180, + MMAL_DISPLAY_ROT270}; + + int src_w = src.x1 - src.x0, src_h = src.y1 - src.y0, + dst_w = dst.x1 - dst.x0, dst_h = dst.y1 - dst.y0; + int p_x, p_y; + av_reduce(&p_x, &p_y, dst_w * src_h, src_w * dst_h, 16000); + MMAL_DISPLAYREGION_T dr = { + .hdr = { .id = MMAL_PARAMETER_DISPLAYREGION, + .size = sizeof(MMAL_DISPLAYREGION_T), }, + .src_rect = { .x = src.x0, .y = src.y0, + .width = src_w, .height = src_h }, + .dest_rect = { .x = dst.x0 + x, .y = dst.y0 + y, + .width = dst_w, .height = dst_h }, + .layer = layer - 1, // under the GL layer + .display_num = display, + .pixel_x = p_x, + .pixel_y = p_y, + .transform = rotate[p->params.rotate / 90], + .fullscreen = 0, + .set = MMAL_DISPLAY_SET_SRC_RECT | MMAL_DISPLAY_SET_DEST_RECT | + MMAL_DISPLAY_SET_LAYER | MMAL_DISPLAY_SET_NUM | + MMAL_DISPLAY_SET_PIXEL | MMAL_DISPLAY_SET_TRANSFORM | + MMAL_DISPLAY_SET_FULLSCREEN, + }; + + if (p->params.rotate % 180 == 90) { + MPSWAP(int, dr.src_rect.x, dr.src_rect.y); + MPSWAP(int, dr.src_rect.width, dr.src_rect.height); + } + + if (mmal_port_parameter_set(input, &dr.hdr)) + MP_WARN(p, "could not set video rectangle\n"); +} + +static int enable_renderer(struct gl_hwdec *hw) +{ + struct priv *p = hw->priv; + MMAL_PORT_T *input = p->renderer->input[0]; + struct mp_image_params *params = &p->params; + + if (p->renderer_enabled) + return 0; + + if (!params->imgfmt) + return -1; + + bool opaque = params->imgfmt == IMGFMT_MMAL; + + input->format->encoding = opaque ? MMAL_ENCODING_OPAQUE : MMAL_ENCODING_I420; + input->format->es->video.width = MP_ALIGN_UP(params->w, ALIGN_W); + input->format->es->video.height = MP_ALIGN_UP(params->h, ALIGN_H); + input->format->es->video.crop = (MMAL_RECT_T){0, 0, params->w, params->h}; + input->format->es->video.par = (MMAL_RATIONAL_T){params->p_w, params->p_h}; + input->format->es->video.color_space = map_csp(params->color.space); + + if (mmal_port_format_commit(input)) + return -1; + + input->buffer_num = MPMAX(input->buffer_num_min, + input->buffer_num_recommended) + 3; + input->buffer_size = MPMAX(input->buffer_size_min, + input->buffer_size_recommended); + + if (!opaque) { + size_t size = layout_buffer(&(struct mp_image){0}, NULL, params); + if (input->buffer_size != size) { + MP_FATAL(hw, "We disagree with MMAL about buffer sizes.\n"); + return -1; + } + + p->swpool = mmal_pool_create(input->buffer_num, input->buffer_size); + if (!p->swpool) { + MP_FATAL(hw, "Could not allocate buffer pool.\n"); + return -1; + } + } + + update_overlay(hw, false); + + p->renderer_enabled = true; + + if (mmal_port_enable(p->renderer->control, control_port_cb)) + return -1; + + if (mmal_port_enable(input, input_port_cb)) + return -1; + + if (mmal_component_enable(p->renderer)) { + MP_FATAL(hw, "Failed to enable video renderer.\n"); + return -1; + } + + return 0; +} + +static void overlay_adjust(struct gl_hwdec *hw, int w, int h, + struct mp_rect *src, struct mp_rect *dst) +{ + struct priv *p = hw->priv; + + p->w = w; + p->h = h; + p->src = *src; + p->dst = *dst; + + update_overlay(hw, false); +} + +static int reinit(struct gl_hwdec *hw, struct mp_image_params *params) +{ + struct priv *p = hw->priv; + + p->params = *params; + + *params = (struct mp_image_params){0}; + + disable_renderer(hw); + + if (enable_renderer(hw) < 0) + return -1; + + return 0; +} + +static void free_mmal_buffer(void *arg) +{ + MMAL_BUFFER_HEADER_T *buffer = arg; + mmal_buffer_header_release(buffer); +} + +// currently dead code; for a force-overlay mode +static struct mp_image *upload(struct gl_hwdec *hw, struct mp_image *hw_image) +{ + struct priv *p = hw->priv; + + MMAL_BUFFER_HEADER_T *buffer = mmal_queue_wait(p->swpool->queue); + if (!buffer) { + MP_ERR(hw, "Can't allocate buffer.\n"); + return NULL; + } + mmal_buffer_header_reset(buffer); + + struct mp_image *new_ref = mp_image_new_custom_ref(NULL, buffer, + free_mmal_buffer); + if (!new_ref) { + mmal_buffer_header_release(buffer); + MP_ERR(hw, "Out of memory.\n"); + return NULL; + } + + mp_image_setfmt(new_ref, IMGFMT_MMAL); + new_ref->planes[3] = (void *)buffer; + + struct mp_image dmpi = {0}; + buffer->length = layout_buffer(&dmpi, buffer, &p->params); + mp_image_copy(&dmpi, hw_image); + + return new_ref; +} + +static int overlay_frame(struct gl_hwdec *hw, struct mp_image *hw_image) +{ + struct priv *p = hw->priv; + + mp_image_unrefp(&p->current_frame); + + if (!hw_image) { + disable_renderer(hw); + return 0; + } + + if (enable_renderer(hw) < 0) + return -1; + + update_overlay(hw, true); + + struct mp_image *mpi = mp_image_new_ref(hw_image); + if (hw_image->imgfmt != IMGFMT_MMAL) + mpi = upload(hw, hw_image); + + if (!mpi) { + disable_renderer(hw); + return -1; + } + + MMAL_BUFFER_HEADER_T *ref = (void *)mpi->planes[3]; + + // Assume this field is free for use by us. + ref->user_data = mpi; + + if (mmal_port_send_buffer(p->renderer->input[0], ref)) { + MP_ERR(hw, "could not queue picture!\n"); + talloc_free(mpi); + return -1; + } + + return 0; +} + +static void destroy(struct gl_hwdec *hw) +{ + struct priv *p = hw->priv; + + disable_renderer(hw); + + if (p->renderer) + mmal_component_release(p->renderer); + + mmal_vc_deinit(); +} + +static int create(struct gl_hwdec *hw) +{ + struct priv *p = talloc_zero(hw, struct priv); + hw->priv = p; + p->log = hw->log; + + bcm_host_init(); + + if (mmal_vc_init()) { + MP_FATAL(hw, "Could not initialize MMAL.\n"); + return -1; + } + + if (mmal_component_create(MMAL_COMPONENT_DEFAULT_VIDEO_RENDERER, &p->renderer)) + { + MP_FATAL(hw, "Could not create MMAL renderer.\n"); + mmal_vc_deinit(); + return -1; + } + + return 0; +} + +const struct gl_hwdec_driver gl_hwdec_rpi_overlay = { + .name = "rpi-overlay", + .api = HWDEC_RPI, + .imgfmt = IMGFMT_MMAL, + .create = create, + .reinit = reinit, + .overlay_frame = overlay_frame, + .overlay_adjust = overlay_adjust, + .destroy = destroy, +}; diff --git a/video/out/opengl/video.c b/video/out/opengl/video.c index d54858bd7e..c41aeb34cd 100644 --- a/video/out/opengl/video.c +++ b/video/out/opengl/video.c @@ -834,6 +834,10 @@ static void init_video(struct gl_video *p) for (int n = 0; exts && exts[n]; n++) gl_sc_enable_extension(p->sc, (char *)exts[n]); p->hwdec_active = true; + if (p->hwdec->driver->overlay_frame) { + MP_WARN(p, "Using HW-overlay mode. No GL filtering is performed " + "on the video!\n"); + } } else { init_format(p, p->image_params.imgfmt, false); } @@ -2679,6 +2683,7 @@ void gl_video_render_frame(struct gl_video *p, struct vo_frame *frame, int fbo) gl->BindFramebuffer(GL_FRAMEBUFFER, fbo); bool has_frame = !!frame->current; + bool is_new = has_frame && !frame->redraw && !frame->repeat; if (!has_frame || p->dst_rect.x0 > 0 || p->dst_rect.y0 > 0 || p->dst_rect.x1 < p->vp_w || p->dst_rect.y1 < abs(p->vp_h)) @@ -2688,6 +2693,28 @@ void gl_video_render_frame(struct gl_video *p, struct vo_frame *frame, int fbo) gl->Clear(GL_COLOR_BUFFER_BIT); } + if (p->hwdec_active && p->hwdec->driver->overlay_frame) { + if (has_frame) { + float *c = p->hwdec->overlay_colorkey; + gl->Scissor(p->dst_rect.x0, p->dst_rect.y0, + p->dst_rect.x1 - p->dst_rect.x0, + p->dst_rect.y1 - p->dst_rect.y0); + gl->Enable(GL_SCISSOR_TEST); + gl->ClearColor(c[0], c[1], c[2], c[3]); + gl->Clear(GL_COLOR_BUFFER_BIT); + gl->Disable(GL_SCISSOR_TEST); + } + + if (is_new || !frame->current) + p->hwdec->driver->overlay_frame(p->hwdec, frame->current); + + if (frame->current) + p->osd_pts = frame->current->pts; + + // Disable GL rendering + has_frame = false; + } + if (has_frame) { gl_sc_set_vao(p->sc, &p->vao); @@ -2702,7 +2729,6 @@ void gl_video_render_frame(struct gl_video *p, struct vo_frame *frame, int fbo) if (interpolate) { gl_video_interpolate_frame(p, frame, fbo); } else { - bool is_new = !frame->redraw && !frame->repeat; if (is_new || !p->output_fbo_valid) { p->output_fbo_valid = false; @@ -2797,6 +2823,9 @@ void gl_video_resize(struct gl_video *p, int vp_w, int vp_h, if (p->osd) mpgl_osd_resize(p->osd, p->osd_rect, p->image_params.stereo_out); + + if (p->hwdec && p->hwdec->driver->overlay_adjust) + p->hwdec->driver->overlay_adjust(p->hwdec, vp_w, vp_h, src, dst); } static struct voctrl_performance_entry gl_video_perfentry(struct gl_timer *t) diff --git a/wscript_build.py b/wscript_build.py index a288a955d6..08cb2d1cea 100644 --- a/wscript_build.py +++ b/wscript_build.py @@ -348,9 +348,10 @@ def build(ctx): ( "video/out/opengl/hwdec_dxva2.c", "gl-win32" ), ( "video/out/opengl/hwdec_dxva2gldx.c", "gl-dxinterop" ), ( "video/out/opengl/hwdec_dxva2egl.c", "egl-angle" ), + ( "video/out/opengl/hwdec_osx.c", "videotoolbox-gl" ), + ( "video/out/opengl/hwdec_rpi.c", "rpi" ), ( "video/out/opengl/hwdec_vaegl.c", "vaapi-egl" ), ( "video/out/opengl/hwdec_vaglx.c", "vaapi-glx" ), - ( "video/out/opengl/hwdec_osx.c", "videotoolbox-gl" ), ( "video/out/opengl/hwdec_vdpau.c", "vdpau-gl-x11" ), ( "video/out/opengl/lcms.c", "gl" ), ( "video/out/opengl/osd.c", "gl" ), -- cgit v1.2.3