diff options
author | sfan5 <sfan5@live.de> | 2021-10-18 16:50:39 +0200 |
---|---|---|
committer | sfan5 <sfan5@live.de> | 2022-10-02 14:12:26 +0200 |
commit | 5463d3eeff0aeff4f9c6d7634ace157ceb2cec57 (patch) | |
tree | 648d819132bd58c5fa476dde9cf03c50d3d76160 | |
parent | 2207236aaae54f32c1861c6fd77219e28139dc78 (diff) | |
download | mpv-5463d3eeff0aeff4f9c6d7634ace157ceb2cec57.tar.bz2 mpv-5463d3eeff0aeff4f9c6d7634ace157ceb2cec57.tar.xz |
vo_gpu: hwdec: add Android hwdec utilizing AImageReader
-rw-r--r-- | DOCS/man/options.rst | 10 | ||||
-rw-r--r-- | DOCS/man/vo.rst | 4 | ||||
-rw-r--r-- | meson.build | 9 | ||||
-rw-r--r-- | meson_options.txt | 1 | ||||
-rw-r--r-- | video/out/gpu/hwdec.c | 4 | ||||
-rw-r--r-- | video/out/hwdec/hwdec_aimagereader.c | 396 | ||||
-rw-r--r-- | wscript | 6 | ||||
-rw-r--r-- | wscript_build.py | 1 |
8 files changed, 427 insertions, 4 deletions
diff --git a/DOCS/man/options.rst b/DOCS/man/options.rst index c2b715c1ce..0d18784b26 100644 --- a/DOCS/man/options.rst +++ b/DOCS/man/options.rst @@ -1264,7 +1264,8 @@ Video :dxva2-copy: copies video back to system RAM (Windows only) :vdpau: requires ``--vo=gpu`` with X11, or ``--vo=vdpau`` (Linux only) :vdpau-copy: copies video back into system RAM (Linux with some GPUs only) - :mediacodec: requires ``--vo=mediacodec_embed`` (Android only) + :mediacodec: requires ``--vo=gpu --gpu-context=android`` + or ``--vo=mediacodec_embed`` (Android only) :mediacodec-copy: copies video back to system RAM (Android only) :mmal: requires ``--vo=gpu`` (Raspberry Pi only - default if available) :mmal-copy: copies video back to system RAM (Raspberry Pi only) @@ -1364,6 +1365,11 @@ Video ``rpi`` always uses the hardware overlay renderer, even with ``--vo=gpu``. + ``mediacodec`` is not safe. It forces RGB conversion (not with ``-copy``) + and how well it handles non-standard colorspaces is not known. + In the rare cases where 10-bit is supported the bit depth of the output + will be reduced to 8. + ``cuda`` should usually be safe, but depending on how a file/stream has been mixed, it has been reported to corrupt the timestamps causing glitched, flashing frames. It can also sometimes cause massive @@ -3317,7 +3323,7 @@ Window On Android, the ID is interpreted as ``android.view.Surface``. Pass it as a value cast to ``intptr_t``. Use with ``--vo=mediacodec_embed`` and ``--hwdec=mediacodec`` for direct rendering using MediaCodec, or with - ``--vo=gpu --gpu-context=android`` (with or without ``--hwdec=mediacodec-copy``). + ``--vo=gpu --gpu-context=android`` (with or without ``--hwdec=mediacodec``). ``--no-window-dragging`` Don't move the window when clicking on it and moving the mouse pointer. diff --git a/DOCS/man/vo.rst b/DOCS/man/vo.rst index 13d4050dc4..5e5b1394d9 100644 --- a/DOCS/man/vo.rst +++ b/DOCS/man/vo.rst @@ -676,8 +676,8 @@ Available video output drivers are: many of mpv's features (subtitle rendering, OSD/OSC, video filters, etc) are not available with this driver. - To use hardware decoding with ``--vo=gpu`` instead, use - ``--hwdec=mediacodec-copy`` along with ``--gpu-context=android``. + To use hardware decoding with ``--vo=gpu`` instead, use ``--hwdec=mediacodec`` + or ``mediacodec-copy`` along with ``--gpu-context=android``. ``wlshm`` (Wayland only) Shared memory video output driver without hardware acceleration that works diff --git a/meson.build b/meson.build index c0c769736b..244977dc9c 100644 --- a/meson.build +++ b/meson.build @@ -1264,6 +1264,15 @@ if features['ffnvcodec'] sources += files('video/cuda.c') endif +android_media_ndk = get_option('android-media-ndk').require( + features['android'] and cc.has_header_symbol('media/NdkImageReader.h', 'AIMAGE_FORMAT_PRIVATE') +) +features += {'android-media-ndk': android_media_ndk.allowed()} +if features['android-media-ndk'] + # header only, library is dynamically loaded + sources += files('video/out/hwdec/hwdec_aimagereader.c') +endif + cuda_hwaccel = get_option('cuda-hwaccel').require( features['ffnvcodec'], error_message: 'ffnvcodec was not found!', diff --git a/meson_options.txt b/meson_options.txt index e24f8d50dd..9093351113 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -94,6 +94,7 @@ option('x11', type: 'feature', value: 'auto', description: 'X11') option('xv', type: 'feature', value: 'auto', description: 'Xv video output') # hwaccel features +option('android-media-ndk', type: 'feature', value: 'auto', description: 'Android Media APIs') option('cuda-hwaccel', type: 'feature', value: 'auto', description: 'CUDA acceleration') option('cuda-interop', type: 'feature', value: 'auto', description: 'CUDA with graphics interop') option('d3d-hwaccel', type: 'feature', value: 'auto', description: 'D3D11VA hwaccel') diff --git a/video/out/gpu/hwdec.c b/video/out/gpu/hwdec.c index 01147be832..8bc60b502c 100644 --- a/video/out/gpu/hwdec.c +++ b/video/out/gpu/hwdec.c @@ -37,6 +37,7 @@ extern const struct ra_hwdec_driver ra_hwdec_cuda; extern const struct ra_hwdec_driver ra_hwdec_rpi_overlay; extern const struct ra_hwdec_driver ra_hwdec_drmprime; extern const struct ra_hwdec_driver ra_hwdec_drmprime_drm; +extern const struct ra_hwdec_driver ra_hwdec_aimagereader; const struct ra_hwdec_driver *const ra_hwdec_drivers[] = { #if HAVE_VAAPI_EGL || HAVE_VAAPI_LIBPLACEBO @@ -75,6 +76,9 @@ const struct ra_hwdec_driver *const ra_hwdec_drivers[] = { &ra_hwdec_drmprime_drm, &ra_hwdec_drmprime, #endif +#if HAVE_ANDROID_MEDIA_NDK + &ra_hwdec_aimagereader, +#endif NULL }; diff --git a/video/out/hwdec/hwdec_aimagereader.c b/video/out/hwdec/hwdec_aimagereader.c new file mode 100644 index 0000000000..7158ba647d --- /dev/null +++ b/video/out/hwdec/hwdec_aimagereader.c @@ -0,0 +1,396 @@ +/* + * Copyright (c) 2021 sfan5 <sfan5@live.de> + * + * This file is part of mpv. + * + * mpv is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * mpv is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with mpv. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <assert.h> +#include <pthread.h> +#include <dlfcn.h> +#include <EGL/egl.h> +#include <media/NdkImageReader.h> +#include <android/native_window_jni.h> +#include <libavcodec/mediacodec.h> +#include <libavutil/hwcontext.h> +#include <libavutil/hwcontext_mediacodec.h> + +#include "misc/jni.h" +#include "osdep/timer.h" +#include "video/out/gpu/hwdec.h" +#include "video/out/opengl/ra_gl.h" + +typedef void *GLeglImageOES; +typedef void *EGLImageKHR; +#define EGL_NATIVE_BUFFER_ANDROID 0x3140 + +struct priv_owner { + struct mp_hwdec_ctx hwctx; + AImageReader *reader; + jobject surface; + void *lib_handle; + + media_status_t (*AImageReader_newWithUsage)( + int32_t, int32_t, int32_t, uint64_t, int32_t, AImageReader **); + media_status_t (*AImageReader_getWindow)( + AImageReader *, ANativeWindow **); + media_status_t (*AImageReader_setImageListener)( + AImageReader *, AImageReader_ImageListener *); + media_status_t (*AImageReader_acquireLatestImage)(AImageReader *, AImage **); + void (*AImageReader_delete)(AImageReader *); + media_status_t (*AImage_getHardwareBuffer)(const AImage *, AHardwareBuffer **); + void (*AImage_delete)(AImage *); + void (*AHardwareBuffer_describe)(const AHardwareBuffer *, AHardwareBuffer_Desc *); + jobject (*ANativeWindow_toSurface)(JNIEnv *, ANativeWindow *); +}; + +struct priv { + struct mp_log *log; + + GLuint gl_texture; + AImage *image; + EGLImageKHR egl_image; + + pthread_mutex_t lock; + pthread_cond_t cond; + bool image_available; + + EGLImageKHR (EGLAPIENTRY *CreateImageKHR)( + EGLDisplay, EGLContext, EGLenum, EGLClientBuffer, const EGLint *); + EGLBoolean (EGLAPIENTRY *DestroyImageKHR)(EGLDisplay, EGLImageKHR); + EGLClientBuffer (EGLAPIENTRY *GetNativeClientBufferANDROID)( + const struct AHardwareBuffer *); + void (EGLAPIENTRY *EGLImageTargetTexture2DOES)(GLenum, GLeglImageOES); +}; + +const static struct { const char *symbol; int offset; } lib_functions[] = { + { "AImageReader_newWithUsage", offsetof(struct priv_owner, AImageReader_newWithUsage) }, + { "AImageReader_getWindow", offsetof(struct priv_owner, AImageReader_getWindow) }, + { "AImageReader_setImageListener", offsetof(struct priv_owner, AImageReader_setImageListener) }, + { "AImageReader_acquireLatestImage", offsetof(struct priv_owner, AImageReader_acquireLatestImage) }, + { "AImageReader_delete", offsetof(struct priv_owner, AImageReader_delete) }, + { "AImage_getHardwareBuffer", offsetof(struct priv_owner, AImage_getHardwareBuffer) }, + { "AImage_delete", offsetof(struct priv_owner, AImage_delete) }, + { "AHardwareBuffer_describe", offsetof(struct priv_owner, AHardwareBuffer_describe) }, + { "ANativeWindow_toSurface", offsetof(struct priv_owner, ANativeWindow_toSurface) }, + { NULL, 0 }, +}; + + +static AVBufferRef *create_mediacodec_device_ref(jobject surface) +{ + AVBufferRef *device_ref = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_MEDIACODEC); + if (!device_ref) + return NULL; + + AVHWDeviceContext *ctx = (void *)device_ref->data; + AVMediaCodecDeviceContext *hwctx = ctx->hwctx; + hwctx->surface = surface; + + if (av_hwdevice_ctx_init(device_ref) < 0) + av_buffer_unref(&device_ref); + + return device_ref; +} + +static bool load_lib_functions(struct priv_owner *p, struct mp_log *log) +{ + p->lib_handle = dlopen("libmediandk.so", RTLD_NOW | RTLD_GLOBAL); + if (!p->lib_handle) + return false; + for (int i = 0; lib_functions[i].symbol; i++) { + const char *sym = lib_functions[i].symbol; + void *fun = dlsym(p->lib_handle, sym); + if (!fun) + fun = dlsym(RTLD_DEFAULT, sym); + if (!fun) { + mp_warn(log, "Could not resolve symbol %s\n", sym); + return false; + } + + *(void **) ((uint8_t*)p + lib_functions[i].offset) = fun; + } + return true; +} + +static int init(struct ra_hwdec *hw) +{ + struct priv_owner *p = hw->priv; + + if (!ra_is_gl(hw->ra)) + return -1; + if (!eglGetCurrentContext()) + return -1; + + const char *exts = eglQueryString(eglGetCurrentDisplay(), EGL_EXTENSIONS); + if (!gl_check_extension(exts, "EGL_ANDROID_image_native_buffer")) + return -1; + + if (!load_lib_functions(p, hw->log)) + return false; + + static const char *es2_exts[] = {"GL_OES_EGL_image_external", 0}; + static const char *es3_exts[] = {"GL_OES_EGL_image_external_essl3", 0}; + GL *gl = ra_gl_get(hw->ra); + if (gl_check_extension(gl->extensions, es3_exts[0])) + hw->glsl_extensions = es3_exts; + else + hw->glsl_extensions = es2_exts; + + // dummy dimensions, AImageReader only transports hardware buffers + media_status_t ret = p->AImageReader_newWithUsage(16, 16, + AIMAGE_FORMAT_PRIVATE, AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE, + 5, &p->reader); + if (ret != AMEDIA_OK) { + MP_ERR(hw, "newWithUsage failed: %d\n", ret); + return -1; + } + assert(p->reader); + + ANativeWindow *window; + ret = p->AImageReader_getWindow(p->reader, &window); + if (ret != AMEDIA_OK) { + MP_ERR(hw, "getWindow failed: %d\n", ret); + return -1; + } + assert(window); + + JNIEnv *env = MP_JNI_GET_ENV(hw); + assert(env); + jobject surface = p->ANativeWindow_toSurface(env, window); + p->surface = (*env)->NewGlobalRef(env, surface); + (*env)->DeleteLocalRef(env, surface); + + p->hwctx = (struct mp_hwdec_ctx) { + .driver_name = hw->driver->name, + .av_device_ref = create_mediacodec_device_ref(p->surface), + }; + hwdec_devices_add(hw->devs, &p->hwctx); + + return 0; +} + +static void uninit(struct ra_hwdec *hw) +{ + struct priv_owner *p = hw->priv; + JNIEnv *env = MP_JNI_GET_ENV(hw); + assert(env); + + if (p->surface) { + (*env)->DeleteGlobalRef(env, p->surface); + p->surface = NULL; + } + + if (p->reader) { + p->AImageReader_delete(p->reader); + p->reader = NULL; + } + + hwdec_devices_remove(hw->devs, &p->hwctx); + av_buffer_unref(&p->hwctx.av_device_ref); + + if (p->lib_handle) { + dlclose(p->lib_handle); + p->lib_handle = NULL; + } +} + +static void image_callback(void *context, AImageReader *reader) +{ + struct priv *p = context; + + pthread_mutex_lock(&p->lock); + p->image_available = true; + pthread_cond_signal(&p->cond); + pthread_mutex_unlock(&p->lock); +} + +static int mapper_init(struct ra_hwdec_mapper *mapper) +{ + struct priv *p = mapper->priv; + struct priv_owner *o = mapper->owner->priv; + GL *gl = ra_gl_get(mapper->ra); + + p->log = mapper->log; + pthread_mutex_init(&p->lock, NULL); + pthread_cond_init(&p->cond, NULL); + + p->CreateImageKHR = (void *)eglGetProcAddress("eglCreateImageKHR"); + p->DestroyImageKHR = (void *)eglGetProcAddress("eglDestroyImageKHR"); + p->GetNativeClientBufferANDROID = + (void *)eglGetProcAddress("eglGetNativeClientBufferANDROID"); + p->EGLImageTargetTexture2DOES = + (void *)eglGetProcAddress("glEGLImageTargetTexture2DOES"); + + if (!p->CreateImageKHR || !p->DestroyImageKHR || + !p->GetNativeClientBufferANDROID || !p->EGLImageTargetTexture2DOES) + return false; + + AImageReader_ImageListener listener = { + .context = p, + .onImageAvailable = image_callback, + }; + o->AImageReader_setImageListener(o->reader, &listener); + + mapper->dst_params = mapper->src_params; + mapper->dst_params.imgfmt = IMGFMT_RGB0; + mapper->dst_params.hw_subfmt = 0; + + // texture creation + gl->GenTextures(1, &p->gl_texture); + gl->BindTexture(GL_TEXTURE_EXTERNAL_OES, p->gl_texture); + gl->TexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + gl->TexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + gl->TexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + gl->TexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + gl->BindTexture(GL_TEXTURE_EXTERNAL_OES, 0); + + struct ra_tex_params params = { + .dimensions = 2, + .w = mapper->src_params.w, + .h = mapper->src_params.h, + .d = 1, + .format = ra_find_unorm_format(mapper->ra, 1, 4), + .render_src = true, + .src_linear = true, + .external_oes = true, + }; + + if (params.format->ctype != RA_CTYPE_UNORM) + return -1; + + mapper->tex[0] = ra_create_wrapped_tex(mapper->ra, ¶ms, p->gl_texture); + if (!mapper->tex[0]) + return -1; + + return 0; +} + +static void mapper_uninit(struct ra_hwdec_mapper *mapper) +{ + struct priv *p = mapper->priv; + struct priv_owner *o = mapper->owner->priv; + GL *gl = ra_gl_get(mapper->ra); + + o->AImageReader_setImageListener(o->reader, NULL); + + gl->DeleteTextures(1, &p->gl_texture); + p->gl_texture = 0; + + ra_tex_free(mapper->ra, &mapper->tex[0]); + + pthread_mutex_destroy(&p->lock); + pthread_cond_destroy(&p->cond); +} + +static void mapper_unmap(struct ra_hwdec_mapper *mapper) +{ + struct priv *p = mapper->priv; + struct priv_owner *o = mapper->owner->priv; + + if (p->egl_image) { + p->DestroyImageKHR(eglGetCurrentDisplay(), p->egl_image); + p->egl_image = 0; + } + + if (p->image) { + o->AImage_delete(p->image); + p->image = NULL; + } +} + +static int mapper_map(struct ra_hwdec_mapper *mapper) +{ + struct priv *p = mapper->priv; + struct priv_owner *o = mapper->owner->priv; + GL *gl = ra_gl_get(mapper->ra); + + { + if (mapper->src->imgfmt != IMGFMT_MEDIACODEC) + return -1; + AVMediaCodecBuffer *buffer = (AVMediaCodecBuffer *)mapper->src->planes[3]; + av_mediacodec_release_buffer(buffer, 1); + } + + bool image_available = false; + pthread_mutex_lock(&p->lock); + if (!p->image_available) { + struct timespec ts = mp_rel_time_to_timespec(0.1); + pthread_cond_timedwait(&p->cond, &p->lock, &ts); + if (!p->image_available) + MP_WARN(mapper, "Waiting for frame timed out!\n"); + } + image_available = p->image_available; + p->image_available = false; + pthread_mutex_unlock(&p->lock); + + media_status_t ret = o->AImageReader_acquireLatestImage(o->reader, &p->image); + if (ret != AMEDIA_OK) { + MP_ERR(mapper, "acquireLatestImage failed: %d\n", ret); + // If we merely timed out waiting return success anyway to avoid + // flashing frames of render errors. + return image_available ? -1 : 0; + } + assert(p->image); + + AHardwareBuffer *hwbuf = NULL; + ret = o->AImage_getHardwareBuffer(p->image, &hwbuf); + if (ret != AMEDIA_OK) { + MP_ERR(mapper, "getHardwareBuffer failed: %d\n", ret); + return -1; + } + assert(hwbuf); + + // Update texture size since it may differ + AHardwareBuffer_Desc d; + o->AHardwareBuffer_describe(hwbuf, &d); + if (mapper->tex[0]->params.w != d.width || mapper->tex[0]->params.h != d.height) { + MP_VERBOSE(p, "Texture dimensions changed to %dx%d\n", d.width, d.height); + mapper->tex[0]->params.w = d.width; + mapper->tex[0]->params.h = d.height; + } + + EGLClientBuffer buf = p->GetNativeClientBufferANDROID(hwbuf); + if (!buf) + return -1; + + const int attribs[] = {EGL_NONE}; + p->egl_image = p->CreateImageKHR(eglGetCurrentDisplay(), + EGL_NO_CONTEXT, EGL_NATIVE_BUFFER_ANDROID, buf, attribs); + if (!p->egl_image) + return -1; + + gl->BindTexture(GL_TEXTURE_EXTERNAL_OES, p->gl_texture); + p->EGLImageTargetTexture2DOES(GL_TEXTURE_EXTERNAL_OES, p->egl_image); + gl->BindTexture(GL_TEXTURE_EXTERNAL_OES, 0); + + return 0; +} + + +const struct ra_hwdec_driver ra_hwdec_aimagereader = { + .name = "aimagereader", + .priv_size = sizeof(struct priv_owner), + .imgfmts = {IMGFMT_MEDIACODEC, 0}, + .init = init, + .uninit = uninit, + .mapper = &(const struct ra_hwdec_mapper_driver){ + .priv_size = sizeof(struct priv), + .init = mapper_init, + .uninit = mapper_uninit, + .map = mapper_map, + .unmap = mapper_unmap, + }, +}; @@ -162,6 +162,12 @@ main_dependencies = [ 'desc': 'Android environment', 'func': check_statement('android/api-level.h', '(void)__ANDROID__'), # arbitrary android-specific header }, { + 'name': '--android-media-ndk', + 'desc': 'Android Media APIs', + 'deps': 'android', + # header only, library is dynamically loaded + 'func': check_statement('media/NdkImageReader.h', 'int x = AIMAGE_FORMAT_PRIVATE'), + }, { 'name': '--tvos', 'desc': 'tvOS environment', 'func': check_statement( diff --git a/wscript_build.py b/wscript_build.py index 980eca19ad..a22fd1f38f 100644 --- a/wscript_build.py +++ b/wscript_build.py @@ -469,6 +469,7 @@ def build(ctx): ( "video/out/gpu/video.c" ), ( "video/out/gpu/video_shaders.c" ), ( "video/out/gpu_next/context.c", "libplacebo-next" ), + ( "video/out/hwdec/hwdec_aimagereader.c", "android-media-ndk" ), ( "video/out/hwdec/hwdec_cuda.c", "cuda-interop" ), ( "video/out/hwdec/hwdec_cuda_gl.c", "cuda-interop && gl" ), ( "video/out/hwdec/hwdec_cuda_vk.c", "cuda-interop && vulkan" ), |