various: avoid function pointer casts

The opt validator functions are casted to generic validator, which has
erased type for value. Calling function by pointer of different
definition is an UB.

Avoid that by generating wrapper function that does proper argument type
conversion and calls validator function. Type erased functions have
mangled type in the name.

Fixes UBSAN failures on Clang 17, which enabled fsanitize=function by
default.

10 files changed, 29 insertions, 45 deletions

diff --git a/options/m_option.c b/options/m_option.c
index 47f9c07883..02dea2bf3c 100644
--- a/options/m_option.c
+++ b/options/m_option.c
@@ -61,11 +61,6 @@ const char m_option_path_separator = OPTION_PATH_SEPARATOR;
 #define OPT_INT_MAX(opt, T, Tm) ((opt)->min < (opt)->max \
     ? ((opt)->max >= (double)(Tm) ? (Tm) : (T)((opt)->max)) : (Tm))
 
-#if defined(__clang__)
-// Last argument of validate functions is always a pointer, but not always void*
-// which triggers UBSAN warning.
-__attribute__((no_sanitize("function")))
-#endif
 int m_option_parse(struct mp_log *log, const m_option_t *opt,
                    struct bstr name, struct bstr param, void *dst)
 {
diff --git a/options/m_option.h b/options/m_option.h
index d818a367b7..891b51794e 100644
--- a/options/m_option.h
+++ b/options/m_option.h
@@ -189,15 +189,20 @@ struct m_opt_choice_alternatives {
 const char *m_opt_choice_str(const struct m_opt_choice_alternatives *choices,
                              int value);
 
-// Validator function signatures. Required to properly type the param value.
 typedef int (*m_opt_generic_validate_fn)(struct mp_log *log, const m_option_t *opt,
                                          struct bstr name, void *value);
 
-typedef int (*m_opt_string_validate_fn)(struct mp_log *log, const m_option_t *opt,
-                                        struct bstr name, const char **value);
-typedef int (*m_opt_int_validate_fn)(struct mp_log *log, const m_option_t *opt,
-                                     struct bstr name, const int *value);
-
+#define OPT_FUNC(name) name
+#define OPT_FUNC_IN(name, suffix) name ## _ ## suffix
+#define OPT_VALIDATE_FUNC(func, value_type, suffix) \
+int OPT_FUNC(func)(struct mp_log *log, const m_option_t *opt, \
+                   struct bstr name, value_type value); \
+static inline int OPT_FUNC_IN(func, suffix)(struct mp_log *log, const m_option_t *opt, \
+                                            struct bstr name, void *value) { \
+    return OPT_FUNC(func)(log, opt, name, value); \
+} \
+int OPT_FUNC(func)(struct mp_log *log, const m_option_t *opt, \
+                   struct bstr name, value_type value)
 
 // m_option.priv points to this if OPT_SUBSTRUCT is used
 struct m_sub_options {
@@ -685,15 +690,17 @@ extern const char m_option_path_separator;
 #define OPT_CHANNELS(field) \
     OPT_TYPED_FIELD(m_option_type_channels, struct m_channels, field)
 
+#define OPT_INT_VALIDATE_FUNC(func) OPT_VALIDATE_FUNC(func, const int *, int)
+
 #define OPT_INT_VALIDATE(field, validate_fn) \
     OPT_TYPED_FIELD(m_option_type_int, int, field), \
-    .validate = (m_opt_generic_validate_fn) \
-        MP_EXPECT_TYPE(m_opt_int_validate_fn, validate_fn)
+    .validate = OPT_FUNC_IN(validate_fn, int)
+
+#define OPT_STRING_VALIDATE_FUNC(func) OPT_VALIDATE_FUNC(func, const char **, str)
 
 #define OPT_STRING_VALIDATE(field, validate_fn) \
     OPT_TYPED_FIELD(m_option_type_string, char*, field), \
-    .validate = (m_opt_generic_validate_fn) \
-        MP_EXPECT_TYPE(m_opt_string_validate_fn, validate_fn)
+    .validate = OPT_FUNC_IN(validate_fn, str)
 
 #define M_CHOICES(...) \
     .priv = (void *)&(const struct m_opt_choice_alternatives[]){ __VA_ARGS__, {0}}
diff --git a/video/out/drm_common.c b/video/out/drm_common.c
index da45ca27a3..1669e249eb 100644
--- a/video/out/drm_common.c
+++ b/video/out/drm_common.c
@@ -65,8 +65,7 @@ static int drm_connector_opt_help(struct mp_log *log, const struct m_option *opt
 static int drm_mode_opt_help(struct mp_log *log, const struct m_option *opt,
                              struct bstr name);
 
-static int drm_validate_mode_opt(struct mp_log *log, const struct m_option *opt,
-                                 struct bstr name, const char **value);
+static OPT_STRING_VALIDATE_FUNC(drm_validate_mode_opt);
 
 static void drm_show_available_modes(struct mp_log *log, const drmModeConnector *connector);
 
diff --git a/video/out/gpu/context.c b/video/out/gpu/context.c
index d842ddda91..88d4f4232d 100644
--- a/video/out/gpu/context.c
+++ b/video/out/gpu/context.c
@@ -129,8 +129,7 @@ static int ra_ctx_api_help(struct mp_log *log, const struct m_option *opt,
     return M_OPT_EXIT;
 }
 
-static int ra_ctx_validate_api(struct mp_log *log, const struct m_option *opt,
-                               struct bstr name, const char **value)
+static inline OPT_STRING_VALIDATE_FUNC(ra_ctx_validate_api)
 {
     struct bstr param = bstr0(*value);
     if (bstr_equals0(param, "auto"))
@@ -154,8 +153,7 @@ static int ra_ctx_context_help(struct mp_log *log, const struct m_option *opt,
     return M_OPT_EXIT;
 }
 
-static int ra_ctx_validate_context(struct mp_log *log, const struct m_option *opt,
-                                   struct bstr name, const char **value)
+static inline OPT_STRING_VALIDATE_FUNC(ra_ctx_validate_context)
 {
     struct bstr param = bstr0(*value);
     if (bstr_equals0(param, "auto"))
diff --git a/video/out/gpu/d3d11_helpers.h b/video/out/gpu/d3d11_helpers.h
index c6954a7a4a..5ba013e747 100644
--- a/video/out/gpu/d3d11_helpers.h
+++ b/video/out/gpu/d3d11_helpers.h
@@ -69,9 +69,7 @@ IDXGIAdapter1 *mp_get_dxgi_adapter(struct mp_log *log,
                                    bstr requested_adapter_name,
                                    bstr *listing);
 
-int mp_dxgi_validate_adapter(struct mp_log *log,
-                             const struct m_option *opt,
-                             struct bstr name, const char **value);
+OPT_STRING_VALIDATE_FUNC(mp_dxgi_validate_adapter);
 
 bool mp_dxgi_list_or_verify_adapters(struct mp_log *log,
                                      bstr adapter_name,
diff --git a/video/out/gpu/hwdec.h b/video/out/gpu/hwdec.h
index 7766073e0c..f195606417 100644
--- a/video/out/gpu/hwdec.h
+++ b/video/out/gpu/hwdec.h
@@ -18,12 +18,8 @@ struct ra_hwdec_ctx {
     int num_hwdecs;
 };
 
-int ra_hwdec_validate_opt(struct mp_log *log, const m_option_t *opt,
-                          struct bstr name, const char **value);
-
-int ra_hwdec_validate_drivers_only_opt(struct mp_log *log,
-                                       const m_option_t *opt,
-                                       struct bstr name, const char **value);
+OPT_STRING_VALIDATE_FUNC(ra_hwdec_validate_opt);
+OPT_STRING_VALIDATE_FUNC(ra_hwdec_validate_drivers_only_opt);
 
 void ra_hwdec_ctx_init(struct ra_hwdec_ctx *ctx, struct mp_hwdec_devices *devs,
                        const char *opt, bool load_all_by_default);
diff --git a/video/out/gpu/lcms.c b/video/out/gpu/lcms.c
index 1e451aa7a2..e8fe1d48c3 100644
--- a/video/out/gpu/lcms.c
+++ b/video/out/gpu/lcms.c
@@ -494,8 +494,7 @@ bool gl_lcms_get_lut3d(struct gl_lcms *p, struct lut3d **result_lut3d,
 
 #endif
 
-static int validate_3dlut_size_opt(struct mp_log *log, const m_option_t *opt,
-                                   struct bstr name, const char **value)
+static inline OPT_STRING_VALIDATE_FUNC(validate_3dlut_size_opt)
 {
     int p1, p2, p3;
     return gl_parse_3dlut_size(*value, &p1, &p2, &p3) ? 0 : M_OPT_INVALID;
diff --git a/video/out/gpu/video.c b/video/out/gpu/video.c
index 07a2bf8947..1de9ae8cd5 100644
--- a/video/out/gpu/video.c
+++ b/video/out/gpu/video.c
@@ -329,14 +329,9 @@ static const struct gl_video_opts gl_video_opts_def = {
     .hwdec_interop = "auto",
 };
 
-static int validate_scaler_opt(struct mp_log *log, const m_option_t *opt,
-                               struct bstr name, const char **value);
-
-static int validate_window_opt(struct mp_log *log, const m_option_t *opt,
-                               struct bstr name, const char **value);
-
-static int validate_error_diffusion_opt(struct mp_log *log, const m_option_t *opt,
-                                        struct bstr name, const char **value);
+static OPT_STRING_VALIDATE_FUNC(validate_scaler_opt);
+static OPT_STRING_VALIDATE_FUNC(validate_window_opt);
+static OPT_STRING_VALIDATE_FUNC(validate_error_diffusion_opt);
 
 #define OPT_BASE_STRUCT struct gl_video_opts
 
diff --git a/video/out/vulkan/context.c b/video/out/vulkan/context.c
index 5087403ade..f04895a2ae 100644
--- a/video/out/vulkan/context.c
+++ b/video/out/vulkan/context.c
@@ -37,8 +37,7 @@ struct vulkan_opts {
     bool async_compute;
 };
 
-static int vk_validate_dev(struct mp_log *log, const struct m_option *opt,
-                           struct bstr name, const char **value)
+static inline OPT_STRING_VALIDATE_FUNC(vk_validate_dev)
 {
     struct bstr param = bstr0(*value);
     int ret = M_OPT_INVALID;
diff --git a/video/vaapi.c b/video/vaapi.c
index d7c884a30e..666fd00f53 100644
--- a/video/vaapi.c
+++ b/video/vaapi.c
@@ -45,9 +45,7 @@
 #define DEV_PATH_DEFAULT "/dev/dri/renderD128"
 #define DEV_PATH_VALIDATE validate_path
 
-static int validate_path(struct mp_log *log,
-                         const struct m_option *opt,
-                         struct bstr name, const char **value)
+static inline OPT_STRING_VALIDATE_FUNC(validate_path)
 {
     return (*value && **value) ? 0 : M_OPT_INVALID;
 }