Avoid misleading error "ASS: cannot add video filter"

When using libass with a VO that has direct EOSD support the vf_ass
video filter is unnecessary and is not added automatically, but the
code that adds the filter when it is needed produced misleading output
messages in this case. The following two messages were printed at
MSGL_ERR level, making it look like an error condition:
Couldn't open video filter 'ass'.
ASS: cannot add video filter

Add a version of vf_open_plugin() called vf_open_plugin_noerr() that
does not itself print an error message if opening a vf fails and that
returns the exact status code returned by the vf open() function. Make
vf_ass return a different status code depending on whether there was
an actual error or if it determined a filter would be redundant. Use
the _noerr function in the code adding the filter to avoid the first
message and check the status code to avoid the second.

3 files changed, 29 insertions, 5 deletions

diff --git a/libmpcodecs/vf.c b/libmpcodecs/vf.c
index d753d3d68a..c6f23eb382 100644
--- a/libmpcodecs/vf.c
+++ b/libmpcodecs/vf.c
@@ -420,7 +420,11 @@ static int vf_default_query_format(struct vf_instance* vf, unsigned int fmt){
   return vf_next_query_format(vf,fmt);
 }
 
-vf_instance_t* vf_open_plugin(struct MPOpts *opts, const vf_info_t* const* filter_list, vf_instance_t* next, const char *name, char **args){
+struct vf_instance *vf_open_plugin_noerr(struct MPOpts *opts,
+                                         const vf_info_t * const *filter_list,
+                                         vf_instance_t *next, const char *name,
+                                         char **args, int *retcode)
+{
     vf_instance_t* vf;
     int i;
     for(i=0;;i++){
@@ -453,12 +457,26 @@ vf_instance_t* vf_open_plugin(struct MPOpts *opts, const vf_info_t* const* filte
 	args = (char**)args[1];
       else
 	args = NULL;
-    if(vf->info->open(vf,(char*)args)>0) return vf; // Success!
+    *retcode = vf->info->open(vf,(char*)args);
+    if (*retcode > 0)
+        return vf;
     free(vf);
-    mp_tmsg(MSGT_VFILTER,MSGL_ERR,"Couldn't open video filter '%s'.\n",name);
     return NULL;
 }
 
+struct vf_instance *vf_open_plugin(struct MPOpts *opts,
+                                   const vf_info_t * const *filter_list,
+                                   vf_instance_t *next, const char *name,
+                                   char **args)
+{
+    struct vf_instance *vf = vf_open_plugin_noerr(opts, filter_list, next,
+                                                  name, args, &(int){0});
+    if (!vf)
+        mp_tmsg(MSGT_VFILTER, MSGL_ERR, "Couldn't open video filter '%s'.\n",
+                name);
+    return vf;
+}
+
 vf_instance_t* vf_open_filter(struct MPOpts *opts, vf_instance_t* next, const char *name, char **args){
   if(args && strcmp(args[0],"_oldargs_")) {
     int i,l = 0;
diff --git a/libmpcodecs/vf.h b/libmpcodecs/vf.h
index f9dbb8b4fb..1af0ecca4f 100644
--- a/libmpcodecs/vf.h
+++ b/libmpcodecs/vf.h
@@ -107,6 +107,10 @@ void vf_mpi_clear(mp_image_t* mpi,int x0,int y0,int w,int h);
 mp_image_t* vf_get_image(vf_instance_t* vf, unsigned int outfmt, int mp_imgtype, int mp_imgflag, int w, int h);
 
 vf_instance_t* vf_open_plugin(struct MPOpts *opts, const vf_info_t* const* filter_list, vf_instance_t* next, const char *name, char **args);
+struct vf_instance *vf_open_plugin_noerr(struct MPOpts *opts,
+                                         const vf_info_t * const *filter_list,
+                                         vf_instance_t *next, const char *name,
+                                         char **args, int *retcode);
 vf_instance_t* vf_open_filter(struct MPOpts *opts, vf_instance_t* next, const char *name, char **args);
 vf_instance_t* vf_add_before_vo(vf_instance_t **vf, char *name, char **args);
 vf_instance_t* vf_open_encoder(struct MPOpts *opts, vf_instance_t* next, const char *name, char *args);
diff --git a/libmpcodecs/vf_ass.c b/libmpcodecs/vf_ass.c
index 5ea2f8b10d..9076ed099e 100644
--- a/libmpcodecs/vf_ass.c
+++ b/libmpcodecs/vf_ass.c
@@ -388,10 +388,12 @@ static int open(vf_instance_t *vf, char* args)
 	vf->priv->outfmt = vf_match_csp(&vf->next,fmt_list,IMGFMT_YV12);
 	if (vf->priv->outfmt)
 		flags = vf_next_query_format(vf, vf->priv->outfmt);
-	if (!vf->priv->outfmt || (vf->priv->auto_insert && flags&VFCAP_EOSD))
-	{
+	if (!vf->priv->outfmt) {
 		uninit(vf);
 		return 0;
+	} else if (vf->priv->auto_insert && flags&VFCAP_EOSD) {
+		uninit(vf);
+		return -1;
 	}
 
 	if (vf->priv->auto_insert)