ao_wasapi: fix fragile format-mapping code

This code tried to play with the format bits, and potentially could
create invalid formats, or reinterpret obscure formats in unexpected
ways.

Also there was an abort() call if the winapi or mpv used a format with
unexpected bit-width. This could probably easily happen; for example,
mpv supports at least one 64 bit format. And what would happen on 8 bit
formats anyway?

Untested.

1 files changed, 19 insertions, 28 deletions

diff --git a/audio/out/ao_wasapi_utils.c b/audio/out/ao_wasapi_utils.c
index c12f3baf61..c39eb6fea2 100755
--- a/audio/out/ao_wasapi_utils.c
+++ b/audio/out/ao_wasapi_utils.c
@@ -142,47 +142,36 @@ static void set_format(WAVEFORMATEXTENSIBLE *wformat, WORD bytepersample,
     wformat->dwChannelMask = chanmask;
 }
 
-static int format_set_bits(int old_format, int bits, int fp) {
-    int format = old_format;
-    format &= (~AF_FORMAT_BITS_MASK) & (~AF_FORMAT_POINT_MASK) & (~AF_FORMAT_SIGN_MASK);
-    format |= AF_FORMAT_SI;
-
-    switch (bits) {
-    case 32:
-        format |= AF_FORMAT_32BIT;
-        break;
-    case 24:
-        format |= AF_FORMAT_24BIT;
-        break;
-    case 16:
-        format |= AF_FORMAT_16BIT;
-        break;
-    default:
-        abort(); // (should be) unreachable
-    }
-
+static int format_set_bits(int old_format, int bits, int fp)
+{
     if (fp) {
-        format |= AF_FORMAT_F;
-    } else {
-        format |= AF_FORMAT_I;
+        switch (bits) {
+        case 64: return AF_FORMAT_DOUBLE;
+        case 32: return AF_FORMAT_FLOAT;
+        default: return 0;
+        }
     }
 
-    return format;
+    return af_fmt_change_bits(old_format, bits);
 }
 
 static int set_ao_format(struct wasapi_state *state,
                          struct ao *const ao,
-                         WAVEFORMATEXTENSIBLE wformat) {
-    // .Data1 == 1 is PCM, .Data1 == 3 is IEEE_FLOAT
-    int format = format_set_bits(ao->format,
-        wformat.Format.wBitsPerSample, wformat.SubFormat.Data1 == 3);
-
+                         WAVEFORMATEXTENSIBLE wformat)
+{
     if (wformat.SubFormat.Data1 != 1 && wformat.SubFormat.Data1 != 3) {
         MP_ERR(ao, "unknown SubFormat %"PRIu32"\n",
                (uint32_t)wformat.SubFormat.Data1);
         return 0;
     }
 
+    // .Data1 == 1 is PCM, .Data1 == 3 is IEEE_FLOAT
+    int format = format_set_bits(ao->format,
+        wformat.Format.wBitsPerSample, wformat.SubFormat.Data1 == 3);
+
+    if (!format)
+        return 0;
+
     ao->samplerate = wformat.Format.nSamplesPerSec;
     ao->bps = wformat.Format.nAvgBytesPerSec;
     ao->format = format;
@@ -204,6 +193,8 @@ static int try_format(struct wasapi_state *state,
     set_format(&wformat, bits / 8, samplerate, channels.num, mp_chmap_to_waveext(&channels));
 
     int af_format = format_set_bits(ao->format, bits, bits == 32);
+    if (!af_format)
+        return 0;
 
     MP_VERBOSE(ao, "trying %dch %s @ %dhz\n",
                channels.num, af_fmt_to_str(af_format), samplerate);