audio: cleanup spdif format definitions

Before this commit, there was AF_FORMAT_AC3 (the original spdif format,
used for AC3 and DTS core), and AF_FORMAT_IEC61937 (used for AC3, DTS
and DTS-HD), which was handled as some sort of superset for
AF_FORMAT_AC3. There also was AF_FORMAT_MPEG2, which used
IEC61937-framing, but still was handled as something "separate".

Technically, all of them are pretty similar, but may use different
bitrates. Since digital passthrough pretends to be PCM (just with
special headers that wrap digital packets), this is easily detectable by
the higher samplerate or higher number of channels, so I don't know why
you'd need a separate "class" of sample formats (AF_FORMAT_AC3 vs.
AF_FORMAT_IEC61937) to distinguish them. Actually, this whole thing is
just a mess.

Simplify this by handling all these formats the same way.
AF_FORMAT_IS_IEC61937() now returns 1 for all spdif formats (even MP3).
All AOs just accept all spdif formats now - whether that works or not is
not really clear (seems inconsistent due to earlier attempts to make
DTS-HD work). But on the other hand, enabling spdif requires manual user
interaction, so it doesn't matter much if initialization fails in
slightly less graceful ways if it can't work at all.

At a later point, we will support passthrough with ao_pulse. It seems
the PulseAudio API wants to know the codec type (or maybe not - feeding
it DTS while telling it it's AC3 works), add separate formats for each
codecs. While this reminds of the earlier chaos, it's stricter, and most
code just uses AF_FORMAT_IS_IEC61937().

Also, modify AF_FORMAT_TYPE_MASK (renamed from AF_FORMAT_POINT_MASK) to
include special formats, so that it always describes the fundamental
sample format type. This also ensures valid AF formats are never 0 (this
was probably broken in one of the earlier commits from today).

7 files changed, 39 insertions, 36 deletions

diff --git a/audio/out/ao_alsa.c b/audio/out/ao_alsa.c
index c344f6f20e..52f410cf5e 100644
--- a/audio/out/ao_alsa.c
+++ b/audio/out/ao_alsa.c
@@ -100,7 +100,7 @@ static int control(struct ao *ao, enum aocontrol cmd, void *arg)
         long get_vol, set_vol;
         float f_multi;
 
-        if (AF_FORMAT_IS_IEC61937(ao->format))
+        if (AF_FORMAT_IS_SPECIAL(ao->format))
             return CONTROL_FALSE;
 
         //allocate simple id
@@ -218,9 +218,6 @@ static const int mp_to_alsa_format[][2] = {
     {AF_FORMAT_S24,
             MP_SELECT_LE_BE(SND_PCM_FORMAT_S24_3LE, SND_PCM_FORMAT_S24_3BE)},
     {AF_FORMAT_FLOAT,       SND_PCM_FORMAT_FLOAT},
-    {AF_FORMAT_AC3,         SND_PCM_FORMAT_S16},
-    {AF_FORMAT_IEC61937,    SND_PCM_FORMAT_S16},
-    {AF_FORMAT_MPEG2,       SND_PCM_FORMAT_MPEG},
     {AF_FORMAT_UNKNOWN,     SND_PCM_FORMAT_UNKNOWN},
 };
 
@@ -403,7 +400,15 @@ static int init(struct ao *ao)
     err = snd_pcm_hw_params_any(p->alsa, alsa_hwparams);
     CHECK_ALSA_ERROR("Unable to get initial parameters");
 
-    p->alsa_fmt = find_alsa_format(ao->format);
+    if (AF_FORMAT_IS_IEC61937(ao->format)) {
+        if (ao->format == AF_FORMAT_S_MP3) {
+            p->alsa_fmt = SND_PCM_FORMAT_MPEG;
+        } else {
+            p->alsa_fmt = SND_PCM_FORMAT_S16;
+        }
+    } else {
+        p->alsa_fmt = find_alsa_format(ao->format);
+    }
     if (p->alsa_fmt == SND_PCM_FORMAT_UNKNOWN) {
         p->alsa_fmt = SND_PCM_FORMAT_S16;
         ao->format = AF_FORMAT_S16;
@@ -411,15 +416,12 @@ static int init(struct ao *ao)
 
     err = snd_pcm_hw_params_test_format(p->alsa, alsa_hwparams, p->alsa_fmt);
     if (err < 0) {
+        if (AF_FORMAT_IS_IEC61937(ao->format))
+            CHECK_ALSA_ERROR("Unable to set IEC61937 format");
         MP_INFO(ao, "Format %s is not supported by hardware, trying default.\n",
                 af_fmt_to_str(ao->format));
         p->alsa_fmt = SND_PCM_FORMAT_S16;
-        if (AF_FORMAT_IS_AC3(ao->format))
-            ao->format = AF_FORMAT_AC3;
-        else if (AF_FORMAT_IS_IEC61937(ao->format))
-            ao->format = AF_FORMAT_IEC61937;
-        else
-            ao->format = AF_FORMAT_S16;
+        ao->format = AF_FORMAT_S16;
     }
 
     err = snd_pcm_hw_params_set_format(p->alsa, alsa_hwparams, p->alsa_fmt);
diff --git a/audio/out/ao_coreaudio_device.c b/audio/out/ao_coreaudio_device.c
index 3ba1f0ce37..c221b62e59 100644
--- a/audio/out/ao_coreaudio_device.c
+++ b/audio/out/ao_coreaudio_device.c
@@ -410,8 +410,9 @@ static int init(struct ao *ao)
     ao->format = af_fmt_from_planar(ao->format);
 
     bool supports_digital = false;
-    /* Probe whether device support S/PDIF stream output if input is AC3 stream. */
-    if (AF_FORMAT_IS_AC3(ao->format)) {
+    /* Probe whether device support S/PDIF stream output if input is AC3 stream,
+     * or anything else IEC61937-framed. */
+    if (AF_FORMAT_IS_IEC61937(ao->format)) {
         if (ca_device_supports_digital(ao, p->device))
             supports_digital = true;
     }
diff --git a/audio/out/ao_coreaudio_utils.c b/audio/out/ao_coreaudio_utils.c
index b39323ce72..08214ef387 100644
--- a/audio/out/ao_coreaudio_utils.c
+++ b/audio/out/ao_coreaudio_utils.c
@@ -130,14 +130,15 @@ bool check_ca_st(struct ao *ao, int level, OSStatus code, const char *message)
 void ca_fill_asbd(struct ao *ao, AudioStreamBasicDescription *asbd)
 {
     asbd->mSampleRate       = ao->samplerate;
-    asbd->mFormatID         = AF_FORMAT_IS_AC3(ao->format) ?
+    // Set "AC3" for other spdif formats too - unknown if that works.
+    asbd->mFormatID         = AF_FORMAT_IS_IEC61937(ao->format) ?
                               kAudioFormat60958AC3 :
                               kAudioFormatLinearPCM;
     asbd->mChannelsPerFrame = ao->channels.num;
     asbd->mBitsPerChannel   = af_fmt2bits(ao->format);
     asbd->mFormatFlags      = kAudioFormatFlagIsPacked;
 
-    if ((ao->format & AF_FORMAT_POINT_MASK) == AF_FORMAT_F)
+    if ((ao->format & AF_FORMAT_TYPE_MASK) == AF_FORMAT_F)
         asbd->mFormatFlags |= kAudioFormatFlagIsFloat;
 
     if ((ao->format & AF_FORMAT_SIGN_MASK) == AF_FORMAT_SI)
diff --git a/audio/out/ao_dsound.c b/audio/out/ao_dsound.c
index 78b39b17b9..cd6a986831 100644
--- a/audio/out/ao_dsound.c
+++ b/audio/out/ao_dsound.c
@@ -382,21 +382,20 @@ static int init(struct ao *ao)
     int format = af_fmt_from_planar(ao->format);
     int rate = ao->samplerate;
 
-    if (AF_FORMAT_IS_AC3(format))
-        format = AF_FORMAT_AC3;
-    else {
+    if (!AF_FORMAT_IS_IEC61937(format)) {
         struct mp_chmap_sel sel = {0};
         mp_chmap_sel_add_waveext(&sel);
         if (!ao_chmap_sel_adjust(ao, &sel, &ao->channels))
             return -1;
     }
     switch (format) {
-    case AF_FORMAT_AC3:
     case AF_FORMAT_S24:
     case AF_FORMAT_S16:
     case AF_FORMAT_U8:
         break;
     default:
+        if (AF_FORMAT_IS_IEC61937(format))
+            break;
         MP_VERBOSE(ao, "format %s not supported defaulting to Signed 16-bit Little-Endian\n",
                    af_fmt_to_str(format));
         format = AF_FORMAT_S16;
@@ -417,7 +416,8 @@ static int init(struct ao *ao)
                     ? sizeof(WAVEFORMATEXTENSIBLE) - sizeof(WAVEFORMATEX) : 0;
     wformat.Format.nChannels = ao->channels.num;
     wformat.Format.nSamplesPerSec = rate;
-    if (AF_FORMAT_IS_AC3(format)) {
+    if (AF_FORMAT_IS_IEC61937(format)) {
+        // Whether it also works with e.g. DTS is unknown, but probably does.
         wformat.Format.wFormatTag = WAVE_FORMAT_DOLBY_AC3_SPDIF;
         wformat.Format.wBitsPerSample = 16;
         wformat.Format.nBlockAlign = 4;
diff --git a/audio/out/ao_oss.c b/audio/out/ao_oss.c
index de71017432..7780a08978 100644
--- a/audio/out/ao_oss.c
+++ b/audio/out/ao_oss.c
@@ -134,16 +134,13 @@ static const int format_table[][2] = {
 #ifdef AFMT_FLOAT
     {AFMT_FLOAT,        AF_FORMAT_FLOAT},
 #endif
-    // SPECIALS
-#ifdef AFMT_MPEG
-    {AFMT_MPEG,         AF_FORMAT_MPEG2},
-#endif
-#ifdef AFMT_AC3
-    {AFMT_AC3,          AF_FORMAT_AC3},
-#endif
     {-1, -1}
 };
 
+#ifndef AFMT_AC3
+#define AFMT_AC3 -1
+#endif
+
 static int format2oss(int format)
 {
     for (int n = 0; format_table[n][0] != -1; n++) {
@@ -204,7 +201,7 @@ static int control(struct ao *ao, enum aocontrol cmd, void *arg)
             return CONTROL_OK;
 #endif
 
-        if (AF_FORMAT_IS_AC3(ao->format))
+        if (AF_FORMAT_IS_SPECIAL(ao->format))
             return CONTROL_TRUE;
 
         if ((fd = open(p->oss_mixer_device, O_RDONLY)) != -1) {
@@ -293,14 +290,16 @@ static int reopen_device(struct ao *ao, bool allow_format_changes)
     fcntl(p->audio_fd, F_SETFD, FD_CLOEXEC);
 #endif
 
-    if (AF_FORMAT_IS_AC3(format)) {
+    if (AF_FORMAT_IS_IEC61937(format)) {
         ioctl(p->audio_fd, SNDCTL_DSP_SPEED, &samplerate);
     }
 
 ac3_retry:
-    if (AF_FORMAT_IS_AC3(format))
-        format = AF_FORMAT_AC3;
-    oss_format = format2oss(format);
+    if (AF_FORMAT_IS_IEC61937(format)) {
+        oss_format = format2oss(format);
+    } else {
+        oss_format = AFMT_AC3;
+    }
     if (oss_format == -1) {
         MP_VERBOSE(ao, "Unknown/not supported internal format: %s\n",
                    af_fmt_to_str(format));
@@ -345,7 +344,7 @@ ac3_retry:
 
     MP_VERBOSE(ao, "sample format: %s\n", af_fmt_to_str(format));
 
-    if (!AF_FORMAT_IS_AC3(format)) {
+    if (!AF_FORMAT_IS_IEC61937(format)) {
         struct mp_chmap_sel sel = {0};
         for (int n = 0; n < MP_NUM_CHANNELS + 1; n++)
             mp_chmap_sel_add_map(&sel, &oss_layouts[n]);
diff --git a/audio/out/ao_pcm.c b/audio/out/ao_pcm.c
index eb089c6c42..8357174f95 100644
--- a/audio/out/ao_pcm.c
+++ b/audio/out/ao_pcm.c
@@ -137,10 +137,10 @@ static int init(struct ao *ao)
         case AF_FORMAT_S24:
         case AF_FORMAT_S32:
         case AF_FORMAT_FLOAT:
-        case AF_FORMAT_AC3:
              break;
         default:
-            ao->format = AF_FORMAT_S16;
+            if (!AF_FORMAT_IS_IEC61937(ao->format))
+                ao->format = AF_FORMAT_S16;
             break;
         }
     }
diff --git a/audio/out/ao_wasapi_utils.c b/audio/out/ao_wasapi_utils.c
index c39eb6fea2..bf59d7802b 100755
--- a/audio/out/ao_wasapi_utils.c
+++ b/audio/out/ao_wasapi_utils.c
@@ -300,7 +300,7 @@ static int find_formats(struct ao *const ao)
 {
     struct wasapi_state *state = (struct wasapi_state *)ao->priv;
 
-    if (AF_FORMAT_IS_IEC61937(ao->format) || ao->format == AF_FORMAT_MPEG2) {
+    if (AF_FORMAT_IS_IEC61937(ao->format)) {
         if (try_passthrough(state, ao))
             return 0;