audio: remove swapped-endian spdif formats

IEC 61937 frames should always be little endian (little endian 16 bit
words). I don't see any apparent need why the audio chain should handle
swapped-endian formats.

It could be that some audio outputs might want them (especially on big
endian architectures). On the other hand, it's not clear how that works
on these architectures, and it's not even known whether the current code
works on big endian at all. If something should break, and it should
turn out that swapped-endian spdif is needed on any platform/AO,
swapping still could be done in-place within the affected AO, and
there's no need for the additional complexity in the rest of the player.

Note that af_lavcac3enc outputs big endian spdif frames for unknown
reasons. Normally, the resulting data is just pulled through an auto-
inserted conversion filter and turned into little endian. Maybe this was
done as a trick so that the code didn't have to byte-swap the actual
audio frame. In any case, just make it output little endian frames.

All of this is untested, because I have no receiver hardware.

7 files changed, 29 insertions, 37 deletions

diff --git a/audio/decode/ad_spdif.c b/audio/decode/ad_spdif.c
index 528e42fbc4..3c5e0ef6e1 100644
--- a/audio/decode/ad_spdif.c
+++ b/audio/decode/ad_spdif.c
@@ -117,13 +117,13 @@ static int init(struct dec_audio *da, const char *decoder)
     switch (stream->codec->codec_id) {
     case AV_CODEC_ID_AAC:
         spdif_ctx->iec61937_packet_size = 16384;
-        sample_format                   = AF_FORMAT_IEC61937_LE;
+        sample_format                   = AF_FORMAT_IEC61937;
         samplerate                      = 48000;
         num_channels                    = 2;
         break;
     case AV_CODEC_ID_AC3:
         spdif_ctx->iec61937_packet_size = 6144;
-        sample_format                   = AF_FORMAT_AC3_LE;
+        sample_format                   = AF_FORMAT_AC3;
         samplerate                      = 48000;
         num_channels                    = 2;
         break;
@@ -131,19 +131,19 @@ static int init(struct dec_audio *da, const char *decoder)
         if (da->opts->dtshd) {
             av_dict_set(&format_opts, "dtshd_rate", "768000", 0); // 4*192000
             spdif_ctx->iec61937_packet_size = 32768;
-            sample_format                   = AF_FORMAT_IEC61937_LE;
+            sample_format                   = AF_FORMAT_IEC61937;
             samplerate                      = 192000;
             num_channels                    = 2*4;
         } else {
             spdif_ctx->iec61937_packet_size = 32768;
-            sample_format                   = AF_FORMAT_AC3_LE;
+            sample_format                   = AF_FORMAT_AC3;
             samplerate                      = 48000;
             num_channels                    = 2;
         }
         break;
     case AV_CODEC_ID_EAC3:
         spdif_ctx->iec61937_packet_size = 24576;
-        sample_format                   = AF_FORMAT_IEC61937_LE;
+        sample_format                   = AF_FORMAT_IEC61937;
         samplerate                      = 192000;
         num_channels                    = 2;
         break;
@@ -155,7 +155,7 @@ static int init(struct dec_audio *da, const char *decoder)
         break;
     case AV_CODEC_ID_TRUEHD:
         spdif_ctx->iec61937_packet_size = 61440;
-        sample_format                   = AF_FORMAT_IEC61937_LE;
+        sample_format                   = AF_FORMAT_IEC61937;
         samplerate                      = 192000;
         num_channels                    = 8;
         break;
diff --git a/audio/filter/af_lavcac3enc.c b/audio/filter/af_lavcac3enc.c
index 8c34b417af..0763057665 100644
--- a/audio/filter/af_lavcac3enc.c
+++ b/audio/filter/af_lavcac3enc.c
@@ -30,6 +30,7 @@
 #include <libavutil/audioconvert.h>
 #include <libavutil/intreadwrite.h>
 #include <libavutil/common.h>
+#include <libavutil/bswap.h>
 #include <libavutil/mem.h>
 
 #include "common/common.h"
@@ -87,7 +88,7 @@ static int control(struct af_instance *af, int cmd, void *arg)
         if (in->nch > AC3_MAX_CHANNELS)
             mp_audio_set_num_channels(in, AC3_MAX_CHANNELS);
 
-        mp_audio_set_format(af->data, AF_FORMAT_AC3_BE);
+        mp_audio_set_format(af->data, AF_FORMAT_AC3);
         mp_audio_set_num_channels(af->data, 2);
 
         if (!mp_audio_config_equals(in, &orig_in))
@@ -150,6 +151,12 @@ static void uninit(struct af_instance* af)
     }
 }
 
+static void swap_16(uint16_t *ptr, size_t size)
+{
+    for (size_t n = 0; n < size; n++)
+        ptr[n] = av_bswap16(ptr[n]);
+}
+
 // Filter data through filter
 static int filter(struct af_instance* af, struct mp_audio* audio, int flags)
 {
@@ -223,11 +230,11 @@ static int filter(struct af_instance* af, struct mp_audio* audio, int flags)
             frame_size = AC3_FRAME_SIZE * 2 * 2;
             header_len = 8;
 
-            AV_WB16(hdr,     0xF872);   // iec 61937 syncword 1
-            AV_WB16(hdr + 2, 0x4E1F);   // iec 61937 syncword 2
+            AV_WL16(hdr,     0xF872);   // iec 61937 syncword 1
+            AV_WL16(hdr + 2, 0x4E1F);   // iec 61937 syncword 2
             hdr[4] = bsmod;             // bsmod
             hdr[5] = 0x01;              // data-type ac3
-            AV_WB16(hdr + 6, len << 3); // number of bits in payload
+            AV_WL16(hdr + 6, len << 3); // number of bits in payload
         }
 
         size_t max_size = (max_out_samples - out->samples) * out->sstride;
@@ -239,6 +246,7 @@ static int filter(struct af_instance* af, struct mp_audio* audio, int flags)
         memcpy(buf + header_len, s->pkt.data, s->pkt.size);
         memset(buf + header_len + s->pkt.size, 0,
                frame_size - (header_len + s->pkt.size));
+        swap_16((uint16_t *)(buf + header_len), s->pkt.size / 2);
         out->samples += frame_size / out->sstride;
     }
 
diff --git a/audio/format.c b/audio/format.c
index 9b14fd167b..b10e574a9e 100644
--- a/audio/format.c
+++ b/audio/format.c
@@ -119,8 +119,8 @@ bool af_fmt_is_planar(int format)
 
 const struct af_fmt_entry af_fmtstr_table[] = {
     FMT("mpeg2",                AF_FORMAT_MPEG2)
-    FMT_ENDIAN("ac3",           AF_FORMAT_AC3)
-    FMT_ENDIAN("iec61937",      AF_FORMAT_IEC61937)
+    FMT("ac3",                  AF_FORMAT_AC3)
+    FMT("iec61937",             AF_FORMAT_IEC61937)
 
     FMT("u8",                   AF_FORMAT_U8)
     FMT("s8",                   AF_FORMAT_S8)
diff --git a/audio/format.h b/audio/format.h
index c4afe7a428..a14d8fe367 100644
--- a/audio/format.h
+++ b/audio/format.h
@@ -101,12 +101,8 @@ enum af_format {
     AF_FORMAT_DOUBLE_LE = (AF_FORMAT_F|AF_FORMAT_64BIT|AF_FORMAT_LE),
     AF_FORMAT_DOUBLE_BE = (AF_FORMAT_F|AF_FORMAT_64BIT|AF_FORMAT_BE),
 
-    AF_FORMAT_AC3_LE    = (AF_FORMAT_S_AC3|AF_FORMAT_16BIT|AF_FORMAT_LE),
-    AF_FORMAT_AC3_BE    = (AF_FORMAT_S_AC3|AF_FORMAT_16BIT|AF_FORMAT_BE),
-
-    AF_FORMAT_IEC61937_LE = (AF_FORMAT_S_IEC61937|AF_FORMAT_16BIT|AF_FORMAT_LE),
-    AF_FORMAT_IEC61937_BE = (AF_FORMAT_S_IEC61937|AF_FORMAT_16BIT|AF_FORMAT_BE),
-
+    AF_FORMAT_AC3       = (AF_FORMAT_S_AC3|AF_FORMAT_16BIT|AF_FORMAT_LE),
+    AF_FORMAT_IEC61937  = (AF_FORMAT_S_IEC61937|AF_FORMAT_16BIT|AF_FORMAT_LE),
     AF_FORMAT_MPEG2     = (AF_FORMAT_S_MPEG2),
 
     // Planar variants
@@ -126,13 +122,10 @@ enum af_format {
 
     AF_FORMAT_FLOAT     = AF_SELECT_LE_BE(AF_FORMAT_FLOAT_LE, AF_FORMAT_FLOAT_BE),
     AF_FORMAT_DOUBLE    = AF_SELECT_LE_BE(AF_FORMAT_DOUBLE_LE, AF_FORMAT_DOUBLE_BE),
-
-    AF_FORMAT_AC3       = AF_SELECT_LE_BE(AF_FORMAT_AC3_LE, AF_FORMAT_AC3_BE),
-    AF_FORMAT_IEC61937  = AF_SELECT_LE_BE(AF_FORMAT_IEC61937_LE, AF_FORMAT_IEC61937_BE),
 };
 
 #define AF_FORMAT_IS_AC3(fmt) \
-    (((fmt) & AF_FORMAT_SPECIAL_MASK) == AF_FORMAT_S_AC3)
+    ((fmt) == AF_FORMAT_AC3)
 
 #define AF_FORMAT_IS_IEC61937(fmt) \
     (AF_FORMAT_IS_AC3(fmt) ||      \
diff --git a/audio/out/ao_alsa.c b/audio/out/ao_alsa.c
index 6b0c34e320..60b8338f9d 100644
--- a/audio/out/ao_alsa.c
+++ b/audio/out/ao_alsa.c
@@ -222,10 +222,8 @@ static const int mp_to_alsa_format[][2] = {
     {AF_FORMAT_S24_BE,      SND_PCM_FORMAT_S24_3BE},
     {AF_FORMAT_FLOAT_LE,    SND_PCM_FORMAT_FLOAT_LE},
     {AF_FORMAT_FLOAT_BE,    SND_PCM_FORMAT_FLOAT_BE},
-    {AF_FORMAT_AC3_LE,      SND_PCM_FORMAT_S16_LE},
-    {AF_FORMAT_AC3_BE,      SND_PCM_FORMAT_S16_BE},
-    {AF_FORMAT_IEC61937_LE, SND_PCM_FORMAT_S16_LE},
-    {AF_FORMAT_IEC61937_BE, SND_PCM_FORMAT_S16_BE},
+    {AF_FORMAT_AC3,         SND_PCM_FORMAT_S16_LE},
+    {AF_FORMAT_IEC61937,    SND_PCM_FORMAT_S16_LE},
     {AF_FORMAT_MPEG2,       SND_PCM_FORMAT_MPEG},
     {AF_FORMAT_UNKNOWN,     SND_PCM_FORMAT_UNKNOWN},
 };
@@ -421,9 +419,9 @@ static int init(struct ao *ao)
                 af_fmt_to_str(ao->format));
         p->alsa_fmt = SND_PCM_FORMAT_S16_LE;
         if (AF_FORMAT_IS_AC3(ao->format))
-            ao->format = AF_FORMAT_AC3_LE;
+            ao->format = AF_FORMAT_AC3;
         else if (AF_FORMAT_IS_IEC61937(ao->format))
-            ao->format = AF_FORMAT_IEC61937_LE;
+            ao->format = AF_FORMAT_IEC61937;
         else
             ao->format = AF_FORMAT_S16_LE;
     }
diff --git a/audio/out/ao_coreaudio_device.c b/audio/out/ao_coreaudio_device.c
index cf9d312003..3ba1f0ce37 100644
--- a/audio/out/ao_coreaudio_device.c
+++ b/audio/out/ao_coreaudio_device.c
@@ -521,14 +521,7 @@ static int init_digital(struct ao *ao, AudioStreamBasicDescription asbd)
     err = ca_enable_device_listener(p->device, changed);
     CHECK_CA_ERROR("cannot install format change listener during init");
 
-#if BYTE_ORDER == BIG_ENDIAN
-    if (!(p->stream_asdb.mFormatFlags & kAudioFormatFlagIsBigEndian))
-#else
-    /* tell mplayer that we need a byteswap on AC3 streams, */
-    if (p->stream_asbd.mFormatID & kAudioFormat60958AC3)
-        ao->format = AF_FORMAT_AC3_LE;
-    else if (p->stream_asbd.mFormatFlags & kAudioFormatFlagIsBigEndian)
-#endif
+    if (p->stream_asdb.mFormatFlags & kAudioFormatFlagIsBigEndian)
         MP_WARN(ao, "stream has non-native byte order, output may fail\n");
 
     ao->samplerate = p->stream_asbd.mSampleRate;
diff --git a/audio/out/ao_pcm.c b/audio/out/ao_pcm.c
index ee7a5eb1ba..3c1f46409a 100644
--- a/audio/out/ao_pcm.c
+++ b/audio/out/ao_pcm.c
@@ -130,7 +130,7 @@ static int init(struct ao *ao)
         case AF_FORMAT_S24_LE:
         case AF_FORMAT_S32_LE:
         case AF_FORMAT_FLOAT_LE:
-        case AF_FORMAT_AC3_LE:
+        case AF_FORMAT_AC3:
              break;
         default:
             ao->format = AF_FORMAT_S16_LE;