ao_coreaudio: fix 7.1(rear) channel mapping

I can't explain this, but it seems to be a similar case to the ALSA HDMI
one. I find it hard to tell because of the slightly different names and
conventions in use in libavcodec, WAVEEXT channel masks, decoders, codec
specifications, HDMI, and platform audio APIs.

The fix is the same as the one for ao_alsa (see commit be49da72). This
should fix at least playing 7.1 sources on OSX with 7.1(rear) selected
in Audio MIDI Setup. The ao_alsa commit mentions XBMC, but I couldn't
find out where it does that or if it also does that for CoreAudio. It's
woth noting that PHT (essentially an old XBMC fork) also exhibited the
incorrect behavior (i.e. side and back speakers were swapped).

1 files changed, 27 insertions, 0 deletions

diff --git a/audio/out/ao_coreaudio_chmap.c b/audio/out/ao_coreaudio_chmap.c
index 2b64d21aae..3db2bdf3d5 100644
--- a/audio/out/ao_coreaudio_chmap.c
+++ b/audio/out/ao_coreaudio_chmap.c
@@ -133,6 +133,29 @@ coreaudio_error:
     return NULL;
 }
 
+
+#define CHMAP(n, ...) &(struct mp_chmap) MP_CONCAT(MP_CHMAP, n) (__VA_ARGS__)
+
+// Replace each channel in a with b (a->num == b->num)
+static void replace_submap(struct mp_chmap *dst, struct mp_chmap *a,
+                           struct mp_chmap *b)
+{
+    struct mp_chmap t = *dst;
+    if (!mp_chmap_is_valid(&t) || mp_chmap_diffn(a, &t) != 0)
+        return;
+    assert(a->num == b->num);
+    for (int n = 0; n < t.num; n++) {
+        for (int i = 0; i < a->num; i++) {
+            if (t.speaker[n] == a->speaker[i]) {
+                t.speaker[n] = b->speaker[i];
+                break;
+            }
+        }
+    }
+    if (mp_chmap_is_valid(&t))
+        *dst = t;
+}
+
 static bool ca_layout_to_mp_chmap(struct ao *ao, AudioChannelLayout *layout,
                                   struct mp_chmap *chmap)
 {
@@ -163,6 +186,10 @@ static bool ca_layout_to_mp_chmap(struct ao *ao, AudioChannelLayout *layout,
         chmap->speaker[n] = speaker;
     }
 
+    // Remap weird 7.1(rear) layouts correctly.
+    replace_submap(chmap, CHMAP(6, FL, FR, BL, BR, SDL, SDR),
+                          CHMAP(6, FL, FR, SL, SR, BL,  BR));
+
     talloc_free(talloc_ctx);
     MP_VERBOSE(ao, "mp chmap: %s\n", mp_chmap_to_str(chmap));
     return mp_chmap_is_valid(chmap) && !mp_chmap_is_unknown(chmap);