af_lavcac3enc, encode: support planar formats

This fixes operation with current ffmpeg releases.

Note that this planarization is slow and should be reverted once proper
planar audio support is there in mpv.

4 files changed, 138 insertions, 7 deletions

diff --git a/audio/filter/af_lavcac3enc.c b/audio/filter/af_lavcac3enc.c
index b54f5bf61e..2b7a4ffb4c 100644
--- a/audio/filter/af_lavcac3enc.c
+++ b/audio/filter/af_lavcac3enc.c
@@ -47,6 +47,7 @@ const uint16_t ac3_bitrate_tab[19] = {
 typedef struct af_ac3enc_s {
     struct AVCodec        *lavc_acodec;
     struct AVCodecContext *lavc_actx;
+    bool planarize;
     int add_iec61937_header;
     int bit_rate;
     int pending_data_size;
@@ -232,8 +233,19 @@ static struct mp_audio* play(struct af_instance* af, struct mp_audio* data)
                                     c->nch,
                                     s->expect_len / samplesize, samplesize);
 
-            len = avcodec_encode_audio(s->lavc_actx, dest, destsize,
-                                       (void *)s->pending_data);
+            void *data = (void *) s->pending_data;
+            if (s->planarize) {
+                void *data2 = malloc(s->expect_len);
+                reorder_to_planar(data2, data, samplesize,
+                        c->nch, s->expect_len / samplesize / c->nch);
+                data = data2;
+            }
+
+            len = avcodec_encode_audio(s->lavc_actx, dest, destsize, data);
+
+            if (s->planarize)
+                free(data);
+
             s->pending_len = 0;
         }
         else {
@@ -243,7 +255,20 @@ static struct mp_audio* play(struct af_instance* af, struct mp_audio* data)
                                     AF_CHANNEL_LAYOUT_LAVC_DEFAULT,
                                     c->nch,
                                     s->expect_len / samplesize, samplesize);
-            len = avcodec_encode_audio(s->lavc_actx,dest,destsize,(void *)src);
+
+            void *data = (void *) src;
+            if (s->planarize) {
+                void *data2 = malloc(s->expect_len);
+                reorder_to_planar(data2, data, samplesize,
+                        c->nch, s->expect_len / samplesize / c->nch);
+                data = data2;
+            }
+
+            len = avcodec_encode_audio(s->lavc_actx, dest, destsize, data);
+
+            if (s->planarize)
+                free(data);
+
             src += s->expect_len;
             left -= s->expect_len;
         }
@@ -305,10 +330,22 @@ static int af_open(struct af_instance* af){
         } else if (fmts[i] == AV_SAMPLE_FMT_S16) {
             s->in_sampleformat = AF_FORMAT_S16_NE;
             s->lavc_actx->sample_fmt = fmts[i];
+            s->planarize = 0;
             break;
         } else if (fmts[i] == AV_SAMPLE_FMT_FLT) {
             s->in_sampleformat = AF_FORMAT_FLOAT_NE;
             s->lavc_actx->sample_fmt = fmts[i];
+            s->planarize = 0;
+            break;
+        } else if (fmts[i] == AV_SAMPLE_FMT_S16P) {
+            s->in_sampleformat = AF_FORMAT_S16_NE;
+            s->lavc_actx->sample_fmt = fmts[i];
+            s->planarize = 1;
+            break;
+        } else if (fmts[i] == AV_SAMPLE_FMT_FLTP) {
+            s->in_sampleformat = AF_FORMAT_FLOAT_NE;
+            s->lavc_actx->sample_fmt = fmts[i];
+            s->planarize = 1;
             break;
         }
     }
@@ -319,6 +356,10 @@ static int af_open(struct af_instance* af){
         af_fmt2bits(s->in_sampleformat) / 8;
     s->pending_data = malloc(s->pending_data_size);
 
+    if (s->planarize)
+        mp_msg(MSGT_AFILTER, MSGL_WARN,
+                "[af_lavcac3enc]: need to planarize audio data\n");
+
     return AF_OK;
 }
 
diff --git a/audio/out/ao_lavc.c b/audio/out/ao_lavc.c
index c2fd2e6d63..0a648e1b6b 100644
--- a/audio/out/ao_lavc.c
+++ b/audio/out/ao_lavc.c
@@ -46,6 +46,7 @@ struct priv {
     uint8_t *buffer;
     size_t buffer_size;
     AVStream *stream;
+    bool planarize;
     int pcmhack;
     int aframesize;
     int aframecount;
@@ -112,27 +113,32 @@ static int init(struct ao *ao, char *params)
              ++sampleformat) {
             switch (*sampleformat) {
             case AV_SAMPLE_FMT_U8:
+            case AV_SAMPLE_FMT_U8P:
                 if (ao->format == AF_FORMAT_U8)
                     goto out_search;
                 break;
             case AV_SAMPLE_FMT_S16:
+            case AV_SAMPLE_FMT_S16P:
                 if (ao->format == AF_FORMAT_S16_BE)
                     goto out_search;
                 if (ao->format == AF_FORMAT_S16_LE)
                     goto out_search;
                 break;
             case AV_SAMPLE_FMT_S32:
+            case AV_SAMPLE_FMT_S32P:
                 if (ao->format == AF_FORMAT_S32_BE)
                     goto out_search;
                 if (ao->format == AF_FORMAT_S32_LE)
                     goto out_search;
                 break;
             case AV_SAMPLE_FMT_FLT:
+            case AV_SAMPLE_FMT_FLTP:
                 if (ao->format == AF_FORMAT_FLOAT_BE)
                     goto out_search;
                 if (ao->format == AF_FORMAT_FLOAT_LE)
                     goto out_search;
                 break;
+            // FIXME do we need support for AV_SAMPLE_FORMAT_DBL/DBLP?
             default:
                 break;
             }
@@ -151,17 +157,22 @@ out_search:
              ++sampleformat) {
             switch (*sampleformat) {
             case AV_SAMPLE_FMT_U8:
+            case AV_SAMPLE_FMT_U8P:
                 ao->format = AF_FORMAT_U8;
                 goto out_takefirst;
             case AV_SAMPLE_FMT_S16:
+            case AV_SAMPLE_FMT_S16P:
                 ao->format = AF_FORMAT_S16_NE;
                 goto out_takefirst;
             case AV_SAMPLE_FMT_S32:
+            case AV_SAMPLE_FMT_S32P:
                 ao->format = AF_FORMAT_S32_NE;
                 goto out_takefirst;
             case AV_SAMPLE_FMT_FLT:
+            case AV_SAMPLE_FMT_FLTP:
                 ao->format = AF_FORMAT_FLOAT_NE;
                 goto out_takefirst;
+            // FIXME do we need support for AV_SAMPLE_FORMAT_DBL/DBLP?
             default:
                 break;
             }
@@ -204,6 +215,32 @@ out_takefirst:
         break;
     }
 
+    // detect if we have to planarize
+    ac->planarize = false;
+    {
+        bool found_format = false;
+        bool found_alternate_format = false;
+        for (sampleformat = codec->sample_fmts;
+             sampleformat && *sampleformat != AV_SAMPLE_FMT_NONE;
+             ++sampleformat) {
+            if (*sampleformat == ac->stream->codec->sample_fmt)
+                found_format = true;
+            if (*sampleformat ==
+                    av_get_alt_sample_fmt(ac->stream->codec->sample_fmt, 1))
+                found_alternate_format = true;
+        }
+        if (!found_format && found_alternate_format) {
+            ac->stream->codec->sample_fmt =
+                av_get_alt_sample_fmt(ac->stream->codec->sample_fmt, 1);
+            ac->planarize = true;
+        }
+        if (!found_format && !found_alternate_format) {
+            // shouldn't happen
+            mp_msg(MSGT_ENCODE, MSGL_ERR,
+                   "ao-lavc: sample format not found\n");
+        }
+    }
+
     ac->stream->codec->bits_per_raw_sample = ac->sample_size * 8;
 
     switch (ao->channels) {
@@ -274,6 +311,10 @@ out_takefirst:
     ao->untimed = true;
     ao->priv = ac;
 
+    if (ac->planarize)
+        mp_msg(MSGT_ENCODE, MSGL_WARN,
+                "ao-lavc: need to planarize audio data\n");
+
     return 0;
 }
 
@@ -297,7 +338,7 @@ static void uninit(struct ao *ao, bool cut_audio)
 
     if (!encode_lavc_start(ectx)) {
         mp_msg(MSGT_ENCODE, MSGL_WARN,
-		"ao-lavc: not even ready to encode audio at end -> dropped");
+                "ao-lavc: not even ready to encode audio at end -> dropped");
         return;
     }
 
@@ -357,8 +398,14 @@ static int encode(struct ao *ao, double apts, void *data)
     {
         frame = avcodec_alloc_frame();
         frame->nb_samples = ac->aframesize;
-        if(avcodec_fill_audio_frame(frame, ao->channels, ac->stream->codec->sample_fmt, data, ac->aframesize * ao->channels * ac->sample_size, 1))
-        {
+
+        if (ac->planarize) {
+            void *data2 = talloc_size(ao, ac->aframesize * ao->channels * ac->sample_size);
+            reorder_to_planar(data2, data, ac->sample_size, ao->channels, ac->aframesize);
+            data = data2;
+        }
+
+        if (avcodec_fill_audio_frame(frame, ao->channels, ac->stream->codec->sample_fmt, data, ac->aframesize * ao->channels * ac->sample_size, 1)) {
             mp_msg(MSGT_ENCODE, MSGL_ERR, "ao-lavc: error filling\n");
             return -1;
         }
@@ -392,6 +439,11 @@ static int encode(struct ao *ao, double apts, void *data)
         }
 
         avcodec_free_frame(&frame);
+
+        if (ac->planarize) {
+            talloc_free(data);
+            data = NULL;
+        }
     }
     else
     {
diff --git a/audio/reorder_ch.c b/audio/reorder_ch.c
index 4b51a79439..330a807eae 100644
--- a/audio/reorder_ch.c
+++ b/audio/reorder_ch.c
@@ -1368,7 +1368,6 @@ void reorder_channel_nch(void *buf,
                         samples, samplesize);
 }
 
-
 #ifdef TEST
 
 static void test_copy(int channels) {
@@ -1398,3 +1397,38 @@ int main(int argc, char *argv[]) {
 }
 
 #endif
+
+static inline void reorder_to_planar_(void *restrict out, const void *restrict in,
+        size_t size, size_t nchan, size_t nmemb) {
+    size_t i, c;
+    char *outptr = (char *) out;
+    size_t instep = nchan * size;
+
+    for (c = 0; c < nchan; ++c) {
+        const char *inptr = ((const char *) in) + c * size;
+        for (i = 0; i < nmemb; ++i, inptr += instep, outptr += size) {
+            memcpy(outptr, inptr, size);
+        }
+    }
+}
+
+void reorder_to_planar(void *restrict out, const void *restrict in,
+        size_t size, size_t nchan, size_t nmemb)
+{
+    // special case for mono (nothing to do...)
+    if (nchan == 1)
+        memcpy(out, in, size * nchan * nmemb);
+    // these calls exist to insert an inline copy of to_planar_ here with known
+    // value of size to help the compiler replace the memcpy calls by mov
+    // instructions
+    else if (size == 1)
+        reorder_to_planar_(out, in, 1, nchan, nmemb);
+    else if (size == 2)
+        reorder_to_planar_(out, in, 2, nchan, nmemb);
+    else if (size == 4)
+        reorder_to_planar_(out, in, 4, nchan, nmemb);
+    // general case (calls memcpy a lot, should actually never happen, but
+    // stays here for correctness purposes)
+    else
+        reorder_to_planar_(out, in, size, nchan, nmemb);
+}
diff --git a/audio/reorder_ch.h b/audio/reorder_ch.h
index 44b533988d..66354403da 100644
--- a/audio/reorder_ch.h
+++ b/audio/reorder_ch.h
@@ -130,4 +130,8 @@ void reorder_channel_nch(void *buf,
                          int samples,
                          int samplesize);
 
+/// Utility function for planar audio conversions
+void reorder_to_planar(void *restrict out, const void *restrict in,
+        size_t size, size_t nchan, size_t nmemb);
+
 #endif /* MPLAYER_REORDER_CH_H */