custom quantization matrix for x264, original patch by Robert Swain < robert POUM swain AH gmail POUM com>

Lots of nits and improvement by the MPlayer team
Original thread:
Date: Jul 12, 2005 5:04 PM
Subject: [MPlayer-dev-eng] [PATCH] CQMs in x264


git-svn-id: svn://svn.mplayerhq.hu/mplayer/trunk@16367 b3059339-0415-0410-9bf9-f77b7e298cf2

3 files changed, 120 insertions, 0 deletions

diff --git a/ChangeLog b/ChangeLog
index 5cc8b4acf9..c59e65ad90 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -72,6 +72,7 @@ MPlayer (1.0)
     * raw audio muxer
     * fixed various bugs in the EDL code
     * x264 "turbo mode" to speed up first pass of multi-pass encoding
+    * x264 custom quantization matrices
     * -delay allows real audio delay instead of just a delay in the header
 
     Ports:
diff --git a/DOCS/man/en/mplayer.1 b/DOCS/man/en/mplayer.1
index 2df8a92983..f513f1e016 100644
--- a/DOCS/man/en/mplayer.1
+++ b/DOCS/man/en/mplayer.1
@@ -8466,6 +8466,62 @@ Use a different quantizer for chroma as compared to luma.
 Useful values are in the range <-2\-2> (default: 0).
 .
 .TP
+.B cqm=<flat|jvt|<filename>>
+Either uses a predefined custom quantization matrix or loads a JM format
+matrix file.
+.PD 0
+.RSs
+.IPs flat\ 
+Use the predefined flat 16 matrix (default).
+.IPs jvt\ \ 
+Use the predefined JVT matrix.
+.IPs <filename>
+Use the provided JM format matrix file.
+.PD 1
+.RE
+.
+.TP
+.B cqm4iy=<list>
+custom 4x4 intra luminance matrix, given as a list of 16 comma separated
+values in the range 1-255.
+.
+.TP
+.B cqm4ic=<list>
+custom 4x4 intra chrominance matrix, given as a list of 16 comma
+separated values in the range 1-255.
+.
+.TP
+.B cqm4py=<list>
+custom 4x4 inter luminance matrix, given as a list of 16 comma separated
+values in the range 1-255.
+.
+.TP
+.B cqm4pc=<list>
+custom 4x4 inter chrominance matrix, given as a list of 16 comma
+separated values in the range 1-255.
+.
+.TP
+.B cqm8iy=<list>
+custom 8x8 intra luminance matrix, given as a list of 64 comma separated
+values in the range 1-255.
+.
+.TP
+.B cqm8py=<list>
+custom 8x8 inter luminance matrix, given as a list of 64 comma separated
+values in the range 1-255.
+.RE
+.I NOTES:
+Files encoded using CQMs are not currently decodable by FFmpeg based
+players.
+.br
+Windows CMD.EXE users may experience problems with parsing the command line
+if they attempt to use all the CQM lists.
+This is due to a command line length limitation.
+In this case it is recommended the lists be put into a JM format CQM
+file and loaded as specified above.
+.RE
+.
+.TP
 .B level_idc=<10\-51>
 Set the bitstream's Level as defined by Annex A of the H.264 standard
 (default: 40 - Level 4.0).
diff --git a/libmpcodecs/ve_x264.c b/libmpcodecs/ve_x264.c
index c79bea46ba..2dd263d9ea 100644
--- a/libmpcodecs/ve_x264.c
+++ b/libmpcodecs/ve_x264.c
@@ -106,6 +106,13 @@ static int psnr = 0;
 static int log_level = 2;
 static int turbo = 0;
 static int visualize = 0;
+static char *cqm = NULL;
+static char *cqm4iy = NULL;
+static char *cqm4ic = NULL;
+static char *cqm4py = NULL;
+static char *cqm4pc = NULL;
+static char *cqm8iy = NULL;
+static char *cqm8py = NULL;
 
 m_option_t x264encopts_conf[] = {
     {"bitrate", &bitrate, CONF_TYPE_INT, CONF_RANGE, 0, 24000000, NULL},
@@ -155,6 +162,13 @@ m_option_t x264encopts_conf[] = {
     {"qp_step", &qp_step, CONF_TYPE_INT, CONF_RANGE, 1, 50, NULL},
     {"pass", &pass, CONF_TYPE_INT, CONF_RANGE, 1, 3, NULL},
     {"rc_eq", &rc_eq, CONF_TYPE_STRING, 0, 0, 0, NULL},
+    {"cqm", &cqm, CONF_TYPE_STRING, 0, 0, 0, NULL},
+    {"cqm4iy", &cqm4iy, CONF_TYPE_STRING, 0, 0, 0, NULL},
+    {"cqm4ic", &cqm4ic, CONF_TYPE_STRING, 0, 0, 0, NULL},
+    {"cqm4py", &cqm4py, CONF_TYPE_STRING, 0, 0, 0, NULL},
+    {"cqm4pc", &cqm4pc, CONF_TYPE_STRING, 0, 0, 0, NULL},
+    {"cqm8iy", &cqm8iy, CONF_TYPE_STRING, 0, 0, 0, NULL},
+    {"cqm8py", &cqm8py, CONF_TYPE_STRING, 0, 0, 0, NULL},
     {"qcomp", &qcomp, CONF_TYPE_FLOAT, CONF_RANGE, 0, 1, NULL},
     {"qblur", &qblur, CONF_TYPE_FLOAT, CONF_RANGE, 0, 99, NULL},
     {"cplx_blur", &complexity_blur, CONF_TYPE_FLOAT, CONF_RANGE, 0, 999, NULL},
@@ -173,6 +187,23 @@ m_option_t x264encopts_conf[] = {
     {NULL, NULL, 0, 0, 0, 0, NULL}
 };
 
+static int parse_cqm(const char *str, uint8_t *cqm, int length,
+                     h264_module_t *mod, char *matrix_name) {
+    int i;
+    if (!str) return 0;
+    for (i = 0; i < length; i++) {
+        long coef = strtol(str, &str, 0);
+        if (coef < 1 || coef > 255 || str[0] != ((i + 1 == length)?0:',')) {
+            mp_msg( MSGT_MENCODER, MSGL_ERR, "x264: Invalid entry in cqm%s at position %d.\n", matrix_name, i+1 );
+            return -1;
+        }
+        cqm[i] = coef;
+        str = &str[1];
+    }
+    mod->param.i_cqm_preset = X264_CQM_CUSTOM;
+    return 0;
+}
+
 static int put_image(struct vf_instance_s *vf, mp_image_t *mpi);
 static int encode_frame(struct vf_instance_s *vf, x264_picture_t *pic_in);
 
@@ -263,6 +294,38 @@ static int config(struct vf_instance_s* vf, int width, int height, int d_width,
     mod->param.vui.i_sar_height = d_height*width;
     mod->param.i_threads = threads;
 
+    if(cqm != NULL)
+    {
+        if( !strcmp(cqm, "flat") )
+            mod->param.i_cqm_preset = X264_CQM_FLAT;
+        else if( !strcmp(cqm, "jvt") )
+            mod->param.i_cqm_preset = X264_CQM_JVT;
+        else
+        {
+            FILE *cqm_test;
+            cqm_test = fopen( cqm, "rb" );
+            if( cqm_test )
+            {
+                mod->param.i_cqm_preset = X264_CQM_CUSTOM;
+                mod->param.psz_cqm_file = cqm;
+                fclose( cqm_test );
+            }
+            else
+            {
+                mp_msg( MSGT_MENCODER, MSGL_ERR, "x264: CQM file failed to open.\n" );
+                return 0;
+            }
+        }
+    }
+
+    if( (parse_cqm(cqm4iy, mod->param.cqm_4iy, 16, mod, "4iy") < 0) ||
+        (parse_cqm(cqm4ic, mod->param.cqm_4ic, 16, mod, "4ic") < 0) ||
+        (parse_cqm(cqm4py, mod->param.cqm_4py, 16, mod, "4py") < 0) ||
+        (parse_cqm(cqm4pc, mod->param.cqm_4pc, 16, mod, "4pc") < 0) ||
+        (parse_cqm(cqm8iy, mod->param.cqm_8iy, 64, mod, "8iy") < 0) ||
+        (parse_cqm(cqm8py, mod->param.cqm_8py, 64, mod, "8py") < 0) )
+        return 0;
+
     switch(pass) {
     case 0:
         mod->param.rc.b_stat_write = 0;