vo_opengl: mess with PixelStorei state management

This is needed for GLES 2 support. GLES 2 doesn't support
GL_UNPACK_ROW_LENGTH, and we shouldn't even use this constant, since a
GLES implementation could raise an error.

So set it only if neccessary, and leave it in the default state
otherwise. This also smuggles in a ES 2 fallback for glUploadTex(),
and querying an extension relevant for ES 2. For the alignment state
(GL_[UN]PACK_ALIGNMENT) do the same for symmetry. All 4 states
(alignment/rows x pack/unpack) are now assumed to be in their initial
states by default.

Also redo the PixelStorei handling in the function table. I could rebase
this, but look at the commit time.

3 files changed, 49 insertions, 44 deletions

diff --git a/video/out/gl_common.c b/video/out/gl_common.c
index 7389d90370..aa5cd763ae 100644
--- a/video/out/gl_common.c
+++ b/video/out/gl_common.c
@@ -72,28 +72,21 @@ void glCheckError(GL *gl, struct mp_log *log, const char *info)
     }
 }
 
-//! \defgroup glcontext OpenGL context management helper functions
-
-//! \defgroup gltexture OpenGL texture handling helper functions
-
-//! \defgroup glconversion OpenGL conversion helper functions
+static int get_alignment(int stride)
+{
+    if (stride % 8 == 0)
+        return 8;
+    if (stride % 4 == 0)
+        return 4;
+    if (stride % 2 == 0)
+        return 2;
+    return 1;
+}
 
-/**
- * \brief adjusts the GL_UNPACK_ALIGNMENT to fit the stride.
- * \param stride number of bytes per line for which alignment should fit.
- * \ingroup glgeneral
- */
+// adjusts the GL_UNPACK_ALIGNMENT to fit the stride.
 void glAdjustAlignment(GL *gl, int stride)
 {
-    GLint gl_alignment;
-    if (stride % 8 == 0)
-        gl_alignment = 8;
-    else if (stride % 4 == 0)
-        gl_alignment = 4;
-    else if (stride % 2 == 0)
-        gl_alignment = 2;
-    else
-        gl_alignment = 1;
+    GLint gl_alignment = get_alignment(stride);
     gl->PixelStorei(GL_UNPACK_ALIGNMENT, gl_alignment);
     gl->PixelStorei(GL_PACK_ALIGNMENT, gl_alignment);
 }
@@ -190,6 +183,7 @@ static const struct gl_functions gl_functions[] = {
             DEF_FN(GetError),
             DEF_FN(GetIntegerv),
             DEF_FN(GetString),
+            DEF_FN(PixelStorei),
             DEF_FN(ReadPixels),
             DEF_FN(TexImage2D),
             DEF_FN(TexParameteri),
@@ -203,6 +197,7 @@ static const struct gl_functions gl_functions[] = {
     // GL 1.1+ desktop only
     {
         .ver_core = 110,
+        .provides = MPGL_CAP_ROW_LENGTH,
         .functions = (const struct gl_function[]) {
             DEF_FN(DrawBuffer),
             DEF_FN(GetTexImage),
@@ -210,9 +205,6 @@ static const struct gl_functions gl_functions[] = {
             DEF_FN(ReadBuffer),
             DEF_FN(TexEnvi),
             DEF_FN(TexImage1D),
-            // This is actually in ES 2.0, but quite useless, because it doesn't
-            // support GL_[UN]PACK_ROW_LENGTH.
-            DEF_FN(PixelStorei),
             {0}
         },
     },
@@ -282,11 +274,16 @@ static const struct gl_functions gl_functions[] = {
             // for ES 3.0
             DEF_FN(ReadBuffer),
             DEF_FN(UnmapBuffer),
-            // ES 3.0 adds support for GL_[UN]PACK_ROW_LENGTH.
-            DEF_FN(PixelStorei),
             {0}
         },
     },
+    // Useful for ES 2.0
+    {
+        .ver_core = 110,
+        .ver_es_core = 300,
+        .extension = "GL_EXT_unpack_subimage",
+        .provides = MPGL_CAP_ROW_LENGTH,
+    },
     // Framebuffers, extension in GL 2.x, core in GL 3.x core.
     {
         .ver_core = 300,
@@ -754,15 +751,24 @@ void glUploadTex(GL *gl, GLenum target, GLenum format, GLenum type,
         data += (h - 1) * stride;
         stride = -stride;
     }
-    // this is not always correct, but should work for MPlayer
-    glAdjustAlignment(gl, stride);
-    gl->PixelStorei(GL_UNPACK_ROW_LENGTH, stride / glFmt2bpp(format, type));
+    gl->PixelStorei(GL_UNPACK_ALIGNMENT, get_alignment(stride));
+    bool use_rowlength = slice > 1 && (gl->mpgl_caps & MPGL_CAP_ROW_LENGTH);
+    if (use_rowlength) {
+        // this is not always correct, but should work for MPlayer
+        gl->PixelStorei(GL_UNPACK_ROW_LENGTH, stride / glFmt2bpp(format, type));
+    } else {
+        if (stride != glFmt2bpp(format, type) * w)
+            slice = 1; // very inefficient, but at least it works
+    }
     for (; y + slice <= y_max; y += slice) {
         gl->TexSubImage2D(target, 0, x, y, w, slice, format, type, data);
         data += stride * slice;
     }
     if (y < y_max)
         gl->TexSubImage2D(target, 0, x, y, w, y_max - y, format, type, data);
+    if (use_rowlength)
+        gl->PixelStorei(GL_UNPACK_ROW_LENGTH, 0);
+    gl->PixelStorei(GL_UNPACK_ALIGNMENT, 4);
 }
 
 // Like glUploadTex, but upload a byte array with all elements set to val.
@@ -780,9 +786,9 @@ void glClearTex(GL *gl, GLenum target, GLenum format, GLenum type,
     if (talloc_get_size(data) < size)
         data = talloc_realloc(NULL, data, char *, size);
     memset(data, val, size);
-    glAdjustAlignment(gl, stride);
-    gl->PixelStorei(GL_UNPACK_ROW_LENGTH, w);
+    gl->PixelStorei(GL_UNPACK_ALIGNMENT, get_alignment(stride));
     gl->TexSubImage2D(target, 0, x, y, w, h, format, type, data);
+    gl->PixelStorei(GL_UNPACK_ALIGNMENT, 4);
     if (scratch) {
         *scratch = data;
     } else {
@@ -804,10 +810,13 @@ void glDownloadTex(GL *gl, GLenum target, GLenum format, GLenum type,
 {
     if (!gl->GetTexImage)
         abort();
+    assert(gl->mpgl_caps & MPGL_CAP_ROW_LENGTH);
     // this is not always correct, but should work for MPlayer
-    glAdjustAlignment(gl, stride);
+    gl->PixelStorei(GL_PACK_ALIGNMENT, get_alignment(stride));
     gl->PixelStorei(GL_PACK_ROW_LENGTH, stride / glFmt2bpp(format, type));
     gl->GetTexImage(target, 0, format, type, dataptr);
+    gl->PixelStorei(GL_PACK_ROW_LENGTH, 0);
+    gl->PixelStorei(GL_PACK_ALIGNMENT, 4);
 }
 
 mp_image_t *glGetWindowScreenshot(GL *gl)
@@ -820,7 +829,6 @@ mp_image_t *glGetWindowScreenshot(GL *gl)
     if (!image)
         return NULL;
     gl->PixelStorei(GL_PACK_ALIGNMENT, 1);
-    gl->PixelStorei(GL_PACK_ROW_LENGTH, 0);
     gl->ReadBuffer(GL_FRONT);
     //flip image while reading (and also avoid stride-related trouble)
     for (int y = 0; y < vp[3]; y++) {
@@ -828,6 +836,7 @@ mp_image_t *glGetWindowScreenshot(GL *gl)
                        GL_RGB, GL_UNSIGNED_BYTE,
                        image->planes[0] + y * image->stride[0]);
     }
+    gl->PixelStorei(GL_PACK_ALIGNMENT, 4);
     return image;
 }
 
diff --git a/video/out/gl_common.h b/video/out/gl_common.h
index 41335e2706..3f0f999eaa 100644
--- a/video/out/gl_common.h
+++ b/video/out/gl_common.h
@@ -70,6 +70,7 @@ mp_image_t *glGetWindowScreenshot(GL *gl);
 enum {
     MPGL_CAP_GL_LEGACY          = (1 << 1),     // GL 1.1 (excluding 3.x)
     MPGL_CAP_GL21               = (1 << 3),     // GL 2.1+ (excluding legacy)
+    MPGL_CAP_ROW_LENGTH         = (1 << 4),     // GL_[UN]PACK_ROW_LENGTH
     MPGL_CAP_FB                 = (1 << 5),
     MPGL_CAP_VAO                = (1 << 6),
     MPGL_CAP_SRGB_TEX           = (1 << 7),
diff --git a/video/out/gl_video.c b/video/out/gl_video.c
index 5fafcd04f1..8d4682bfe7 100644
--- a/video/out/gl_video.c
+++ b/video/out/gl_video.c
@@ -1249,8 +1249,6 @@ static void init_scaler(struct gl_video *p, struct scaler *scaler)
         gl->GenTextures(1, &scaler->gl_lut);
 
     gl->BindTexture(GL_TEXTURE_2D, scaler->gl_lut);
-    gl->PixelStorei(GL_UNPACK_ALIGNMENT, 4);
-    gl->PixelStorei(GL_UNPACK_ROW_LENGTH, 0);
 
     float *weights = talloc_array(NULL, float, LOOKUP_TEXTURE_SIZE * size);
     mp_compute_lut(scaler->kernel, LOOKUP_TEXTURE_SIZE, weights);
@@ -1329,13 +1327,13 @@ static void init_dither(struct gl_video *p)
     gl->GenTextures(1, &p->dither_texture);
     gl->BindTexture(GL_TEXTURE_2D, p->dither_texture);
     gl->PixelStorei(GL_UNPACK_ALIGNMENT, 1);
-    gl->PixelStorei(GL_UNPACK_ROW_LENGTH, 0);
     gl->TexImage2D(GL_TEXTURE_2D, 0, tex_iformat, tex_size, tex_size, 0,
                    tex_format, tex_type, tex_data);
     gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
     gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
     gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT);
     gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT);
+    gl->PixelStorei(GL_UNPACK_ALIGNMENT, 4);
     gl->ActiveTexture(GL_TEXTURE0);
 
     debug_check_gl(p, "dither setup");
@@ -1432,8 +1430,6 @@ void gl_video_set_lut3d(struct gl_video *p, struct lut3d *lut3d)
 
     gl->ActiveTexture(GL_TEXTURE0 + TEXUNIT_3DLUT);
     gl->BindTexture(GL_TEXTURE_3D, p->lut_3d_texture);
-    gl->PixelStorei(GL_UNPACK_ALIGNMENT, 4);
-    gl->PixelStorei(GL_UNPACK_ROW_LENGTH, 0);
     gl->TexImage3D(GL_TEXTURE_3D, 0, GL_RGB16, lut3d->size[0], lut3d->size[1],
                    lut3d->size[2], 0, GL_RGB, GL_UNSIGNED_SHORT, lut3d->data);
     gl->TexParameteri(GL_TEXTURE_3D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
@@ -2060,8 +2056,6 @@ static bool test_fbo(struct gl_video *p, GLenum format)
     GL *gl = p->gl;
     bool success = false;
     struct fbotex fbo = {0};
-    gl->PixelStorei(GL_PACK_ALIGNMENT, 1);
-    gl->PixelStorei(GL_PACK_ROW_LENGTH, 0);
     if (fbotex_init(p, &fbo, 16, 16, format)) {
         gl->BindFramebuffer(GL_FRAMEBUFFER, fbo.fbo);
         gl->BindFramebuffer(GL_FRAMEBUFFER, 0);
@@ -2234,16 +2228,17 @@ void gl_video_set_gl_state(struct gl_video *p)
     struct m_color c = p->opts.background;
     gl->ClearColor(c.r / 255.0, c.g / 255.0, c.b / 255.0, c.a / 255.0);
     gl->ActiveTexture(GL_TEXTURE0);
+    if (gl->mpgl_caps & MPGL_CAP_ROW_LENGTH) {
+        gl->PixelStorei(GL_PACK_ROW_LENGTH, 0);
+        gl->PixelStorei(GL_UNPACK_ROW_LENGTH, 0);
+    }
+    gl->PixelStorei(GL_PACK_ALIGNMENT, 4);
+    gl->PixelStorei(GL_UNPACK_ALIGNMENT, 4);
 }
 
 void gl_video_unset_gl_state(struct gl_video *p)
 {
-    GL *gl = p->gl;
-
-    gl->PixelStorei(GL_PACK_ROW_LENGTH, 0);
-    gl->PixelStorei(GL_UNPACK_ROW_LENGTH, 0);
-    gl->PixelStorei(GL_PACK_ALIGNMENT, 4);
-    gl->PixelStorei(GL_UNPACK_ALIGNMENT, 4);
+    // nop
 }
 
 // dest = src.<w> (always using 4 components)