From f8d32b03bfa8ad733da7ae824a580b3474c1be58 Mon Sep 17 00:00:00 2001 From: Martin Herkt Date: Wed, 10 Apr 2013 00:10:59 +0200 Subject: vf_divtc, vf_phase: Fix handling of subsampled formats These filters incorrectly calculated the amount of bytes per line in each plane for chroma subsampled formats, causing undefined behavior. --- video/filter/vf_divtc.c | 5 +++-- video/filter/vf_phase.c | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'video') diff --git a/video/filter/vf_divtc.c b/video/filter/vf_divtc.c index 5b57e5a4cb..b4718b355f 100644 --- a/video/filter/vf_divtc.c +++ b/video/filter/vf_divtc.c @@ -202,8 +202,9 @@ static int imgop(int(*planeop)(unsigned char *, unsigned char *, int sum = 0; for (int p = 0; p < dst->num_planes; p++) { sum += planeop(dst->planes[p], src ? src->planes[p] : NULL, - dst->w * (dst->fmt.bpp[p] / 8), dst->plane_h[p], - dst->stride[p], src ? src->stride[p] : 0, arg); + (dst->w * dst->fmt.bytes[p]) >> dst->fmt.xs[p], + dst->plane_h[p], dst->stride[p], + src ? src->stride[p] : 0, arg); } return sum; } diff --git a/video/filter/vf_phase.c b/video/filter/vf_phase.c index 45d5e4ae44..0b31ede5e5 100644 --- a/video/filter/vf_phase.c +++ b/video/filter/vf_phase.c @@ -205,7 +205,7 @@ static struct mp_image *filter(struct vf_instance *vf, struct mp_image *mpi) int pw[MP_MAX_PLANES] = {0}; for (int p = 0; p < mpi->num_planes; p++) - pw[p] = (mpi->w * mpi->fmt.bpp[p] + 7) / 8; + pw[p] = ((mpi->w * mpi->fmt.bpp[p] + 7) / 8) >> mpi->fmt.xs[p]; mode=vf->priv->mode; -- cgit v1.2.3