compare: introduce customizable threshold to pass test

2 files changed, 71 insertions, 24 deletions

diff --git a/compare/README.md b/compare/README.md
index c161c1c..2586f43 100644
--- a/compare/README.md
+++ b/compare/README.md
@@ -5,11 +5,16 @@ To build a test utility configure libass with the `--enable-compare` flag.
 The utility works with `png` image files so there is external dependency of libpng.
 
 Test program command line:  
-`compare ([-i] <input-dir>)+ [-o <output-dir>] [-s <scale:1-8>]`
+`compare ([-i] <input-dir>)+ [-o <output-dir>] [-s <scale:1-8>] [-p <pass-level:0-3>]`
 
 * `<input-dir>` is a test input directory, can be several of them;
 * `<output-dir>` if present sets directory to store the rendering results;
-* `<scale>` sets an oversampling factor (positive integer up to 8, default 1).
+* `<scale>` sets an oversampling factor (positive integer up to 8, default 1);
+* `<pass-level>` corresponds to the level of image differences required to pass test:
+  - 0: only `SAME` level accepted, bitwise comparison mode;
+  - 1: `GOOD` level or less required;
+  - 2: `BAD` level or less required, default mode;
+  - 3: `FAIL` level or less required, i. e. any difference accepted, error checking mode.
 
 An input directory consists of font files (`*.ttf`, `*.otf` and `*.pfb`), subtitle files (`*.ass`), and image files (`*.png`).
 All the fonts required for rendering should be present in the input directories as
@@ -62,11 +67,13 @@ Processing 'sub2.ass':
   Time 0:02:33.000 - 0.728 OK
 Only 3 of 4 images have passed test
 ```
-For each target image file the program reports a maximal ratio of the per pixel comparison error to the baseline error scale.
-The baseline error scale tries to take account of neighboring pixels to estimate visibility of an error in that specific pixel location.
-Currently the range of [0&ndash;2.0] is marked as `OK`, the range of [2.0&ndash;4.0] as `BAD` and more than 4.0 as `FAIL`.
-At the end there is a summary of how many images don't fail tests.
-If all images pass that test the program returns 0 otherwise 1.
+For each target image file the program reports a maximal ratio of the per pixel comparison difference to the baseline error scale.
+The baseline error scale tries to take account of neighboring pixels to estimate visibility of a difference in that specific pixel location.
+Zero difference is marked as `SAME` (level 0), nonzero value in the range of [0–2.0] is marked as `GOOD` (level 1),
+in the range of [2.0–4.0] as `BAD` (level 2) and more than 4.0 as `FAIL` (level 3).
+Any problem during processing of a specific image is treated as a level 4 difference.
+If all images have level of difference less or equal to pass level (`-p` switch, default 2) then the test is considered as passed
+and program returns 0. Otherwise program returns maximal level of difference throughout all images (4 in case of error).
 
 Note that almost any type of a rendering error can be greatly exaggerated by the specially tailored test cases.
 Therefore test cases should be chosen to represent generic real world scenarios only.
diff --git a/compare/compare.c b/compare/compare.c
index 73a6527..f5603e7 100644
--- a/compare/compare.c
+++ b/compare/compare.c
@@ -300,9 +300,29 @@ static bool out_of_memory()
     return false;
 }
 
-static bool process_image(ASS_Renderer *renderer, ASS_Track *track,
-                          const char *input, const char *output,
-                          const char *file, int64_t time, int scale)
+typedef enum {
+    R_SAME, R_GOOD, R_BAD, R_FAIL, R_ERROR
+} Result;
+
+static const char *result_text[R_ERROR] = {
+    "SAME", "GOOD", "BAD", "FAIL"
+};
+
+Result classify_result(double error)
+{
+    if (error == 0)
+        return R_SAME;
+    else if (error < 2)
+        return R_GOOD;
+    else if (error < 4)
+        return R_BAD;
+    else
+        return R_FAIL;
+}
+
+static Result process_image(ASS_Renderer *renderer, ASS_Track *track,
+                            const char *input, const char *output,
+                            const char *file, int64_t time, int scale)
 {
     uint64_t tm = time;
     unsigned msec = tm % 1000;  tm /= 1000;
@@ -316,13 +336,14 @@ static bool process_image(ASS_Renderer *renderer, ASS_Track *track,
     Image16 target;
     if (!read_png(path, &target)) {
         printf("PNG reading failed!\n");
-        return false;
+        return R_ERROR;
     }
 
     uint16_t *grad = malloc(2 * target.width * target.height);
     if (!grad) {
         free(target.buffer);
-        return out_of_memory();
+        out_of_memory();
+        return R_ERROR;
     }
     calc_grad(&target, grad);
 
@@ -339,10 +360,12 @@ static bool process_image(ASS_Renderer *renderer, ASS_Track *track,
     int res = compare(&target, grad, img, out_file, &max_err, scale);
     free(target.buffer);
     free(grad);
-    if (!res)
-        return out_of_memory();
-    bool flag = max_err < 4;
-    printf("%.3f %s\n", max_err, flag ? (max_err < 2 ? "OK" : "BAD") : "FAIL");
+    if (!res) {
+        out_of_memory();
+        return R_ERROR;
+    }
+    Result flag = classify_result(max_err);
+    printf("%.3f %s\n", max_err, result_text[flag]);
     if (res < 0)
         printf("Cannot write PNG to file '%s'!\n", path);
     return flag;
@@ -511,7 +534,7 @@ static bool process_input(ItemList *list, const char *path, ASS_Library *lib)
 
 
 enum {
-    OUTPUT, SCALE, INPUT
+    OUTPUT, SCALE, LEVEL, INPUT
 };
 
 static int *parse_cmdline(int argc, char *argv[])
@@ -532,6 +555,7 @@ static int *parse_cmdline(int argc, char *argv[])
         case 'i':  index = input++;  break;
         case 'o':  index = OUTPUT;   break;
         case 's':  index = SCALE;    break;
+        case 'p':  index = LEVEL;    break;
         default:   goto fail;
         }
         if (argv[i][2] || ++i >= argc || pos[index])
@@ -544,7 +568,7 @@ static int *parse_cmdline(int argc, char *argv[])
 fail:
     free(pos);
     const char *fmt =
-        "Usage: %s ([-i] <input-dir>)+ [-o <output-dir>] [-s <scale:1-8>]\n";
+        "Usage: %s ([-i] <input-dir>)+ [-o <output-dir>] [-s <scale:1-8>] [-p <pass-level:0-3>]\n";
     printf(fmt, argv[0]);
     return NULL;
 }
@@ -562,11 +586,11 @@ int main(int argc, char *argv[])
 {
     int *pos = parse_cmdline(argc, argv);
     if (!pos)
-        return 1;
+        return R_ERROR;
 
     ASS_Library *lib = NULL;
     ItemList list = {0};
-    int result = 1;
+    int result = R_ERROR;
 
     int scale = 1;
     if (pos[SCALE]) {
@@ -578,6 +602,16 @@ int main(int argc, char *argv[])
         scale = arg[0] - '0';
     }
 
+    int level = R_BAD;
+    if (pos[LEVEL]) {
+        const char *arg = argv[pos[LEVEL]];
+        if (arg[0] < '0' || arg[0] > '3' || arg[1]) {
+            printf("Invalid pass level value, should be 0-3!\n");
+            goto end;
+        }
+        level = arg[0] - '0';
+    }
+
     const char *output = NULL;
     if (pos[OUTPUT]) {
         output = argv[pos[OUTPUT]];
@@ -613,6 +647,7 @@ int main(int argc, char *argv[])
     }
     ass_set_fonts(renderer, NULL, NULL, ASS_FONTPROVIDER_NONE, NULL, 0);
 
+    result = 0;
     size_t prefix = 0;
     const char *prev = "";
     ASS_Track *track = NULL;
@@ -642,8 +677,10 @@ int main(int argc, char *argv[])
         total++;
         if (!track)
             continue;
-        if (process_image(renderer, track, list.items[i].dir, output,
-                          name, list.items[i].time, scale))
+        Result res = process_image(renderer, track, list.items[i].dir, output,
+                                   name, list.items[i].time, scale);
+        result = FFMAX(result, res);
+        if (res <= level)
             good++;
     }
     if (track)
@@ -652,10 +689,13 @@ int main(int argc, char *argv[])
 
     if (!total) {
         printf("No images found!\n");
+        result = R_ERROR;
     } else if (good < total) {
-        printf("Only %u of %u images have passed test\n", good, total);
+        printf("Only %u of %u images have passed test (%s or better)\n",
+               good, total, result_text[level]);
     } else {
-        printf("All %u images have passed test\n", total);
+        printf("All %u images have passed test (%s or better)\n",
+               total, result_text[level]);
         result = 0;
     }