win32: cache GetConsoleMode state for stdout/stderr

GetConsoleMode() can be quite slow and in mpv the mode never changes, so
we can just check it once.

Fixes performance when writing lots of logs to terminal.

5 files changed, 45 insertions, 33 deletions

diff --git a/osdep/io.c b/osdep/io.c
index d2f1a60c6e..1d99e4d207 100644
--- a/osdep/io.c
+++ b/osdep/io.c
@@ -304,30 +304,6 @@ static int mp_vfprintf(FILE *stream, const char *format, va_list args)
     return vfprintf(stream, format, args);
 }
 #else
-static int mp_check_console(HANDLE wstream)
-{
-    if (wstream != INVALID_HANDLE_VALUE) {
-        unsigned int filetype = GetFileType(wstream);
-
-        if (!((filetype == FILE_TYPE_UNKNOWN) &&
-            (GetLastError() != ERROR_SUCCESS)))
-        {
-            filetype &= ~(FILE_TYPE_REMOTE);
-
-            if (filetype == FILE_TYPE_CHAR) {
-                DWORD ConsoleMode;
-                int ret = GetConsoleMode(wstream, &ConsoleMode);
-
-                if (!(!ret && (GetLastError() == ERROR_INVALID_HANDLE))) {
-                    // This seems to be a console
-                    return 1;
-                }
-            }
-        }
-    }
-
-    return 0;
-}
 
 static int mp_vfprintf(FILE *stream, const char *format, va_list args)
 {
diff --git a/osdep/terminal-win.c b/osdep/terminal-win.c
index 5ff70cbf3c..d4647faefd 100644
--- a/osdep/terminal-win.c
+++ b/osdep/terminal-win.c
@@ -53,20 +53,16 @@ static void attempt_native_out_vt(HANDLE hOut, DWORD basemode)
         SetConsoleMode(hOut, basemode);
 }
 
-static bool is_native_out_vt(HANDLE hOut)
-{
-    DWORD cmode;
-    return GetConsoleMode(hOut, &cmode) &&
-           (cmode & ENABLE_VIRTUAL_TERMINAL_PROCESSING) &&
-           !(cmode & DISABLE_NEWLINE_AUTO_RETURN);
-}
 
+#define hSTDIN GetStdHandle(STD_INPUT_HANDLE)
 #define hSTDOUT GetStdHandle(STD_OUTPUT_HANDLE)
 #define hSTDERR GetStdHandle(STD_ERROR_HANDLE)
 
 #define FOREGROUND_ALL (FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE)
 #define BACKGROUND_ALL (BACKGROUND_RED | BACKGROUND_GREEN | BACKGROUND_BLUE)
 
+static bool is_console[STDERR_FILENO + 1];
+static bool is_vt[STDERR_FILENO + 1];
 static short stdoutAttrs = 0;  // copied from the screen buffer on init
 static const unsigned char ansi2win32[8] = {
     0,
@@ -94,6 +90,23 @@ static HANDLE death;
 static mp_thread input_thread;
 static struct input_ctx *input_ctx;
 
+static bool is_native_out_vt_internal(HANDLE hOut)
+{
+    DWORD cmode;
+    return GetConsoleMode(hOut, &cmode) &&
+           (cmode & ENABLE_VIRTUAL_TERMINAL_PROCESSING) &&
+           !(cmode & DISABLE_NEWLINE_AUTO_RETURN);
+}
+
+static bool is_native_out_vt(HANDLE hOut)
+{
+    if (hOut == hSTDOUT)
+        return is_vt[STDOUT_FILENO];
+    if (hOut == hSTDERR)
+        return is_vt[STDERR_FILENO];
+    return is_native_out_vt_internal(hOut);
+}
+
 void terminal_get_size(int *w, int *h)
 {
     CONSOLE_SCREEN_BUFFER_INFO cinfo;
@@ -360,6 +373,17 @@ static bool is_a_console(HANDLE h)
     return GetConsoleMode(h, &(DWORD){0});
 }
 
+bool mp_check_console(void *handle)
+{
+    if (handle == hSTDIN)
+        return is_console[STDIN_FILENO];
+    if (handle == hSTDOUT)
+        return is_console[STDOUT_FILENO];
+    if (handle == hSTDERR)
+        return is_console[STDERR_FILENO];
+    return is_a_console(handle);
+}
+
 static void reopen_console_handle(DWORD std, int fd, FILE *stream)
 {
     HANDLE handle = GetStdHandle(std);
@@ -419,6 +443,16 @@ void terminal_init(void)
     cmode |= (ENABLE_PROCESSED_OUTPUT | ENABLE_WRAP_AT_EOL_OUTPUT);
     attempt_native_out_vt(hSTDOUT, cmode);
     attempt_native_out_vt(hSTDERR, cmode);
+
+    // Init for mp_check_console(), this never changes during runtime
+    is_console[STDIN_FILENO] = is_a_console(hSTDIN);
+    is_console[STDOUT_FILENO] = is_a_console(hSTDOUT);
+    is_console[STDERR_FILENO] = is_a_console(hSTDERR);
+
+    // Init for is_native_out_vt(), this is never disabled/changed during runtime
+    is_vt[STDOUT_FILENO] = is_native_out_vt_internal(hSTDOUT);
+    is_vt[STDERR_FILENO] = is_native_out_vt_internal(hSTDERR);
+
     GetConsoleScreenBufferInfo(hSTDOUT, &cinfo);
     stdoutAttrs = cinfo.wAttributes;
 }
diff --git a/osdep/terminal.h b/osdep/terminal.h
index 5383a175da..5cc516fe29 100644
--- a/osdep/terminal.h
+++ b/osdep/terminal.h
@@ -53,6 +53,7 @@ void terminal_get_size2(int *rows, int *cols, int *px_width, int *px_height);
 
 // Windows only.
 void mp_write_console_ansi(void *wstream, char *buf);
+bool mp_check_console(void *handle);
 
 /* Windows-only function to attach to the parent process's console */
 bool terminal_try_attach(void);
diff --git a/test/test_utils.c b/test/test_utils.c
index 98597c9847..7e5f2e6ce7 100644
--- a/test/test_utils.c
+++ b/test/test_utils.c
@@ -4,6 +4,7 @@
 #include "options/m_option.h"
 #include "options/path.h"
 #include "osdep/subprocess.h"
+#include "osdep/terminal.h"
 #include "test_utils.h"
 
 #ifdef NDEBUG
@@ -106,7 +107,8 @@ void mp_msg(struct mp_log *log, int lev, const char *format, ...) {};
 int mp_msg_find_level(const char *s) {return 0;};
 int mp_msg_level(struct mp_log *log) {return 0;};
 void mp_msg_set_max_level(struct mp_log *log, int lev) {};
-void mp_write_console_ansi(void) {};
+void mp_write_console_ansi(void *wstream, char *buf) {};
+bool mp_check_console(void *handle) { return false; };
 void mp_set_avdict(AVDictionary **dict, char **kv) {};
 struct mp_log *mp_log_new(void *talloc_ctx, struct mp_log *parent,
                           const char *name) { return NULL; };
diff --git a/test/test_utils.h b/test/test_utils.h
index df8c567ce9..7db358e21a 100644
--- a/test/test_utils.h
+++ b/test/test_utils.h
@@ -52,6 +52,5 @@ void mp_msg(struct mp_log *log, int lev, const char *format, ...)
 int mp_msg_find_level(const char *s);
 int mp_msg_level(struct mp_log *log);
 void mp_msg_set_max_level(struct mp_log *log, int lev);
-void mp_write_console_ansi(void);
 typedef struct AVDictionary AVDictionary;
 void mp_set_avdict(AVDictionary **dict, char **kv);