win32: don't pass std handles if they are attached to console

This is default behavior to attach to existing console, passing custom
handles is useful if we want to replace them, but in case they are
already attached to console we want to attach to console directly.

In theory, it should work out of the box because "console-like" handles
should be managed by Windows internally, which works for INPUT and
OUTPUT, but in certain cases, not for ERROR. This allows using VT
features in those cases for stderr too.

1 files changed, 20 insertions, 15 deletions

diff --git a/osdep/win32-console-wrapper.c b/osdep/win32-console-wrapper.c
index 787abafb23..c1e557832d 100644
--- a/osdep/win32-console-wrapper.c
+++ b/osdep/win32-console-wrapper.c
@@ -41,26 +41,31 @@ static void cr_perror(const wchar_t *prefix)
 
 static int cr_runproc(wchar_t *name, wchar_t *cmdline)
 {
-    STARTUPINFOW si;
-    STARTUPINFOW our_si;
-    PROCESS_INFORMATION pi;
     DWORD retval = 1;
 
-    ZeroMemory(&si, sizeof(si));
-    si.cb = sizeof(si);
-    si.hStdInput = GetStdHandle(STD_INPUT_HANDLE);
-    si.hStdOutput = GetStdHandle(STD_OUTPUT_HANDLE);
-    si.hStdError = GetStdHandle(STD_ERROR_HANDLE);
-    si.dwFlags |= STARTF_USESTDHANDLES;
-
     // Copy the list of inherited CRT file descriptors to the new process
-    our_si.cb = sizeof(our_si);
+    STARTUPINFOW our_si = {sizeof(our_si)};
     GetStartupInfoW(&our_si);
-    si.lpReserved2 = our_si.lpReserved2;
-    si.cbReserved2 = our_si.cbReserved2;
-
-    ZeroMemory(&pi, sizeof(pi));
 
+    // Don't redirect std streams if they are attached to a console. Let mpv
+    // attach to the console directly in this case. In theory, it should work
+    // out of the box because "console-like" handles should be managed by Windows
+    // internally, which works for INPUT and OUTPUT, but in certain cases,
+    // not for ERROR.
+    DWORD mode;
+    HANDLE hStdInput  = GetStdHandle(STD_INPUT_HANDLE);
+    HANDLE hStdOutput = GetStdHandle(STD_OUTPUT_HANDLE);
+    HANDLE hStdError  = GetStdHandle(STD_ERROR_HANDLE);
+    STARTUPINFOW si = {
+        .cb          = sizeof(si),
+        .lpReserved2 = our_si.lpReserved2,
+        .cbReserved2 = our_si.cbReserved2,
+        .hStdInput   = GetConsoleMode(hStdInput, &mode)  ? NULL : hStdInput,
+        .hStdOutput  = GetConsoleMode(hStdOutput, &mode) ? NULL : hStdOutput,
+        .hStdError   = GetConsoleMode(hStdError, &mode)  ? NULL : hStdError,
+    };
+    si.dwFlags = (si.hStdInput || si.hStdOutput || si.hStdError) ? STARTF_USESTDHANDLES : 0;
+    PROCESS_INFORMATION pi = {0};
     if (!CreateProcessW(name, cmdline, NULL, NULL, TRUE, 0,
                         NULL, NULL, &si, &pi)) {