win32: fix console output with raw stdio functions

reopen_console_handle() was never properly tested because mpv overrides
printf in most source files. Turns out that when there's no console on
startup, the CRT sets the fds of stdout and stderr to -2, so the old
method of using dup2 to manipulate these fds didn't work. As far as I
can tell, the only way to give stdout and stderr valid fds is to use
freopen, so this uses freopen to set them both to the console output.
This also uses dup2 to change STDOUT_FILENO and STDERR_FILENO, so low-
level functions like isatty still work.

Note that this means fileno(stdout) != STDOUT_FILENO. I don't think this
will cause any problems.

This should fix MPV_LEAK_REPORT on the Windows console.

1 files changed, 7 insertions, 8 deletions

diff --git a/osdep/terminal-win.c b/osdep/terminal-win.c
index 8dd2258bcf..cda9e69040 100644
--- a/osdep/terminal-win.c
+++ b/osdep/terminal-win.c
@@ -254,12 +254,11 @@ static bool is_a_console(HANDLE h)
     return GetConsoleMode(h, &(DWORD){0});
 }
 
-static void reopen_console_handle(DWORD std, FILE *stream)
+static void reopen_console_handle(DWORD std, int fd, FILE *stream)
 {
-    HANDLE wstream = GetStdHandle(std);
-    if (is_a_console(wstream)) {
-        int fd = _open_osfhandle((intptr_t)wstream, _O_TEXT);
-        dup2(fd, fileno(stream));
+    if (is_a_console(GetStdHandle(std))) {
+        freopen("CONOUT$", "wt", stream);
+        dup2(fileno(stream), fd);
         setvbuf(stream, NULL, _IONBF, 0);
     }
 }
@@ -282,9 +281,9 @@ bool terminal_try_attach(void)
         return false;
 
     // We have a console window. Redirect output streams to that console's
-    // low-level handles, so we can actually use WriteConsole later on.
-    reopen_console_handle(STD_OUTPUT_HANDLE, stdout);
-    reopen_console_handle(STD_ERROR_HANDLE, stderr);
+    // low-level handles, so things that use printf directly work later on.
+    reopen_console_handle(STD_OUTPUT_HANDLE, STDOUT_FILENO, stdout);
+    reopen_console_handle(STD_ERROR_HANDLE, STDERR_FILENO, stderr);
 
     return true;
 }