win32: only attach to the console from mpv.com

Previously, mpv.exe used the --terminal option to decide whether to
attach to the parent process's console, which made it impossible to tell
whether mpv would attach to the console before the config files were
parsed. Instead, make mpv always attach to the console when launched
from the console wrapper (mpv.com) and never attach otherwise. This will
be useful for the next commit, which will use the presence of the
console to decide whether to use the pseudo-gui profile.

This change should also be an improvement in behavior. The old code
would attach to the parent process's console, regardless of whether it
was mpv.com or some other program like cmd.exe. This could be confusing,
since mpv.exe is marked as a Windows GUI program and shouldn't write
text to its parent process's console when launched directly. (See #768.)

Visual Studio does something similar with its devenv.com wrapper.
devenv.exe only attaches to the console when launched from devenv.com.

4 files changed, 49 insertions, 13 deletions

diff --git a/osdep/terminal-win.c b/osdep/terminal-win.c
index 678cedb775..03a1ba28c6 100644
--- a/osdep/terminal-win.c
+++ b/osdep/terminal-win.c
@@ -250,24 +250,48 @@ void mp_write_console_ansi(HANDLE wstream, char *buf)
     }
 }
 
-int terminal_init(void)
+static bool is_a_console(HANDLE h)
+{
+    return GetConsoleMode(h, &(DWORD){0});
+}
+
+static void reopen_console_handle(DWORD std, FILE *stream)
+{
+    HANDLE wstream = GetStdHandle(std);
+    if (is_a_console(wstream)) {
+        int fd = _open_osfhandle((intptr_t)wstream, _O_TEXT);
+        dup2(fd, fileno(stream));
+        setvbuf(stream, NULL, _IONBF, 0);
+    }
+}
+
+bool terminal_try_attach(void)
 {
-    if (AttachConsole(ATTACH_PARENT_PROCESS)) {
-        // We have been started by something with a console window.
-        // Redirect output streams to that console's low-level handles,
-        // so we can actually use WriteConsole later on.
+    // mpv.exe is a flagged as a GUI application, but it acts as a console
+    // application when started from the console wrapper (see
+    // osdep/win32-console-wrapper.c). The console wrapper sets
+    // _started_from_console=yes, so check that variable before trying to
+    // attach to the console.
+    wchar_t console_env[4] = { 0 };
+    if (!GetEnvironmentVariableW(L"_started_from_console", console_env, 4))
+        return false;
+    if (wcsncmp(console_env, L"yes", 4))
+        return false;
+    SetEnvironmentVariableW(L"_started_from_console", NULL);
 
-        int hConHandle;
+    if (!AttachConsole(ATTACH_PARENT_PROCESS))
+        return false;
 
-        hConHandle = _open_osfhandle((intptr_t)hSTDOUT, _O_TEXT);
-        *stdout = *_fdopen(hConHandle, "w");
-        setvbuf(stdout, NULL, _IONBF, 0);
+    // We have a console window. Redirect output streams to that console's
+    // low-level handles, so we can actually use WriteConsole later on.
+    reopen_console_handle(STD_OUTPUT_HANDLE, stdout);
+    reopen_console_handle(STD_ERROR_HANDLE, stderr);
 
-        hConHandle = _open_osfhandle((intptr_t)hSTDERR, _O_TEXT);
-        *stderr = *_fdopen(hConHandle, "w");
-        setvbuf(stderr, NULL, _IONBF, 0);
-    }
+    return true;
+}
 
+int terminal_init(void)
+{
     CONSOLE_SCREEN_BUFFER_INFO cinfo;
     DWORD cmode = 0;
     GetConsoleMode(hSTDOUT, &cmode);
diff --git a/osdep/terminal.h b/osdep/terminal.h
index fc98f0c469..89d4071d93 100644
--- a/osdep/terminal.h
+++ b/osdep/terminal.h
@@ -47,4 +47,7 @@ void terminal_get_size(int *w, int *h);
 // Windows only.
 void mp_write_console_ansi(void *wstream, char *buf);
 
+/* Windows-only function to attach to the parent process's console */
+bool terminal_try_attach(void);
+
 #endif /* MPLAYER_GETCH2_H */
diff --git a/osdep/win32-console-wrapper.c b/osdep/win32-console-wrapper.c
index c8c297b482..8cebcf8c83 100644
--- a/osdep/win32-console-wrapper.c
+++ b/osdep/win32-console-wrapper.c
@@ -72,5 +72,9 @@ int wmain(int argc, wchar_t **argv, wchar_t **envp)
     GetModuleFileNameW(NULL, exe, MAX_PATH);
     wcscpy(wcsrchr(exe, '.') + 1, L"exe");
 
+    // Set an environment variable so the child process can tell whether it
+    // was started from this wrapper and attach to the console accordingly
+    SetEnvironmentVariableW(L"_started_from_console", L"yes");
+
     return cr_runproc(exe, cmd);
 }
diff --git a/player/main-fn-win.c b/player/main-fn-win.c
index 125b4116f8..9e46e4af5f 100644
--- a/player/main-fn-win.c
+++ b/player/main-fn-win.c
@@ -1,6 +1,7 @@
 #include "config.h"
 #include "core.h"
 #include "osdep/io.h"
+#include "osdep/terminal.h"
 
 int wmain(int argc, wchar_t *argv[]);
 
@@ -9,6 +10,10 @@ int _dowildcard = 0;
 
 int wmain(int argc, wchar_t *argv[])
 {
+    // If started from the console wrapper (see osdep/win32-console-wrapper.c),
+    // attach to the console and set up the standard IO handles
+    terminal_try_attach();
+
     char **argv_u8 = talloc_zero_array(NULL, char*, argc + 1);
     for (int i = 0; i < argc; i++)
         argv_u8[i] = mp_to_utf8(argv_u8, argv[i]);