win32: only attach to the console from mpv.com

Previously, mpv.exe used the --terminal option to decide whether to
attach to the parent process's console, which made it impossible to tell
whether mpv would attach to the console before the config files were
parsed. Instead, make mpv always attach to the console when launched
from the console wrapper (mpv.com) and never attach otherwise. This will
be useful for the next commit, which will use the presence of the
console to decide whether to use the pseudo-gui profile.

This change should also be an improvement in behavior. The old code
would attach to the parent process's console, regardless of whether it
was mpv.com or some other program like cmd.exe. This could be confusing,
since mpv.exe is marked as a Windows GUI program and shouldn't write
text to its parent process's console when launched directly. (See #768.)

Visual Studio does something similar with its devenv.com wrapper.
devenv.exe only attaches to the console when launched from devenv.com.

1 files changed, 37 insertions, 13 deletions

diff --git a/osdep/terminal-win.c b/osdep/terminal-win.c
index 678cedb775..03a1ba28c6 100644
--- a/osdep/terminal-win.c
+++ b/osdep/terminal-win.c
@@ -250,24 +250,48 @@ void mp_write_console_ansi(HANDLE wstream, char *buf)
     }
 }
 
-int terminal_init(void)
+static bool is_a_console(HANDLE h)
+{
+    return GetConsoleMode(h, &(DWORD){0});
+}
+
+static void reopen_console_handle(DWORD std, FILE *stream)
+{
+    HANDLE wstream = GetStdHandle(std);
+    if (is_a_console(wstream)) {
+        int fd = _open_osfhandle((intptr_t)wstream, _O_TEXT);
+        dup2(fd, fileno(stream));
+        setvbuf(stream, NULL, _IONBF, 0);
+    }
+}
+
+bool terminal_try_attach(void)
 {
-    if (AttachConsole(ATTACH_PARENT_PROCESS)) {
-        // We have been started by something with a console window.
-        // Redirect output streams to that console's low-level handles,
-        // so we can actually use WriteConsole later on.
+    // mpv.exe is a flagged as a GUI application, but it acts as a console
+    // application when started from the console wrapper (see
+    // osdep/win32-console-wrapper.c). The console wrapper sets
+    // _started_from_console=yes, so check that variable before trying to
+    // attach to the console.
+    wchar_t console_env[4] = { 0 };
+    if (!GetEnvironmentVariableW(L"_started_from_console", console_env, 4))
+        return false;
+    if (wcsncmp(console_env, L"yes", 4))
+        return false;
+    SetEnvironmentVariableW(L"_started_from_console", NULL);
 
-        int hConHandle;
+    if (!AttachConsole(ATTACH_PARENT_PROCESS))
+        return false;
 
-        hConHandle = _open_osfhandle((intptr_t)hSTDOUT, _O_TEXT);
-        *stdout = *_fdopen(hConHandle, "w");
-        setvbuf(stdout, NULL, _IONBF, 0);
+    // We have a console window. Redirect output streams to that console's
+    // low-level handles, so we can actually use WriteConsole later on.
+    reopen_console_handle(STD_OUTPUT_HANDLE, stdout);
+    reopen_console_handle(STD_ERROR_HANDLE, stderr);
 
-        hConHandle = _open_osfhandle((intptr_t)hSTDERR, _O_TEXT);
-        *stderr = *_fdopen(hConHandle, "w");
-        setvbuf(stderr, NULL, _IONBF, 0);
-    }
+    return true;
+}
 
+int terminal_init(void)
+{
     CONSOLE_SCREEN_BUFFER_INFO cinfo;
     DWORD cmode = 0;
     GetConsoleMode(hSTDOUT, &cmode);