win32.c: disable console colorizing

* win32/win32.c (console_emulator_p, constat_handle): disable built-in
  console colorizing when console-emulator-like DLL is injected.
  [Feature #8201]

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@43131 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

4 files changed, 48 insertions, 4 deletions

diff --git a/ChangeLog b/ChangeLog
index c954ceb4c3..6409182d72 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+Thu Oct  3 18:20:47 2013  Nobuyoshi Nakada  <nobu@ruby-lang.org>
+
+	* win32/win32.c (console_emulator_p, constat_handle): disable built-in
+	  console colorizing when console-emulator-like DLL is injected.
+	  [Feature #8201]
+
 Thu Oct  3 18:01:44 2013  Koichi Sasada  <ko1@atdot.net>
 
 	* gc.c: define gc_profile_record::allocated_size if 
diff --git a/configure.in b/configure.in
index 4417a4ef45..c23f691325 100644
--- a/configure.in
+++ b/configure.in
@@ -933,7 +933,7 @@ main()
 		AC_CHECK_FUNCS(cygwin_conv_path)
 		AC_LIBOBJ([langinfo])
 		],
-[mingw*], [	LIBS="-lshell32 -lws2_32 -liphlpapi -limagehlp -lshlwapi $LIBS"
+[mingw*], [	LIBS="-lshell32 -lws2_32 -lpsapi -liphlpapi -limagehlp -lshlwapi $LIBS"
 		ac_cv_header_a_out_h=no
 		ac_cv_header_pwd_h=no
 		ac_cv_header_utime_h=no
diff --git a/win32/Makefile.sub b/win32/Makefile.sub
index 1eef8b47ad..d10e82986c 100644
--- a/win32/Makefile.sub
+++ b/win32/Makefile.sub
@@ -231,7 +231,7 @@ LIBS = oldnames.lib user32.lib advapi32.lib shell32.lib ws2_32.lib
 !if $(MSC_VER) >= 1400
 LIBS = $(LIBS) iphlpapi.lib
 !endif
-LIBS = $(LIBS) imagehlp.lib shlwapi.lib $(EXTLIBS)
+LIBS = $(LIBS) psapi.lib imagehlp.lib shlwapi.lib $(EXTLIBS)
 !endif
 !if !defined(MISSING)
 MISSING = acosh.obj cbrt.obj crypt.obj erf.obj ffs.obj langinfo.obj lgamma_r.obj strlcat.obj strlcpy.obj tgamma.obj win32/win32.obj win32/file.obj setproctitle.obj
diff --git a/win32/win32.c b/win32/win32.c
index d673636e57..f374bfa209 100644
--- a/win32/win32.c
+++ b/win32/win32.c
@@ -39,6 +39,8 @@
 #include <share.h>
 #include <shlobj.h>
 #include <mbstring.h>
+#include <psapi.h>
+#include <shlwapi.h>
 #if _MSC_VER >= 1400
 #include <crtdbg.h>
 #include <rtcapi.h>
@@ -604,6 +606,7 @@ static CRITICAL_SECTION select_mutex;
 static int NtSocketsInitialized = 0;
 static st_table *socklist = NULL;
 static st_table *conlist = NULL;
+#define conlist_disabled ((st_table *)-1)
 static char *envarea;
 static char *uenvarea;
 
@@ -629,7 +632,7 @@ free_conlist(st_data_t key, st_data_t val, st_data_t arg)
 static void
 constat_delete(HANDLE h)
 {
-    if (conlist) {
+    if (conlist && conlist != conlist_disabled) {
 	st_data_t key = (st_data_t)h, val;
 	st_delete(conlist, &key, &val);
 	xfree((struct constat *)val);
@@ -649,7 +652,7 @@ exit_handler(void)
 	DeleteCriticalSection(&select_mutex);
 	NtSocketsInitialized = 0;
     }
-    if (conlist) {
+    if (conlist && conlist != conlist_disabled) {
 	st_foreach(conlist, free_conlist, 0);
 	st_free_table(conlist);
 	conlist = NULL;
@@ -5832,14 +5835,48 @@ rb_w32_pipe(int fds[2])
 }
 
 /* License: Ruby's */
+static int
+console_emulator_p(void)
+{
+    HMODULE module_buf[10], *pmodule = module_buf;
+    DWORD nmodule = numberof(module_buf), needed = 0, i;
+    HANDLE proch = GetCurrentProcess();
+
+    if (!EnumProcessModules(proch, pmodule, nmodule * sizeof(HMODULE), &needed))
+	return FALSE;
+    if (needed / sizeof(HMODULE) > nmodule) {
+	nmodule = needed / sizeof(HMODULE);
+	pmodule = alloca(sizeof(HMODULE) * nmodule);
+	if (!EnumProcessModules(proch, pmodule, needed, &needed))
+	    return FALSE;
+    }
+    for (i = 0; i < nmodule; i++) {
+	WCHAR modname[MAX_PATH];
+
+	if (GetModuleBaseNameW(proch, pmodule[i], modname, numberof(modname))) {
+	    if (PathMatchSpecW(modname, L"conemu*.dll")) return TRUE;
+        }
+    }
+
+    return 0;
+}
+
+/* License: Ruby's */
 static struct constat *
 constat_handle(HANDLE h)
 {
     st_data_t data;
     struct constat *p;
     if (!conlist) {
+	if (console_emulator_p()) {
+	    conlist = conlist_disabled;
+	    return NULL;
+	}
 	conlist = st_init_numtable();
     }
+    else if (conlist == conlist_disabled) {
+	return NULL;
+    }
     if (st_lookup(conlist, (st_data_t)h, &data)) {
 	p = (struct constat *)data;
     }
@@ -6483,6 +6520,7 @@ rb_w32_write_console(uintptr_t strarg, int fd)
 	return -1L;
 
     s = constat_handle(handle);
+    if (!s) return -1L;
     encindex = ENCODING_GET(str);
     switch (encindex) {
       default: