Reline: Use a more robust detection of MinTTY

The previous detection per get_screen_size fails when stdout is passed
to a pipe. That is the case when running ruby tests in parallel ("-j" switch).
In this case Reline believes that it's running on MinTTY and the tests
are running with ANSI IOGate instead of the Windows adapter on MINGW.
So parallel test results were different to that of a single process.
This commit fixes these differencies.

The code is taken from git sources and translated to ruby.
NtQueryObject() is replaced by GetFileInformationByHandleEx(), because
NtQueryObject() is undocumented and is more difficult to use:
  https://github.com/git-for-windows/git/blob/c5a03b1e29c69f3f06c8fabd92493edb73469176/compat/winansi.c#L558

2 files changed, 30 insertions, 2 deletions

diff --git a/lib/reline.rb b/lib/reline.rb
index 606dd4645b..ee4d1c4180 100644
--- a/lib/reline.rb
+++ b/lib/reline.rb
@@ -415,8 +415,7 @@ end
 
 if RbConfig::CONFIG['host_os'] =~ /mswin|msys|mingw|cygwin|bccwin|wince|emc/
   require 'reline/windows'
-  if Reline::Windows.get_screen_size == [0, 0]
-    # Maybe Mintty on Cygwin
+  if Reline::Windows.msys_tty?
     require 'reline/ansi'
     Reline::IOGate = Reline::ANSI
   else
diff --git a/lib/reline/windows.rb b/lib/reline/windows.rb
index 4fbf243c37..74b62233e5 100644
--- a/lib/reline/windows.rb
+++ b/lib/reline/windows.rb
@@ -72,6 +72,8 @@ class Reline::Windows
   STD_INPUT_HANDLE = -10
   STD_OUTPUT_HANDLE = -11
   WINDOW_BUFFER_SIZE_EVENT = 0x04
+  FILE_TYPE_PIPE = 0x0003
+  FILE_NAME_INFO = 2
   @@getwch = Win32API.new('msvcrt', '_getwch', [], 'I')
   @@kbhit = Win32API.new('msvcrt', '_kbhit', [], 'I')
   @@GetKeyState = Win32API.new('user32', 'GetKeyState', ['L'], 'L')
@@ -84,9 +86,36 @@ class Reline::Windows
   @@hConsoleInputHandle = @@GetStdHandle.call(STD_INPUT_HANDLE)
   @@GetNumberOfConsoleInputEvents = Win32API.new('kernel32', 'GetNumberOfConsoleInputEvents', ['L', 'P'], 'L')
   @@ReadConsoleInput = Win32API.new('kernel32', 'ReadConsoleInput', ['L', 'P', 'L', 'P'], 'L')
+  @@GetFileType = Win32API.new('kernel32', 'GetFileType', ['L'], 'L')
+  @@GetFileInformationByHandleEx = Win32API.new('kernel32', 'GetFileInformationByHandleEx', ['L', 'I', 'P', 'L'], 'I')
+
   @@input_buf = []
   @@output_buf = []
 
+  def self.msys_tty?(io=@@hConsoleInputHandle)
+    # check if fd is a pipe
+    if @@GetFileType.call(io) != FILE_TYPE_PIPE
+      return false
+    end
+
+    bufsize = 1024
+    p_buffer = "\0" * bufsize
+    res = @@GetFileInformationByHandleEx.call(io, FILE_NAME_INFO, p_buffer, bufsize - 2)
+    return false if res == 0
+
+    # get pipe name: p_buffer layout is:
+    #   struct _FILE_NAME_INFO {
+    #     DWORD FileNameLength;
+    #     WCHAR FileName[1];
+    #   } FILE_NAME_INFO
+    len = p_buffer[0, 4].unpack("L")[0]
+    name = p_buffer[4, len].encode(Encoding::UTF_8, Encoding::UTF_16LE, invalid: :replace)
+
+    # Check if this could be a MSYS2 pty pipe ('\msys-XXXX-ptyN-XX')
+    # or a cygwin pty pipe ('\cygwin-XXXX-ptyN-XX')
+    name =~ /(msys-|cygwin-).*-pty/ ? true : false
+  end
+
   def self.getwch
     unless @@input_buf.empty?
       return @@input_buf.shift