lib/isatty.c

   1 /* isatty() replacement.
   2    Copyright (C) 2012-2024 Free Software Foundation, Inc.
   3
   4    This file is free software: you can redistribute it and/or modify
   5    it under the terms of the GNU Lesser General Public License as
   6    published by the Free Software Foundation; either version 2.1 of the
   7    License, or (at your option) any later version.
   8
   9    This file is distributed in the hope that it will be useful,
  10    but WITHOUT ANY WARRANTY; without even the implied warranty of
  11    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12    GNU Lesser General Public License for more details.
  13
  14    You should have received a copy of the GNU Lesser General Public License
  15    along with this program.  If not, see <https://www.gnu.org/licenses/>.  */
  16
  17 #include <config.h>
  18
  19 /* Specification.  */
  20 #include <unistd.h>
  21
  22 /* This replacement is enabled on native Windows.  */
  23
  24 #include <errno.h>
  25 #include <string.h>
  26
  27 /* Get declarations of the Win32 API functions.  */
  28 #define WIN32_LEAN_AND_MEAN
  29 #include <windows.h>
  30
  31 #if HAVE_MSVC_INVALID_PARAMETER_HANDLER
  32 # include "msvc-inval.h"
  33 #endif
  34
  35 /* Get _get_osfhandle().  */
  36 #if GNULIB_MSVC_NOTHROW
  37 # include "msvc-nothrow.h"
  38 #else
  39 # include <io.h>
  40 #endif
  41
  42 /* Don't assume that UNICODE is not defined.  */
  43 #undef LoadLibrary
  44 #define LoadLibrary LoadLibraryA
  45 #undef QueryFullProcessImageName
  46 #define QueryFullProcessImageName QueryFullProcessImageNameA
  47
  48 #if !(_WIN32_WINNT >= _WIN32_WINNT_VISTA)
  49
  50 /* Avoid warnings from gcc -Wcast-function-type.  */
  51 # define GetProcAddress \
  52    (void *) GetProcAddress
  53
  54 /* GetNamedPipeClientProcessId was introduced only in Windows Vista.  */
  55 typedef BOOL (WINAPI * GetNamedPipeClientProcessIdFuncType) (HANDLE hPipe,
  56                                                              PULONG pClientProcessId);
  57 static GetNamedPipeClientProcessIdFuncType GetNamedPipeClientProcessIdFunc = NULL;
  58 /* QueryFullProcessImageName was introduced only in Windows Vista.  */
  59 typedef BOOL (WINAPI * QueryFullProcessImageNameFuncType) (HANDLE hProcess,
  60                                                            DWORD dwFlags,
  61                                                            LPSTR lpExeName,
  62                                                            PDWORD pdwSize);
  63 static QueryFullProcessImageNameFuncType QueryFullProcessImageNameFunc = NULL;
  64 static BOOL initialized = FALSE;
  65
  66 static void
  67 initialize (void)
  68 {
  69   HMODULE kernel32 = LoadLibrary ("kernel32.dll");
  70   if (kernel32 != NULL)
  71     {
  72       GetNamedPipeClientProcessIdFunc =
  73         (GetNamedPipeClientProcessIdFuncType) GetProcAddress (kernel32, "GetNamedPipeClientProcessId");
  74       QueryFullProcessImageNameFunc =
  75         (QueryFullProcessImageNameFuncType) GetProcAddress (kernel32, "QueryFullProcessImageNameA");
  76     }
  77   initialized = TRUE;
  78 }
  79
  80 #else
  81
  82 # define GetNamedPipeClientProcessIdFunc GetNamedPipeClientProcessId
  83 # define QueryFullProcessImageNameFunc QueryFullProcessImageName
  84
  85 #endif
  86
  87 static BOOL IsConsoleHandle (HANDLE h)
  88 {
  89   DWORD mode;
  90   /* GetConsoleMode
  91      <https://docs.microsoft.com/en-us/windows/console/getconsolemode> */
  92   return GetConsoleMode (h, &mode) != 0;
  93 }
  94
  95 static BOOL IsCygwinConsoleHandle (HANDLE h)
  96 {
  97   /* A handle to a Cygwin console is in fact a named pipe whose client process
  98      and server process is <CYGWIN_INSTALL_DIR>\bin\mintty.exe.  */
  99   BOOL result = FALSE;
 100   ULONG processId;
 101
 102 #if !(_WIN32_WINNT >= _WIN32_WINNT_VISTA)
 103   if (!initialized)
 104     initialize ();
 105 #endif
 106
 107   /* GetNamedPipeClientProcessId
 108      <https://docs.microsoft.com/en-us/windows/desktop/api/winbase/nf-winbase-getnamedpipeclientprocessid>
 109      It requires -D_WIN32_WINNT=_WIN32_WINNT_VISTA or higher.  */
 110   if (GetNamedPipeClientProcessIdFunc && QueryFullProcessImageNameFunc
 111       && GetNamedPipeClientProcessIdFunc (h, &processId))
 112     {
 113       /* OpenProcess
 114          <https://docs.microsoft.com/en-us/windows/desktop/api/processthreadsapi/nf-processthreadsapi-openprocess> */
 115       HANDLE processHandle =
 116         OpenProcess (PROCESS_QUERY_LIMITED_INFORMATION, FALSE, processId);
 117       if (processHandle != NULL)
 118         {
 119           char buf[1024];
 120           DWORD bufsize = sizeof (buf);
 121           /* The file name can be determined through
 122              GetProcessImageFileName
 123              <https://docs.microsoft.com/en-us/windows/desktop/api/psapi/nf-psapi-getprocessimagefilenamea>
 124              or
 125              QueryFullProcessImageName
 126              <https://docs.microsoft.com/en-us/windows/desktop/api/winbase/nf-winbase-queryfullprocessimagenamea>
 127              The former returns a file name in non-standard notation (it starts
 128              with '\Device\') and may require linking with psapi.dll.
 129              The latter is better, but requires -D_WIN32_WINNT=_WIN32_WINNT_VISTA
 130              or higher.  */
 131           if (QueryFullProcessImageNameFunc (processHandle, 0, buf, &bufsize))
 132             {
 133               if (strlen (buf) >= 11
 134                   && strcmp (buf + strlen (buf) - 11, "\\mintty.exe") == 0)
 135                 result = TRUE;
 136             }
 137           CloseHandle (processHandle);
 138         }
 139     }
 140   return result;
 141 }
 142
 143 #if HAVE_MSVC_INVALID_PARAMETER_HANDLER
 144 static int
 145 _isatty_nothrow (int fd)
 146 {
 147   int result;
 148
 149   TRY_MSVC_INVAL
 150     {
 151       result = _isatty (fd);
 152     }
 153   CATCH_MSVC_INVAL
 154     {
 155       result = 0;
 156     }
 157   DONE_MSVC_INVAL;
 158
 159   return result;
 160 }
 161 #else
 162 # define _isatty_nothrow _isatty
 163 #endif
 164
 165 /* Determine whether FD refers to a console device.  Return 1 if yes.
 166    Return 0 and set errno if no. (ptsname_r relies on the errno value.)  */
 167 int
 168 isatty (int fd)
 169 {
 170   HANDLE h = (HANDLE) _get_osfhandle (fd);
 171   if (h == INVALID_HANDLE_VALUE)
 172     {
 173       errno = EBADF;
 174       return 0;
 175     }
 176   /* _isatty (fd) tests whether GetFileType of the handle is FILE_TYPE_CHAR.
 177      But it does not set errno when it returns 0.  */
 178   if (_isatty_nothrow (fd))
 179     {
 180       if (IsConsoleHandle (h))
 181         return 1;
 182     }
 183   if (IsCygwinConsoleHandle (h))
 184     return 1;
 185   errno = ENOTTY;
 186   return 0;
 187 }