Win32: let mingw_execve() return an int
[git/mingw/4msysgit.git] / compat / winansi.c
blobefc5bb3a4b63166eccb33f6ec6a8ad57e4c9ac36
1 /*
2 * Copyright 2008 Peter Harris <git@peter.is-a-geek.org>
3 */
5 #undef NOGDI
6 #include "../git-compat-util.h"
7 #include <wingdi.h>
8 #include <winreg.h>
11 ANSI codes used by git: m, K
13 This file is git-specific. Therefore, this file does not attempt
14 to implement any codes that are not used by git.
17 static HANDLE console;
18 static WORD plain_attr;
19 static WORD attr;
20 static int negative;
21 static int non_ascii_used = 0;
22 static HANDLE hthread, hread, hwrite;
23 static HANDLE hconsole1, hconsole2;
25 #ifdef __MINGW32__
26 typedef struct _CONSOLE_FONT_INFOEX {
27 ULONG cbSize;
28 DWORD nFont;
29 COORD dwFontSize;
30 UINT FontFamily;
31 UINT FontWeight;
32 WCHAR FaceName[LF_FACESIZE];
33 } CONSOLE_FONT_INFOEX, *PCONSOLE_FONT_INFOEX;
34 #endif
36 typedef BOOL (WINAPI *PGETCURRENTCONSOLEFONTEX)(HANDLE, BOOL,
37 PCONSOLE_FONT_INFOEX);
39 static void warn_if_raster_font(void)
41 DWORD fontFamily = 0;
42 PGETCURRENTCONSOLEFONTEX pGetCurrentConsoleFontEx;
44 /* don't bother if output was ascii only */
45 if (!non_ascii_used)
46 return;
48 /* GetCurrentConsoleFontEx is available since Vista */
49 pGetCurrentConsoleFontEx = (PGETCURRENTCONSOLEFONTEX) GetProcAddress(
50 GetModuleHandle("kernel32.dll"),
51 "GetCurrentConsoleFontEx");
52 if (pGetCurrentConsoleFontEx) {
53 CONSOLE_FONT_INFOEX cfi;
54 cfi.cbSize = sizeof(cfi);
55 if (pGetCurrentConsoleFontEx(console, 0, &cfi))
56 fontFamily = cfi.FontFamily;
57 } else {
58 /* pre-Vista: check default console font in registry */
59 HKEY hkey;
60 if (ERROR_SUCCESS == RegOpenKeyExA(HKEY_CURRENT_USER, "Console",
61 0, KEY_READ, &hkey)) {
62 DWORD size = sizeof(fontFamily);
63 RegQueryValueExA(hkey, "FontFamily", NULL, NULL,
64 (LPVOID) &fontFamily, &size);
65 RegCloseKey(hkey);
69 if (!(fontFamily & TMPF_TRUETYPE)) {
70 const wchar_t *msg = L"\nWarning: Your console font probably "
71 L"doesn\'t support Unicode. If you experience strange "
72 L"characters in the output, consider switching to a "
73 L"TrueType font such as Consolas!\n";
74 DWORD dummy;
75 WriteConsoleW(console, msg, wcslen(msg), &dummy, NULL);
79 static int is_console(int fd)
81 CONSOLE_SCREEN_BUFFER_INFO sbi;
82 HANDLE hcon;
84 static int initialized = 0;
86 /* get OS handle of the file descriptor */
87 hcon = (HANDLE) _get_osfhandle(fd);
88 if (hcon == INVALID_HANDLE_VALUE)
89 return 0;
91 /* check if its a device (i.e. console, printer, serial port) */
92 if (GetFileType(hcon) != FILE_TYPE_CHAR)
93 return 0;
95 /* check if its a handle to a console output screen buffer */
96 if (!GetConsoleScreenBufferInfo(hcon, &sbi))
97 return 0;
99 /* initialize attributes */
100 if (!initialized) {
101 console = hcon;
102 attr = plain_attr = sbi.wAttributes;
103 negative = 0;
104 initialized = 1;
107 return 1;
110 #define BUFFER_SIZE 4096
111 #define MAX_PARAMS 16
113 static void write_console(unsigned char *str, size_t len)
115 /* only called from console_thread, so a static buffer will do */
116 static wchar_t wbuf[2 * BUFFER_SIZE + 1];
117 DWORD dummy;
119 /* convert utf-8 to utf-16 */
120 int wlen = xutftowcsn(wbuf, (char*) str, ARRAY_SIZE(wbuf), len);
122 /* write directly to console */
123 WriteConsoleW(console, wbuf, wlen, &dummy, NULL);
125 /* remember if non-ascii characters are printed */
126 if (wlen != len)
127 non_ascii_used = 1;
130 #define FOREGROUND_ALL (FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE)
131 #define BACKGROUND_ALL (BACKGROUND_RED | BACKGROUND_GREEN | BACKGROUND_BLUE)
133 static void set_console_attr(void)
135 WORD attributes = attr;
136 if (negative) {
137 attributes &= ~FOREGROUND_ALL;
138 attributes &= ~BACKGROUND_ALL;
140 /* This could probably use a bitmask
141 instead of a series of ifs */
142 if (attr & FOREGROUND_RED)
143 attributes |= BACKGROUND_RED;
144 if (attr & FOREGROUND_GREEN)
145 attributes |= BACKGROUND_GREEN;
146 if (attr & FOREGROUND_BLUE)
147 attributes |= BACKGROUND_BLUE;
149 if (attr & BACKGROUND_RED)
150 attributes |= FOREGROUND_RED;
151 if (attr & BACKGROUND_GREEN)
152 attributes |= FOREGROUND_GREEN;
153 if (attr & BACKGROUND_BLUE)
154 attributes |= FOREGROUND_BLUE;
156 SetConsoleTextAttribute(console, attributes);
159 static void erase_in_line(void)
161 CONSOLE_SCREEN_BUFFER_INFO sbi;
162 DWORD dummy; /* Needed for Windows 7 (or Vista) regression */
164 if (!console)
165 return;
167 GetConsoleScreenBufferInfo(console, &sbi);
168 FillConsoleOutputCharacterA(console, ' ',
169 sbi.dwSize.X - sbi.dwCursorPosition.X, sbi.dwCursorPosition,
170 &dummy);
173 static void set_attr(char func, const int *params, int paramlen)
175 int i;
176 switch (func) {
177 case 'm':
178 for (i = 0; i < paramlen; i++) {
179 switch (params[i]) {
180 case 0: /* reset */
181 attr = plain_attr;
182 negative = 0;
183 break;
184 case 1: /* bold */
185 attr |= FOREGROUND_INTENSITY;
186 break;
187 case 2: /* faint */
188 case 22: /* normal */
189 attr &= ~FOREGROUND_INTENSITY;
190 break;
191 case 3: /* italic */
192 /* Unsupported */
193 break;
194 case 4: /* underline */
195 case 21: /* double underline */
196 /* Wikipedia says this flag does nothing */
197 /* Furthermore, mingw doesn't define this flag
198 attr |= COMMON_LVB_UNDERSCORE; */
199 break;
200 case 24: /* no underline */
201 /* attr &= ~COMMON_LVB_UNDERSCORE; */
202 break;
203 case 5: /* slow blink */
204 case 6: /* fast blink */
205 /* We don't have blink, but we do have
206 background intensity */
207 attr |= BACKGROUND_INTENSITY;
208 break;
209 case 25: /* no blink */
210 attr &= ~BACKGROUND_INTENSITY;
211 break;
212 case 7: /* negative */
213 negative = 1;
214 break;
215 case 27: /* positive */
216 negative = 0;
217 break;
218 case 8: /* conceal */
219 case 28: /* reveal */
220 /* Unsupported */
221 break;
222 case 30: /* Black */
223 attr &= ~FOREGROUND_ALL;
224 break;
225 case 31: /* Red */
226 attr &= ~FOREGROUND_ALL;
227 attr |= FOREGROUND_RED;
228 break;
229 case 32: /* Green */
230 attr &= ~FOREGROUND_ALL;
231 attr |= FOREGROUND_GREEN;
232 break;
233 case 33: /* Yellow */
234 attr &= ~FOREGROUND_ALL;
235 attr |= FOREGROUND_RED | FOREGROUND_GREEN;
236 break;
237 case 34: /* Blue */
238 attr &= ~FOREGROUND_ALL;
239 attr |= FOREGROUND_BLUE;
240 break;
241 case 35: /* Magenta */
242 attr &= ~FOREGROUND_ALL;
243 attr |= FOREGROUND_RED | FOREGROUND_BLUE;
244 break;
245 case 36: /* Cyan */
246 attr &= ~FOREGROUND_ALL;
247 attr |= FOREGROUND_GREEN | FOREGROUND_BLUE;
248 break;
249 case 37: /* White */
250 attr |= FOREGROUND_RED |
251 FOREGROUND_GREEN |
252 FOREGROUND_BLUE;
253 break;
254 case 38: /* Unknown */
255 break;
256 case 39: /* reset */
257 attr &= ~FOREGROUND_ALL;
258 attr |= (plain_attr & FOREGROUND_ALL);
259 break;
260 case 40: /* Black */
261 attr &= ~BACKGROUND_ALL;
262 break;
263 case 41: /* Red */
264 attr &= ~BACKGROUND_ALL;
265 attr |= BACKGROUND_RED;
266 break;
267 case 42: /* Green */
268 attr &= ~BACKGROUND_ALL;
269 attr |= BACKGROUND_GREEN;
270 break;
271 case 43: /* Yellow */
272 attr &= ~BACKGROUND_ALL;
273 attr |= BACKGROUND_RED | BACKGROUND_GREEN;
274 break;
275 case 44: /* Blue */
276 attr &= ~BACKGROUND_ALL;
277 attr |= BACKGROUND_BLUE;
278 break;
279 case 45: /* Magenta */
280 attr &= ~BACKGROUND_ALL;
281 attr |= BACKGROUND_RED | BACKGROUND_BLUE;
282 break;
283 case 46: /* Cyan */
284 attr &= ~BACKGROUND_ALL;
285 attr |= BACKGROUND_GREEN | BACKGROUND_BLUE;
286 break;
287 case 47: /* White */
288 attr |= BACKGROUND_RED |
289 BACKGROUND_GREEN |
290 BACKGROUND_BLUE;
291 break;
292 case 48: /* Unknown */
293 break;
294 case 49: /* reset */
295 attr &= ~BACKGROUND_ALL;
296 attr |= (plain_attr & BACKGROUND_ALL);
297 break;
298 default:
299 /* Unsupported code */
300 break;
303 set_console_attr();
304 break;
305 case 'K':
306 erase_in_line();
307 break;
308 default:
309 /* Unsupported code */
310 break;
314 enum {
315 TEXT = 0, ESCAPE = 033, BRACKET = '['
318 static DWORD WINAPI console_thread(LPVOID unused)
320 unsigned char buffer[BUFFER_SIZE];
321 DWORD bytes;
322 int start, end = 0, c, parampos = 0, state = TEXT;
323 int params[MAX_PARAMS];
325 while (1) {
326 /* read next chunk of bytes from the pipe */
327 if (!ReadFile(hread, buffer + end, BUFFER_SIZE - end, &bytes,
328 NULL)) {
329 /* exit if pipe has been closed or disconnected */
330 if (GetLastError() == ERROR_PIPE_NOT_CONNECTED ||
331 GetLastError() == ERROR_BROKEN_PIPE)
332 break;
333 /* ignore other errors */
334 continue;
337 /* scan the bytes and handle ANSI control codes */
338 bytes += end;
339 start = end = 0;
340 while (end < bytes) {
341 c = buffer[end++];
342 switch (state) {
343 case TEXT:
344 if (c == ESCAPE) {
345 /* print text seen so far */
346 if (end - 1 > start)
347 write_console(buffer + start,
348 end - 1 - start);
350 /* then start parsing escape sequence */
351 start = end - 1;
352 memset(params, 0, sizeof(params));
353 parampos = 0;
354 state = ESCAPE;
356 break;
358 case ESCAPE:
359 /* continue if "\033[", otherwise bail out */
360 state = (c == BRACKET) ? BRACKET : TEXT;
361 break;
363 case BRACKET:
364 /* parse [0-9;]* into array of parameters */
365 if (c >= '0' && c <= '9') {
366 params[parampos] *= 10;
367 params[parampos] += c - '0';
368 } else if (c == ';') {
370 * next parameter, bail out if out of
371 * bounds
373 parampos++;
374 if (parampos >= MAX_PARAMS)
375 state = TEXT;
376 } else {
378 * end of escape sequence, change
379 * console attributes
381 set_attr(c, params, parampos + 1);
382 start = end;
383 state = TEXT;
385 break;
389 /* print remaining text unless parsing an escape sequence */
390 if (state == TEXT && end > start) {
391 /* check for incomplete UTF-8 sequences and fix end */
392 if (buffer[end - 1] >= 0x80) {
393 if (buffer[end -1] >= 0xc0)
394 end--;
395 else if (end - 1 > start &&
396 buffer[end - 2] >= 0xe0)
397 end -= 2;
398 else if (end - 2 > start &&
399 buffer[end - 3] >= 0xf0)
400 end -= 3;
403 /* print remaining complete UTF-8 sequences */
404 if (end > start)
405 write_console(buffer + start, end - start);
407 /* move remaining bytes to the front */
408 if (end < bytes)
409 memmove(buffer, buffer + end, bytes - end);
410 end = bytes - end;
411 } else {
412 /* all data has been consumed, mark buffer empty */
413 end = 0;
417 /* check if the console font supports unicode */
418 warn_if_raster_font();
420 CloseHandle(hread);
421 return 0;
424 static void winansi_exit(void)
426 /* flush all streams */
427 _flushall();
429 /* signal console thread to exit */
430 FlushFileBuffers(hwrite);
431 DisconnectNamedPipe(hwrite);
433 /* wait for console thread to copy remaining data */
434 WaitForSingleObject(hthread, INFINITE);
436 /* cleanup handles... */
437 CloseHandle(hwrite);
438 CloseHandle(hthread);
441 static void die_lasterr(const char *fmt, ...)
443 va_list params;
444 va_start(params, fmt);
445 errno = err_win_to_posix(GetLastError());
446 die_errno(fmt, params);
447 va_end(params);
450 static HANDLE duplicate_handle(HANDLE hnd)
452 HANDLE hresult, hproc = GetCurrentProcess();
453 if (!DuplicateHandle(hproc, hnd, hproc, &hresult, 0, TRUE,
454 DUPLICATE_SAME_ACCESS))
455 die_lasterr("DuplicateHandle(%li) failed", (long) hnd);
456 return hresult;
461 * Make MSVCRT's internal file descriptor control structure accessible
462 * so that we can tweak OS handles and flags directly (we need MSVCRT
463 * to treat our pipe handle as if it were a console).
465 * We assume that the ioinfo structure (exposed by MSVCRT.dll via
466 * __pioinfo) starts with the OS handle and the flags. The exact size
467 * varies between MSVCRT versions, so we try different sizes until
468 * toggling the FDEV bit of _pioinfo(1)->osflags is reflected in
469 * isatty(1).
471 typedef struct {
472 HANDLE osfhnd;
473 char osflags;
474 } ioinfo;
476 extern __declspec(dllimport) ioinfo *__pioinfo[];
478 static size_t sizeof_ioinfo = 0;
480 #define IOINFO_L2E 5
481 #define IOINFO_ARRAY_ELTS (1 << IOINFO_L2E)
483 #define FDEV 0x40
485 static inline ioinfo* _pioinfo(int fd)
487 return (ioinfo*)((char*)__pioinfo[fd >> IOINFO_L2E] +
488 (fd & (IOINFO_ARRAY_ELTS - 1)) * sizeof_ioinfo);
491 static int init_sizeof_ioinfo()
493 int istty, wastty;
494 /* don't init twice */
495 if (sizeof_ioinfo)
496 return sizeof_ioinfo >= 256;
498 sizeof_ioinfo = sizeof(ioinfo);
499 wastty = isatty(1);
500 while (sizeof_ioinfo < 256) {
501 /* toggle FDEV flag, check isatty, then toggle back */
502 _pioinfo(1)->osflags ^= FDEV;
503 istty = isatty(1);
504 _pioinfo(1)->osflags ^= FDEV;
505 /* return if we found the correct size */
506 if (istty != wastty)
507 return 0;
508 sizeof_ioinfo += sizeof(void*);
510 error("Tweaking file descriptors doesn't work with this MSVCRT.dll");
511 return 1;
514 static HANDLE swap_osfhnd(int fd, HANDLE new_handle)
516 ioinfo *pioinfo;
517 HANDLE old_handle;
519 /* init ioinfo size if we haven't done so */
520 if (init_sizeof_ioinfo())
521 return INVALID_HANDLE_VALUE;
523 /* get ioinfo pointer and change the handles */
524 pioinfo = _pioinfo(fd);
525 old_handle = pioinfo->osfhnd;
526 pioinfo->osfhnd = new_handle;
527 return old_handle;
530 void winansi_init(void)
532 int con1, con2;
533 char name[32];
535 /* check if either stdout or stderr is a console output screen buffer */
536 con1 = is_console(1);
537 con2 = is_console(2);
538 if (!con1 && !con2)
539 return;
541 /* create a named pipe to communicate with the console thread */
542 sprintf(name, "\\\\.\\pipe\\winansi%lu", GetCurrentProcessId());
543 hwrite = CreateNamedPipe(name, PIPE_ACCESS_OUTBOUND,
544 PIPE_TYPE_BYTE | PIPE_WAIT, 1, BUFFER_SIZE, 0, 0, NULL);
545 if (hwrite == INVALID_HANDLE_VALUE)
546 die_lasterr("CreateNamedPipe failed");
548 hread = CreateFile(name, GENERIC_READ, 0, NULL, OPEN_EXISTING, 0, NULL);
549 if (hread == INVALID_HANDLE_VALUE)
550 die_lasterr("CreateFile for named pipe failed");
552 /* start console spool thread on the pipe's read end */
553 hthread = CreateThread(NULL, 0, console_thread, NULL, 0, NULL);
554 if (hthread == INVALID_HANDLE_VALUE)
555 die_lasterr("CreateThread(console_thread) failed");
557 /* schedule cleanup routine */
558 if (atexit(winansi_exit))
559 die_errno("atexit(winansi_exit) failed");
561 /* redirect stdout / stderr to the pipe */
562 if (con1)
563 hconsole1 = swap_osfhnd(1, duplicate_handle(hwrite));
564 if (con2)
565 hconsole2 = swap_osfhnd(2, duplicate_handle(hwrite));
569 * Returns the real console handle if stdout / stderr is a pipe redirecting
570 * to the console. Allows spawn / exec to pass the console to the next process.
572 HANDLE winansi_get_osfhandle(int fd)
574 HANDLE hnd = (HANDLE) _get_osfhandle(fd);
575 if ((fd == 1 || fd == 2) && isatty(fd)
576 && GetFileType(hnd) == FILE_TYPE_PIPE)
577 return (fd == 1) ? hconsole1 : hconsole2;
578 return hnd;