bump product version to 5.0.4.1
[LibreOffice.git] / soltools / cpp / _lex.c
blob3c35320a9bdc1045b30d365c8d4b3f772ff47540
1 /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
2 /*
3 * This file is part of the LibreOffice project.
5 * This Source Code Form is subject to the terms of the Mozilla Public
6 * License, v. 2.0. If a copy of the MPL was not distributed with this
7 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
9 * This file incorporates work covered by the following license notice:
11 * Licensed to the Apache Software Foundation (ASF) under one or more
12 * contributor license agreements. See the NOTICE file distributed
13 * with this work for additional information regarding copyright
14 * ownership. The ASF licenses this file to you under the Apache
15 * License, Version 2.0 (the "License"); you may not use this file
16 * except in compliance with the License. You may obtain a copy of
17 * the License at http://www.apache.org/licenses/LICENSE-2.0 .
20 #include <stdio.h>
21 #include <stdlib.h>
22 #include <string.h>
23 #if (defined(_WIN32) || defined(__IBMC__))
24 #include <io.h>
25 #else
26 #include <unistd.h>
27 #endif
28 #include "cpp.h"
30 * lexical FSM encoding
31 * when in state state, and one of the characters
32 * in ch arrives, enter nextstate.
33 * States >= S_SELF are either final, or at least require special action.
34 * In 'fsm' there is a line for each state X charset X nextstate.
35 * List chars that overwrite previous entries later (e.g. C_ALPH
36 * can be overridden by '_' by a later entry; and C_XX is the
37 * the universal set, and should always be first.
38 * States above S_SELF are represented in the big table as negative values.
39 * S_SELF and S_SELFB encode the resulting token type in the upper bits.
40 * These actions differ in that S_SELF doesn't have a lookahead char,
41 * S_SELFB does.
43 * The encoding is blown out into a big table for time-efficiency.
44 * Entries have
45 * nextstate: 6 bits; ?\ marker: 1 bit; tokentype: 9 bits.
48 #define MAXSTATE 32
49 #define ACT(tok,act) ((tok<<7)+act)
50 #define QBSBIT 0100
51 #define GETACT(st) ((st>>7)&0x1ff)
53 /* character classes */
54 #define C_ALPH 1
55 #define C_NUM 2
56 #define C_XX 3
58 enum state
60 START = 0, NUM1, NUM2, NUM3, ID1, ST1, ST2, ST3, COM1, COM2, COM3, COM4,
61 CC1, CC2, WS1, PLUS1, MINUS1, STAR1, SLASH1, PCT1, SHARP1,
62 CIRC1, GT1, GT2, LT1, LT2, OR1, AND1, ASG1, NOT1, DOTS1,
63 S_SELF = MAXSTATE, S_SELFB, S_EOF, S_NL, S_EOFSTR,
64 S_STNL, S_COMNL, S_EOFCOM, S_COMMENT, S_EOB, S_WS, S_NAME
67 int tottok;
68 int tokkind[256];
69 struct fsm
71 int state; /* if in this state */
72 uchar ch[4]; /* and see one of these characters */
73 int nextstate; /* enter this state if +ve */
76 /*const*/ struct fsm fsm[] = {
77 /* start state */
78 {START, {C_XX}, ACT(UNCLASS, S_SELF)},
79 {START, {' ', '\t', '\v'}, WS1},
80 {START, {C_NUM}, NUM1},
81 {START, {'.'}, NUM3},
82 {START, {C_ALPH}, ID1},
83 {START, {'L'}, ST1},
84 {START, {'"'}, ST2},
85 {START, {'\''}, CC1},
86 {START, {'/'}, COM1},
87 {START, {EOFC}, S_EOF},
88 {START, {'\n'}, S_NL},
89 {START, {'-'}, MINUS1},
90 {START, {'+'}, PLUS1},
91 {START, {'<'}, LT1},
92 {START, {'>'}, GT1},
93 {START, {'='}, ASG1},
94 {START, {'!'}, NOT1},
95 {START, {'&'}, AND1},
96 {START, {'|'}, OR1},
97 {START, {'#'}, SHARP1},
98 {START, {'%'}, PCT1},
99 {START, {'['}, ACT(SBRA, S_SELF)},
100 {START, {']'}, ACT(SKET, S_SELF)},
101 {START, {'('}, ACT(LP, S_SELF)},
102 {START, {')'}, ACT(RP, S_SELF)},
103 {START, {'*'}, STAR1},
104 {START, {','}, ACT(COMMA, S_SELF)},
105 {START, {'?'}, ACT(QUEST, S_SELF)},
106 {START, {':'}, ACT(COLON, S_SELF)},
107 {START, {';'}, ACT(SEMIC, S_SELF)},
108 {START, {'{'}, ACT(CBRA, S_SELF)},
109 {START, {'}'}, ACT(CKET, S_SELF)},
110 {START, {'~'}, ACT(TILDE, S_SELF)},
111 {START, {'^'}, CIRC1},
113 /* saw a digit */
114 {NUM1, {C_XX}, ACT(NUMBER, S_SELFB)},
115 {NUM1, {C_NUM, C_ALPH, '.'}, NUM1},
116 {NUM1, {'E', 'e'}, NUM2},
117 {NUM1, {'_'}, ACT(NUMBER, S_SELFB)},
119 /* saw possible start of exponent, digits-e */
120 {NUM2, {C_XX}, ACT(NUMBER, S_SELFB)},
121 {NUM2, {'+', '-'}, NUM1},
122 {NUM2, {C_NUM, C_ALPH}, NUM1},
123 {NUM2, {'_'}, ACT(NUMBER, S_SELFB)},
125 /* saw a '.', which could be a number or an operator */
126 {NUM3, {C_XX}, ACT(DOT, S_SELFB)},
127 {NUM3, {'.'}, DOTS1},
128 {NUM3, {C_NUM}, NUM1},
130 {DOTS1, {C_XX}, ACT(UNCLASS, S_SELFB)},
131 {DOTS1, {C_NUM}, NUM1},
132 {DOTS1, {'.'}, ACT(ELLIPS, S_SELF)},
134 /* saw a letter or _ */
135 {ID1, {C_XX}, ACT(NAME, S_NAME)},
136 {ID1, {C_ALPH, C_NUM}, ID1},
138 /* saw L (start of wide string?) */
139 {ST1, {C_XX}, ACT(NAME, S_NAME)},
140 {ST1, {C_ALPH, C_NUM}, ID1},
141 {ST1, {'"'}, ST2},
142 {ST1, {'\''}, CC1},
144 /* saw " beginning string */
145 {ST2, {C_XX}, ST2},
146 {ST2, {'"'}, ACT(STRING, S_SELF)},
147 {ST2, {'\\'}, ST3},
148 {ST2, {'\n'}, S_STNL},
149 {ST2, {EOFC}, S_EOFSTR},
151 /* saw \ in string */
152 {ST3, {C_XX}, ST2},
153 {ST3, {'\n'}, S_STNL},
154 {ST3, {EOFC}, S_EOFSTR},
156 /* saw ' beginning character const */
157 {CC1, {C_XX}, CC1},
158 {CC1, {'\''}, ACT(CCON, S_SELF)},
159 {CC1, {'\\'}, CC2},
160 {CC1, {'\n'}, S_STNL},
161 {CC1, {EOFC}, S_EOFSTR},
163 /* saw \ in ccon */
164 {CC2, {C_XX}, CC1},
165 {CC2, {'\n'}, S_STNL},
166 {CC2, {EOFC}, S_EOFSTR},
168 /* saw /, perhaps start of comment */
169 {COM1, {C_XX}, ACT(SLASH, S_SELFB)},
170 {COM1, {'='}, ACT(ASSLASH, S_SELF)},
171 {COM1, {'*'}, COM2},
172 {COM1, {'/'}, COM4},
174 /* saw / followed by *, start of comment */
175 {COM2, {C_XX}, COM2},
176 {COM2, {'\n'}, S_COMNL},
177 {COM2, {'*'}, COM3},
178 {COM2, {EOFC}, S_EOFCOM},
180 /* saw the * possibly ending a comment */
181 {COM3, {C_XX}, COM2},
182 {COM3, {'\n'}, S_COMNL},
183 {COM3, {'*'}, COM3},
184 {COM3, {'/'}, S_COMMENT},
186 /* // comment */
187 {COM4, {C_XX}, COM4},
188 {COM4, {'\n'}, S_NL},
189 {COM4, {EOFC}, S_EOFCOM},
191 /* saw white space, eat it up */
192 {WS1, {C_XX}, S_WS},
193 {WS1, {'\t', '\v', ' '}, WS1},
195 /* saw -, check --, -=, -> */
196 {MINUS1, {C_XX}, ACT(MINUS, S_SELFB)},
197 {MINUS1, {'-'}, ACT(MMINUS, S_SELF)},
198 {MINUS1, {'='}, ACT(ASMINUS, S_SELF)},
199 {MINUS1, {'>'}, ACT(ARROW, S_SELF)},
201 /* saw +, check ++, += */
202 {PLUS1, {C_XX}, ACT(PLUS, S_SELFB)},
203 {PLUS1, {'+'}, ACT(PPLUS, S_SELF)},
204 {PLUS1, {'='}, ACT(ASPLUS, S_SELF)},
206 /* saw <, check <<, <<=, <= */
207 {LT1, {C_XX}, ACT(LT, S_SELFB)},
208 {LT1, {'<'}, LT2},
209 {LT1, {'='}, ACT(LEQ, S_SELF)},
210 {LT2, {C_XX}, ACT(LSH, S_SELFB)},
211 {LT2, {'='}, ACT(ASLSH, S_SELF)},
213 /* saw >, check >>, >>=, >= */
214 {GT1, {C_XX}, ACT(GT, S_SELFB)},
215 {GT1, {'>'}, GT2},
216 {GT1, {'='}, ACT(GEQ, S_SELF)},
217 {GT2, {C_XX}, ACT(RSH, S_SELFB)},
218 {GT2, {'='}, ACT(ASRSH, S_SELF)},
220 /* = */
221 {ASG1, {C_XX}, ACT(ASGN, S_SELFB)},
222 {ASG1, {'='}, ACT(EQ, S_SELF)},
224 /* ! */
225 {NOT1, {C_XX}, ACT(NOT, S_SELFB)},
226 {NOT1, {'='}, ACT(NEQ, S_SELF)},
228 /* & */
229 {AND1, {C_XX}, ACT(AND, S_SELFB)},
230 {AND1, {'&'}, ACT(LAND, S_SELF)},
231 {AND1, {'='}, ACT(ASAND, S_SELF)},
233 /* | */
234 {OR1, {C_XX}, ACT(OR, S_SELFB)},
235 {OR1, {'|'}, ACT(LOR, S_SELF)},
236 {OR1, {'='}, ACT(ASOR, S_SELF)},
238 /* # */
239 {SHARP1, {C_XX}, ACT(SHARP, S_SELFB)},
240 {SHARP1, {'#'}, ACT(DSHARP, S_SELF)},
242 /* % */
243 {PCT1, {C_XX}, ACT(PCT, S_SELFB)},
244 {PCT1, {'='}, ACT(ASPCT, S_SELF)},
246 /* * */
247 {STAR1, {C_XX}, ACT(STAR, S_SELFB)},
248 {STAR1, {'='}, ACT(ASSTAR, S_SELF)},
250 /* ^ */
251 {CIRC1, {C_XX}, ACT(CIRC, S_SELFB)},
252 {CIRC1, {'='}, ACT(ASCIRC, S_SELF)},
254 {-1, "", 0}
257 /* first index is char, second is state */
258 /* increase #states to power of 2 to encourage use of shift */
259 short bigfsm[256][MAXSTATE];
261 void
262 expandlex(void)
264 /* const */ struct fsm *fp;
265 int i, j, nstate;
267 for (fp = fsm; fp->state >= 0; fp++)
269 for (i = 0; fp->ch[i]; i++)
271 nstate = fp->nextstate;
272 if (nstate >= S_SELF)
273 nstate = ~nstate;
274 switch (fp->ch[i])
277 case C_XX: /* random characters */
278 for (j = 0; j < 256; j++)
279 bigfsm[j][fp->state] = (short) nstate;
280 continue;
281 case C_ALPH:
282 for (j = 0; j < 256; j++)
283 if (('a' <= j && j <= 'z') || ('A' <= j && j <= 'Z')
284 || j == '_')
285 bigfsm[j][fp->state] = (short) nstate;
286 continue;
287 case C_NUM:
288 for (j = '0'; j <= '9'; j++)
289 bigfsm[j][fp->state] = (short) nstate;
290 continue;
291 default:
292 bigfsm[fp->ch[i]][fp->state] = (short) nstate;
298 * install special cases for ? (trigraphs), \ (splicing), runes, and
299 * EOB
301 for (i = 0; i < MAXSTATE; i++)
303 for (j = 0; j < 0xFF; j++)
304 if (j == '?' || j == '\\' || j == '\n' || j == '\r')
306 if (bigfsm[j][i] > 0)
307 bigfsm[j][i] = ~bigfsm[j][i];
308 bigfsm[j][i] &= ~QBSBIT;
310 bigfsm[EOB][i] = ~S_EOB;
311 if (bigfsm[EOFC][i] >= 0)
312 bigfsm[EOFC][i] = ~S_EOF;
316 void
317 fixlex(void)
319 /* do C++ comments? */
320 if ((Cplusplus == 0) || (Cflag != 0))
321 bigfsm['/'][COM1] = bigfsm['x'][COM1];
325 * fill in a row of tokens from input, terminated by NL or END
326 * First token is put at trp->lp.
327 * Reset is non-zero when the input buffer can be "rewound."
328 * The value is a flag indicating that possible macros have
329 * been seen in the row.
332 gettokens(Tokenrow * trp, int reset)
334 int c, state, oldstate;
335 uchar *ip;
336 Token *tp, *maxp;
337 int runelen;
338 Source *s = cursource;
339 int nmac = 0;
341 tp = trp->lp;
342 ip = s->inp;
343 if (reset)
345 s->lineinc = 0;
346 if (ip >= s->inl)
347 { /* nothing in buffer */
348 s->inl = s->inb;
349 fillbuf(s);
350 ip = s->inp = s->inb;
352 else
353 if (ip >= s->inb + (3 * INS / 4))
355 memmove(s->inb, ip, 4 + s->inl - ip);
356 s->inl = s->inb + (s->inl - ip);
357 ip = s->inp = s->inb;
360 maxp = &trp->bp[trp->max];
361 runelen = 1;
362 for (;;)
364 continue2:
365 if (tp >= maxp)
367 trp->lp = tp;
368 tp = growtokenrow(trp);
369 maxp = &trp->bp[trp->max];
371 tp->type = UNCLASS;
372 tp->t = ip;
373 tp->wslen = 0;
374 tp->flag = 0;
375 state = START;
376 for (;;)
378 oldstate = state;
380 c = *ip;
382 if ((state = bigfsm[c][state]) >= 0)
384 ip += runelen;
385 runelen = 1;
386 continue;
388 state = ~state;
389 reswitch:
390 switch (state & 0177)
392 case S_SELF:
393 ip += runelen;
394 runelen = 1;
395 /*fall-through*/
396 case S_SELFB:
397 tp->type = (unsigned char) GETACT(state);
398 tp->len = ip - tp->t;
399 tp++;
400 goto continue2;
402 case S_NAME: /* like S_SELFB but with nmac check */
403 tp->type = NAME;
404 tp->len = ip - tp->t;
405 nmac |= quicklook(tp->t[0], tp->len > 1 ? tp->t[1] : 0);
406 tp++;
407 goto continue2;
409 case S_WS:
410 tp->wslen = ip - tp->t;
411 tp->t = ip;
412 state = START;
413 continue;
415 default:
416 if ((state & QBSBIT) == 0)
418 ip += runelen;
419 runelen = 1;
420 continue;
422 state &= ~QBSBIT;
423 s->inp = ip;
425 if (c == '\n')
427 while (s->inp + 1 >= s->inl && fillbuf(s) != EOF);
429 if (s->inp[1] == '\r')
431 memmove(s->inp + 1, s->inp + 2, s->inl - s->inp + 2);
432 s->inl -= 1;
435 goto reswitch;
438 if (c == '\r')
440 while (s->inp + 1 >= s->inl && fillbuf(s) != EOF);
442 if (s->inp[1] == '\n')
444 memmove(s->inp, s->inp + 1, s->inl - s->inp + 1);
445 s->inl -= 1;
447 else
448 *s->inp = '\n';
450 state = oldstate;
451 continue;
454 if (c == '?')
455 { /* check trigraph */
456 if (trigraph(s))
458 state = oldstate;
459 continue;
461 goto reswitch;
463 if (c == '\\')
464 { /* line-folding */
465 if (foldline(s))
467 s->lineinc++;
468 state = oldstate;
469 continue;
471 goto reswitch;
473 error(WARNING, "Lexical botch in cpp");
474 ip += runelen;
475 runelen = 1;
476 continue;
478 case S_EOB:
479 s->inp = ip;
480 fillbuf(cursource);
481 state = oldstate;
482 continue;
484 case S_EOF:
485 tp->type = END;
486 tp->len = 0;
487 s->inp = ip;
488 if (tp != trp->bp && (tp - 1)->type != NL && cursource->fd != -1)
489 error(WARNING, "No newline at end of file");
490 trp->lp = tp + 1;
491 return nmac;
493 case S_STNL:
494 error(ERROR, "Unterminated string or char const");
495 case S_NL:
496 tp->t = ip;
497 tp->type = NL;
498 tp->len = 1;
499 tp->wslen = 0;
500 s->lineinc++;
501 s->inp = ip + 1;
502 trp->lp = tp + 1;
503 return nmac;
505 case S_EOFSTR:
506 error(FATAL, "EOF in string or char constant");
507 break;
509 case S_COMNL:
510 s->lineinc++;
511 state = COM2;
512 ip += runelen;
513 runelen = 1;
514 continue;
516 case S_EOFCOM:
517 error(WARNING, "EOF inside comment");
518 --ip;
519 case S_COMMENT:
520 if (!Cflag)
522 tp->t = ++ip;
523 tp->t[-1] = ' ';
524 tp->wslen = 1;
525 state = START;
526 continue;
528 else
530 runelen = 1;
531 s->lineinc = 0;;
532 tp->type = COMMENT;
533 tp->flag |= XTWS;
536 break;
538 ip += runelen;
539 runelen = 1;
540 tp->len = ip - tp->t;
541 tp++;
545 /* have seen ?; handle the trigraph it starts (if any) else 0 */
547 trigraph(Source * s)
549 uchar c;
551 while (s->inp + 2 >= s->inl && fillbuf(s) != EOF);
553 if (s->inp[1] != '?')
554 return 0;
555 c = 0;
556 switch (s->inp[2])
558 case '=':
559 c = '#';
560 break;
561 case '(':
562 c = '[';
563 break;
564 case '/':
565 c = '\\';
566 break;
567 case ')':
568 c = ']';
569 break;
570 case '\'':
571 c = '^';
572 break;
573 case '<':
574 c = '{';
575 break;
576 case '!':
577 c = '|';
578 break;
579 case '>':
580 c = '}';
581 break;
582 case '-':
583 c = '~';
584 break;
586 if (c)
588 *s->inp = c;
589 memmove(s->inp + 1, s->inp + 3, s->inl - s->inp + 2);
590 s->inl -= 2;
592 return c;
596 foldline(Source * s)
598 int n = 1;
600 /* skip pending wihite spaces */
601 while ((s->inp[n] == ' ') || (s->inp[n] == '\t'))
603 n++;
604 if ((s->inp + n >= s->inl) && (fillbuf(s) == EOF))
605 break;
608 /* refill buffer */
609 while (s->inp + (n + 1) >= s->inl && fillbuf(s) != EOF);
611 /* skip DOS line ends */
612 if (((s->inp[n] == '\r') && (s->inp[n+1] == '\n')) ||
613 ((s->inp[n] == '\n') && (s->inp[n+1] == '\r')))
614 n++;
616 if ((s->inp[n] == '\n') || (s->inp[n] == '\r'))
618 memmove(s->inp, s->inp + n + 1, s->inl - s->inp + n + 2);
619 s->inl -= n + 1;
620 return 1;
622 return 0;
626 fillbuf(Source * s)
628 int n;
630 if (s->fd < 0 || (n = read(s->fd, (char *) s->inl, INS / 8)) <= 0)
631 n = 0;
632 s->inl += n;
633 s->inl[0] = s->inl[1] = s->inl[2] = s->inl[3] = EOB;
634 if (n == 0)
636 s->inl[0] = s->inl[1] = s->inl[2] = s->inl[3] = EOFC;
637 return EOF;
639 return 0;
643 * Push down to new source of characters.
644 * If fd>0 and str==NULL, then from a file `name';
645 * if fd==-1 and str, then from the string.
647 Source *
648 setsource(char *name, int path, int fd, char *str, int wrap)
650 Source *s = new(Source);
651 size_t len;
653 s->line = 1;
654 s->lineinc = 0;
655 s->fd = fd;
656 s->filename = name;
657 s->next = cursource;
658 s->ifdepth = 0;
659 s->pathdepth = path;
660 s->wrap = wrap;
662 cursource = s;
664 if (s->wrap)
665 genwrap(0);
667 /* slop at right for EOB */
668 if (str)
670 len = strlen(str);
671 s->inb = domalloc(len + 4);
672 s->inp = s->inb;
673 strncpy((char *) s->inp, str, len);
675 else
677 s->inb = domalloc(INS + 4);
678 s->inp = s->inb;
679 len = 0;
681 s->inl = s->inp + len;
682 s->inl[0] = s->inl[1] = EOB;
684 return s;
687 void
688 unsetsource(void)
690 Source *s = cursource;
692 if (s->wrap)
693 genwrap(1);
695 if (s->fd >= 0)
697 close(s->fd);
698 dofree(s->inb);
700 cursource = s->next;
701 dofree(s);
704 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */