idlc/source/preproc/tokens.c

   1 /*************************************************************************
   2  *
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * Copyright 2008 by Sun Microsystems, Inc.
   6  *
   7  * OpenOffice.org - a multi-platform office productivity suite
   8  *
   9  * $RCSfile: tokens.c,v $
  10  * $Revision: 1.6 $
  11  *
  12  * This file is part of OpenOffice.org.
  13  *
  14  * OpenOffice.org is free software: you can redistribute it and/or modify
  15  * it under the terms of the GNU Lesser General Public License version 3
  16  * only, as published by the Free Software Foundation.
  17  *
  18  * OpenOffice.org is distributed in the hope that it will be useful,
  19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21  * GNU Lesser General Public License version 3 for more details
  22  * (a copy is included in the LICENSE file that accompanied this code).
  23  *
  24  * You should have received a copy of the GNU Lesser General Public License
  25  * version 3 along with OpenOffice.org.  If not, see
  26  * <http://www.openoffice.org/license.html>
  27  * for a copy of the LGPLv3 License.
  28  *
  29  ************************************************************************/
  30 #include <stdio.h>
  31 #include <stdlib.h>
  32 #include <string.h>
  33 #include <ctype.h>
  34 #if (defined(_WIN32) || defined(_MSDOS) || defined(__IBMC__))
  35 #include <io.h>
  36 #else
  37 #include <unistd.h>
  38 #endif
  39 #include "cpp.h"
  40
  41
  42 static char wbuf[4 * OBS];
  43 static char *wbp = wbuf;
  44 static int EBCDIC_ExternTokenDetected = 0;
  45 static int EBCDIC_StartTokenDetected = 0;
  46
  47 unsigned char toLatin1[256] =
  48 {
  49     0x00, 0x01, 0x02, 0x03, 0x9c, 0x09, 0x86, 0x7f, 0x97, 0x8d,
  50     0x8e, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13,
  51     0x9d, 0x0a, 0x08, 0x87, 0x18, 0x19, 0x92, 0x8f, 0x1c, 0x1d,
  52     0x1e, 0x1f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x17, 0x1b,
  53     0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x05, 0x06, 0x07, 0x90, 0x91,
  54     0x16, 0x93, 0x94, 0x95, 0x96, 0x04, 0x98, 0x99, 0x9a, 0x9b,
  55     0x14, 0x15, 0x9e, 0x1a, 0x20, 0xa0, 0xe2, 0xe4, 0xe0, 0xe1,
  56     0xe3, 0xe5, 0xe7, 0xf1, 0xa2, 0x2e, 0x3c, 0x28, 0x2b, 0x7c,
  57     0x26, 0xe9, 0xea, 0xeb, 0xe8, 0xed, 0xee, 0xef, 0xec, 0xdf,
  58     0x21, 0x24, 0x2a, 0x29, 0x3b, 0x5e, 0x2d, 0x2f, 0xc2, 0xc4,
  59     0xc0, 0xc1, 0xc3, 0xc5, 0xc7, 0xd1, 0xa6, 0x2c, 0x25, 0x5f,
  60     0x3e, 0x3f, 0xf8, 0xc9, 0xca, 0xcb, 0xc8, 0xcd, 0xce, 0xcf,
  61     0xcc, 0x60, 0x3a, 0x23, 0x40, 0x27, 0x3d, 0x22,
  62     0xd8, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69,
  63     0xab, 0xbb, 0xf0, 0xfd, 0xfe, 0xb1, 0xb0, 0x6a, 0x6b, 0x6c,
  64     0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0xaa, 0xba, 0xe6, 0xb8,
  65     0xc6, 0xa4, 0xb5, 0x7e, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78,
  66     0x79, 0x7a, 0xa1, 0xbf, 0xd0, 0x5b, 0xde, 0xae, 0xac, 0xa3,
  67     0xa5, 0xb7, 0xa9, 0xa7, 0xb6, 0xbc, 0xbd, 0xbe, 0xdd, 0xa8,
  68     0xaf, 0x5d, 0xb4, 0xd7, 0x7b, 0x41, 0x42, 0x43, 0x44, 0x45,
  69     0x46, 0x47, 0x48, 0x49, 0xad, 0xf4, 0xf6, 0xf2, 0xf3, 0xf5,
  70     0x7d, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52,
  71     0xb9, 0xfb, 0xfc, 0xf9, 0xfa, 0xff, 0x5c, 0xf7, 0x53, 0x54,
  72     0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0xb2, 0xd4, 0xd6, 0xd2,
  73     0xd3, 0xd5, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
  74     0x38, 0x39, 0xb3, 0xdb, 0xdc, 0xd9, 0xda, 0x9f
  75 };
  76
  77 #define MASK    "\\x%x"
  78
  79 int
  80     memcpy_EBCDIC( char * pwbuf, uchar *p, int len )
  81 {
  82     int currpos = 0;
  83     int processedchars = 0;
  84
  85     if( len == 0 )
  86         return 0;
  87
  88     if( len == 1 )
  89     {
  90         *pwbuf = *p;
  91         return 1;
  92     }
  93
  94     /* copy spaces until " or ' */
  95     while( (p[ processedchars ] != '\"') && (p[ processedchars ] != '\'') )
  96         pwbuf[ currpos++ ] = p[ processedchars++ ];
  97
  98     /* copy first " or ' */
  99     pwbuf[ currpos++ ] = p[ processedchars++ ];
 100
 101     /* convert all characters until " or ' */
 102     while( processedchars < (len - 1) )
 103     {
 104         if( p[ processedchars ] == '\\' )
 105         {
 106             switch( p[ ++processedchars ] )
 107             {
 108                 case 'n':
 109                     currpos += sprintf( &pwbuf[ currpos ], MASK, toLatin1['\n'] );
 110                     processedchars++;
 111                     break;
 112
 113                 case 't':
 114                     currpos += sprintf( &pwbuf[ currpos ], MASK, toLatin1['\t'] );
 115                     processedchars++;
 116                     break;
 117
 118                 case 'v':
 119                     currpos += sprintf( &pwbuf[ currpos ], MASK, toLatin1['\v'] );
 120                     processedchars++;
 121                     break;
 122
 123                 case 'b':
 124                     currpos += sprintf( &pwbuf[ currpos ], MASK, toLatin1['\b'] );
 125                     processedchars++;
 126                     break;
 127
 128                 case 'r':
 129                     currpos += sprintf( &pwbuf[ currpos ], MASK, toLatin1['\r'] );
 130                     processedchars++;
 131                     break;
 132
 133                 case 'f':
 134                     currpos += sprintf( &pwbuf[ currpos ], MASK, toLatin1['\f'] );
 135                     processedchars++;
 136                     break;
 137
 138                 case 'a':
 139                     currpos += sprintf( &pwbuf[ currpos ], MASK, toLatin1['\a'] );
 140                     processedchars++;
 141                     break;
 142
 143                 case '\\':
 144                     currpos += sprintf( &pwbuf[ currpos ], MASK, toLatin1['\\'] );
 145                     processedchars++;
 146                     break;
 147
 148                 case '?':
 149                     currpos += sprintf( &pwbuf[ currpos ], MASK, toLatin1['\?'] );
 150                     processedchars++;
 151                     break;
 152
 153                 case '\'':
 154                     currpos += sprintf( &pwbuf[ currpos ], MASK, toLatin1['\''] );
 155                     processedchars++;
 156                     break;
 157
 158                 case '"':
 159                     currpos += sprintf( &pwbuf[ currpos ], MASK, toLatin1['\"'] );
 160                     processedchars++;
 161                     break;
 162
 163                 /* octal coded character? -> copy */
 164                 case '0':
 165                 case '1':
 166                 case '2':
 167                 case '3':
 168                 case '4':
 169                 case '5':
 170                 case '6':
 171                 case '7':
 172                     {
 173                     int startpos = currpos;
 174
 175                     pwbuf[ currpos++ ] = '\\';
 176
 177                     while( p[ processedchars ] >= '0' && p[ processedchars ] <= '7' && (currpos < startpos + 4) )
 178                           pwbuf[ currpos++ ] = (unsigned char)p[ processedchars++ ];
 179                     break;
 180                     }
 181
 182                 /* hex coded character? -> copy */
 183                 case 'x':
 184                 case 'X':
 185                     {
 186                     int startpos = currpos;
 187
 188                     pwbuf[ currpos++ ] = '\\';
 189                     pwbuf[ currpos++ ] = 'x';
 190                     processedchars++;
 191
 192                     while( isxdigit( p[ processedchars ] ) && (currpos < startpos + 4) )
 193                           pwbuf[ currpos++ ] = (unsigned char)p[ processedchars++ ];
 194                     break;
 195                     }
 196
 197             }
 198         }
 199         else
 200             currpos += sprintf( &pwbuf[ currpos ], MASK, toLatin1[p[ processedchars++ ]] );
 201
 202     }
 203
 204     /* copy last " or ' */
 205     pwbuf[ currpos++ ] = p[ processedchars ];
 206
 207     return currpos;
 208 }
 209
 210 void
 211     maketokenrow(int size, Tokenrow * trp)
 212 {
 213     trp->max = size;
 214     if (size > 0)
 215         trp->bp = (Token *) domalloc(size * sizeof(Token));
 216     else
 217         trp->bp = NULL;
 218     trp->tp = trp->bp;
 219     trp->lp = trp->bp;
 220 }
 221
 222 Token *
 223     growtokenrow(Tokenrow * trp)
 224 {
 225     int ncur = trp->tp - trp->bp;
 226     int nlast = trp->lp - trp->bp;
 227
 228     trp->max = 3 * trp->max / 2 + 1;
 229     trp->bp = (Token *) realloc(trp->bp, trp->max * sizeof(Token));
 230     trp->lp = &trp->bp[nlast];
 231     trp->tp = &trp->bp[ncur];
 232     return trp->lp;
 233 }
 234
 235 /*
 236  * Compare a row of tokens, ignoring the content of WS; return !=0 if different
 237  */
 238 int
 239     comparetokens(Tokenrow * tr1, Tokenrow * tr2)
 240 {
 241     Token *tp1, *tp2;
 242
 243     tp1 = tr1->tp;
 244     tp2 = tr2->tp;
 245     if (tr1->lp - tp1 != tr2->lp - tp2)
 246         return 1;
 247     for (; tp1 < tr1->lp; tp1++, tp2++)
 248     {
 249         if (tp1->type != tp2->type
 250             || (tp1->wslen == 0) != (tp2->wslen == 0)
 251             || tp1->len != tp2->len
 252             || strncmp((char *) tp1->t, (char *) tp2->t, tp1->len) != 0)
 253             return 1;
 254     }
 255     return 0;
 256 }
 257
 258 /*
 259  * replace ntok tokens starting at dtr->tp with the contents of str.
 260  * tp ends up pointing just beyond the replacement.
 261  * Canonical whitespace is assured on each side.
 262  */
 263 void
 264     insertrow(Tokenrow * dtr, int ntok, Tokenrow * str)
 265 {
 266     int nrtok = rowlen(str);
 267
 268     dtr->tp += ntok;
 269     adjustrow(dtr, nrtok - ntok);
 270     dtr->tp -= ntok;
 271     movetokenrow(dtr, str);
 272     dtr->tp += nrtok;
 273 }
 274
 275 /*
 276  * make sure there is WS before trp->tp, if tokens might merge in the output
 277  */
 278 void
 279     makespace(Tokenrow * trp, Token * ntp)
 280 {
 281     uchar *tt;
 282     Token *tp = trp->tp;
 283
 284     if (tp >= trp->lp)
 285         return;
 286
 287     if (ntp->wslen)
 288     {
 289         tt = newstring(tp->t, tp->len, ntp->wslen);
 290         strncpy((char *)tt, (char *)ntp->t - ntp->wslen, ntp->wslen);
 291         tp->t = tt + ntp->wslen;
 292         tp->wslen = ntp->wslen;
 293         tp->flag |= XPWS;
 294     }
 295 }
 296
 297 /*
 298  * Copy an entire tokenrow into another, at tp.
 299  * It is assumed that there is enough space.
 300  *  Not strictly conforming.
 301  */
 302 void
 303     movetokenrow(Tokenrow * dtr, Tokenrow * str)
 304 {
 305     int nby;
 306
 307     /* nby = sizeof(Token) * (str->lp - str->bp); */
 308     nby = (char *) str->lp - (char *) str->bp;
 309     memmove(dtr->tp, str->bp, nby);
 310 }
 311
 312 /*
 313  * Move the tokens in a row, starting at tr->tp, rightward by nt tokens;
 314  * nt may be negative (left move).
 315  * The row may need to be grown.
 316  * Non-strictly conforming because of the (char *), but easily fixed
 317  */
 318 void
 319     adjustrow(Tokenrow * trp, int nt)
 320 {
 321     int nby, size;
 322
 323     if (nt == 0)
 324         return;
 325     size = (trp->lp - trp->bp) + nt;
 326     while (size > trp->max)
 327         growtokenrow(trp);
 328     /* nby = sizeof(Token) * (trp->lp - trp->tp); */
 329     nby = (char *) trp->lp - (char *) trp->tp;
 330     if (nby)
 331         memmove(trp->tp + nt, trp->tp, nby);
 332     trp->lp += nt;
 333 }
 334
 335 /*
 336  * Copy a row of tokens into the destination holder, allocating
 337  * the space for the contents.  Return the destination.
 338  */
 339 Tokenrow *
 340     copytokenrow(Tokenrow * dtr, Tokenrow * str)
 341 {
 342     int len = rowlen(str);
 343
 344     maketokenrow(len, dtr);
 345     movetokenrow(dtr, str);
 346     dtr->lp += len;
 347     return dtr;
 348 }
 349
 350 /*
 351  * Produce a copy of a row of tokens.  Start at trp->tp.
 352  * The value strings are copied as well.  The first token
 353  * has WS available.
 354  */
 355 Tokenrow *
 356     normtokenrow(Tokenrow * trp)
 357 {
 358     Token *tp;
 359     Tokenrow *ntrp = new(Tokenrow);
 360     int len;
 361
 362     len = trp->lp - trp->tp;
 363     if (len <= 0)
 364         len = 1;
 365     maketokenrow(len, ntrp);
 366     for (tp = trp->tp; tp < trp->lp; tp++)
 367     {
 368         *ntrp->lp = *tp;
 369         if (tp->len)
 370         {
 371             ntrp->lp->t = newstring(tp->t, tp->len, 1);
 372             *ntrp->lp->t++ = ' ';
 373             if (tp->wslen)
 374                 ntrp->lp->wslen = 1;
 375         }
 376         ntrp->lp++;
 377     }
 378     if (ntrp->lp > ntrp->bp)
 379         ntrp->bp->wslen = 0;
 380     return ntrp;
 381 }
 382
 383 /*
 384  * Debugging
 385  */
 386 void
 387     peektokens(Tokenrow * trp, char *str)
 388 {
 389     Token *tp;
 390
 391     tp = trp->tp;
 392     flushout();
 393     if (str)
 394         fprintf(stderr, "%s ", str);
 395     if (tp < trp->bp || tp > trp->lp)
 396         fprintf(stderr, "(tp offset %ld) ", (long)(tp - trp->bp));
 397     for (tp = trp->bp; tp < trp->lp && tp < trp->bp + 32; tp++)
 398     {
 399         if (tp->type != NL)
 400         {
 401             int c = tp->t[tp->len];
 402
 403             tp->t[tp->len] = 0;
 404             fprintf(stderr, "%s", tp->t);
 405             tp->t[tp->len] = (uchar) c;
 406         }
 407         fprintf(stderr, tp == trp->tp ? "{%x*} " : "{%x} ", tp->type);
 408     }
 409     fprintf(stderr, "\n");
 410     fflush(stderr);
 411 }
 412
 413 void
 414     puttokens(Tokenrow * trp)
 415 {
 416     Token *tp;
 417     int len;
 418     uchar *p;
 419
 420     if (Vflag)
 421         peektokens(trp, "");
 422     tp = trp->bp;
 423     for (; tp < trp->lp; tp++)
 424     {
 425         if (tp->type != NL)
 426         {
 427             len = tp->len + tp->wslen;
 428             p = tp->t - tp->wslen;
 429
 430             /* EBCDIC to ANSI conversion requested? */
 431             if( Aflag )
 432             {
 433                 /* keyword __ToLatin1__ found? -> do conversion! */
 434                 if( EBCDIC_StartTokenDetected )
 435                 {
 436                     /* previous token was 'extern'? -> don't convert current token! */
 437                     if( EBCDIC_ExternTokenDetected )
 438                     {
 439                         EBCDIC_ExternTokenDetected = 0;
 440                         memcpy(wbp, p, len);
 441                     }
 442                     else
 443                     {
 444                         /* current token is keyword 'extern'? -> don't convert following token! */
 445                         if( (tp->wslen == 0) && (strncmp( (char*)p, "extern", len ) == 0) )
 446                         {
 447                             EBCDIC_ExternTokenDetected = 1;
 448                             memcpy(wbp, p, len);
 449                         }
 450                         else
 451                         {
 452                             /* token is string or char? -> process EBCDIC to ANSI conversion */
 453                             if ((tp->type == STRING) || (tp->type == CCON))
 454                                 len = memcpy_EBCDIC(wbp,  p, len);
 455                             else
 456                                 memcpy(wbp, p, len);
 457                         }
 458                     }
 459                 }
 460                 else
 461                     /* keyword __ToLatin1__ found? -> don't copy keyword and start conversion */
 462                     if( (tp->type == NAME) && (strncmp( (char*)p, "__ToLatin1__", len) == 0) )
 463                     {
 464                         EBCDIC_StartTokenDetected = 1;
 465                         len = 0;
 466                     }
 467                     else
 468                         memcpy(wbp, p, len);
 469             }
 470             else
 471                 memcpy(wbp, p, len);
 472
 473             wbp += len;
 474         }
 475         else
 476             *wbp++ = '\n';
 477
 478         if (wbp >= &wbuf[OBS])
 479         {
 480             write(1, wbuf, OBS);
 481             if (wbp > &wbuf[OBS])
 482                 memcpy(wbuf, wbuf + OBS, wbp - &wbuf[OBS]);
 483             wbp -= OBS;
 484         }
 485     }
 486     trp->tp = tp;
 487     if (cursource->fd == 0)
 488         flushout();
 489 }
 490
 491 void
 492     flushout(void)
 493 {
 494     if (wbp > wbuf)
 495     {
 496         write(1, wbuf, wbp - wbuf);
 497         wbp = wbuf;
 498     }
 499 }
 500
 501 /*
 502  * turn a row into just a newline
 503  */
 504 void
 505     setempty(Tokenrow * trp)
 506 {
 507     trp->tp = trp->bp;
 508     trp->lp = trp->bp + 1;
 509     *trp->bp = nltoken;
 510 }
 511
 512 /*
 513  * generate a number
 514  */
 515 char *
 516     outnum(char *p, int n)
 517 {
 518     if (n >= 10)
 519         p = outnum(p, n / 10);
 520     *p++ = (char) (n % 10 + '0');
 521     return p;
 522 }
 523
 524 /*
 525  * allocate and initialize a new string from s, of length l, at offset o
 526  * Null terminated.
 527  */
 528 uchar *
 529     newstring(uchar * s, int l, int o)
 530 {
 531     uchar *ns = (uchar *) domalloc(l + o + 1);
 532
 533     ns[l + o] = '\0';
 534     return (uchar *) strncpy((char *) ns + o, (char *) s, l) - o;
 535 }