src/config/lxrgmr-code/tokenproc/code.c

   1
   2 /*
   3  * TBD:
   4  * @ use xxx_string() serial of functions, instead of buff opr. put caching buff
   5  *   pointer in state structure.
   6  * @ lookup first token in comment string for MACRO.
   7  *  + define MACRO process (#define, #if, #include, #pragma)
   8  * @ state-doc: c89-cmnt/ext-cmnt/dquoted-str/here-doc/help-doc/macro-code
   9  * @ put global env into a struct.
  10  *
  11  * @ xxx
  12  * # $'', $ flag, and using.
  13  * @ {}
  14  * # ${}, $ flag, envar state.
  15  * # foo () {}, func-flag, {} paired-token,
  16  * # {}, iTokenIdnum,
  17  *
  18  * XXX:
  19  * @ readuntil-doc: c99-cmnt/sh-cmnt/backquoted-str/quoted-str/func-code
  20  */
  21
  22 /*
  23  * bracket & quote & string.
  24  * @ //, #, comment prefix (readuntil).
  25  * # use caching buffer.
  26  * # MACRO state
  27  * @ '', ``, $'', $``, get string/code from pair-char to pair-char.
  28  * @ $, state & flag
  29  * @ "", $"", get string in string-state.
  30  * @ (), $(), <(), >(), foo(){} code-block.
  31  * @ $[], $(()), expr string.
  32  * @ $XXX, ${}, parsing env-var format in envar-state.
  33  */
  34
  35 #include "../StrBuff.h"
  36
  37 /* TBD: use CACHING_STR */
  38 char str_buff[8192];
  39 int str_idx = 0;
  40
  41 int blank_flag = 0;
  42 char blank_str[8192];
  43
  44 int recent_token = 0;
  45 int token_len = 0;
  46 char *token_str = NULL;
  47 TOKEN_ID_PROC pfn_token_proc = NULL;
  48
  49 #define BYTE_MASK(n)                            ((1<<((n)<<3)) - 1)
  50
  51
  52
  53
  54
  55 char mask_char = '\\';
  56 char pfx_char = 0;
  57 char pair_begin = 0;
  58 char pair_end = 0;
  59 char endstr[MAX_RSVWD_LEN];
  60 int endstr_len = 0;
  61
  62 extern void token_ungetc (char c, SRC_FILE_DESC *pfile);
  63
  64 char read_getc (SRC_FILE_DESC *pfile)
  65 {
  66         extern char token_getc (SRC_FILE_DESC *pfile);
  67
  68         register char c;
  69
  70         if ((pfile->iTokenStrLen - pfile->iPrevIdx) > 0)
  71         {
  72                 c = pfile->aTokenStrBuff[pfile->iPrevIdx];
  73                 pfile->iPrevIdx++;
  74         }
  75         else
  76                 c = token_getc(pfile);
  77
  78         return c;
  79 }
  80
  81 void read_ungetc (char c, SRC_FILE_DESC *pfile)
  82 {
  83         if (pfile->iPrevIdx > 0)
  84         {
  85                 pfile->iPrevIdx--;
  86                 pfile->aTokenStrBuff[pfile->iPrevIdx] = c;
  87         }
  88         else
  89                 token_ungetc(c, pfile);
  90 }
  91
  92 int read_token_str (SRC_FILE_DESC *pfile)
  93 {
  94         return 0;
  95 }
  96
  97 /*
  98  * \retval: 0, in normal.
  99  *                      -1, buffer is full.
 100  */
 101 int readuntil (SRC_FILE_DESC *pfile)
 102 {
 103         int count = 0;
 104         char c;
 105         int idx = 0;
 106
 107         /* cotinus string must be less then 10MByte. */
 108         while (idx < 0xa00000)
 109         {
 110                 c = read_getc(pfile);
 111                 idx++;
 112 //              printf("c=%c(%x)(%x)\n", c, c, endstr[0]);
 113                 // ignore NULL char in txt src file.
 114                 if (c == 0)
 115                         continue;
 116                 else if (c == mask_char)
 117                 {
 118                         c = read_getc(pfile);
 119                 }
 120                 else if (c == pfx_char)
 121                 {
 122                         //
 123                 }
 124                 else if (c == pair_begin)
 125                 {
 126                         count++;
 127                 }
 128                 else if (c == pair_end)
 129                 {
 130                         count--;
 131                 }
 132                 /*
 133                  * '',
 134                  * '\n', str='X'
 135                  * '\n', str='\n'
 136                  * 'EOF', str='aaa\n'
 137                  * 'EOF', str='EXX'
 138                  * 'EOF', str='EOL'
 139                  * 'EOF', str='EEOL'
 140                  * 'EOF', str='EOEOL'
 141                  */
 142                 else if (endstr[0] == 0)
 143                 {
 144                         read_ungetc(c, pfile);
 145                         if (count == 0) break;
 146                 }
 147                 else if (c == endstr[0])
 148                 {
 149                         int i = 1;
 150
 151                         while (i < endstr_len && (c=read_getc(pfile)) == endstr[i]) i++;
 152
 153                         if (i < endstr_len)
 154                         {
 155                                 read_ungetc(c, pfile);
 156                                 while (--i) read_ungetc(endstr[i], pfile);
 157                                 c = endstr[0];
 158 //                              memcpy(&str_buff[str_idx], endstr, i);
 159 //                              str_idx += i;
 160 //                              str_buff[str_idx] = 0;
 161                         }
 162                         else
 163                         {
 164                                 if (count == 0) break;
 165                         }
 166                 }
 167
 168                 //str_buff[str_idx] = c;
 169                 //str_idx++;
 170                 append_string(&strbuff, &c, 1);
 171 //              str_buff[str_idx] = 0;
 172         }
 173         //str_buff[str_idx] = 0;
 174 //      if (str_idx == sizeof(str_buff)-1)
 175 //              return -1;
 176 //      else
 177 //              return 0;
 178 }
 179
 180 /*
 181  * c99 comment string proc.
 182  */
 183 int on_c99_cmnt_proc (int evt, int token, register const char *str, register unsigned int len, SRC_FILE_DESC *psrcfile)
 184 {
 185         printf("################################################ c99 comment\n");
 186
 187         mask_char = 0;
 188         pfx_char = 0;
 189         pair_begin = 0;
 190         pair_end = 0;
 191         endstr[0] = '\n';
 192         endstr[1] = 0;
 193         endstr_len = 1;
 194
 195         /* TBD: use xxx_string() function to store txt. */
 196         /* ignore overlimited comment string. */
 197         //while (readuntil(gpcurr_src_file) == -1) str_idx = 0;
 198         readuntil(gpcurr_src_file);
 199
 200 //      printf("str_idx = %d\n", str_idx);
 201 //      if (str_idx != 0)
 202 //              printf(str_buff);
 203
 204         return T_C99_CMNT;
 205 }
 206
 207 /*
 208  * sh comment string proc.
 209  */
 210 int on_sh_cmnt_proc (int evt, int token, register const char *str, register unsigned int len, SRC_FILE_DESC *psrcfile)
 211 {
 212         int ret;
 213         char c;
 214
 215         int Macro_TokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState);
 216
 217         printf("################################################ sh comment\n");
 218
 219         str_idx = 1;
 220         str_buff[0] = '#';
 221         str_buff[1] = 0;
 222         while (1)
 223         {
 224                 c = read_getc(psrcfile);
 225                 str_buff[str_idx] = c;
 226                 str_idx++;
 227                 str_buff[str_idx] = 0;
 228
 229                 if (c == ' ' || c == '\t')
 230                         break;
 231                 if (c == '\n')
 232                         return T_SH_CMNT;
 233         }
 234         str_buff[str_idx] = 0;
 235
 236         ret = Macro_TokenProc(&str_buff[1], str_idx-1, gpcurr_src_file->pstCurrTokenState);
 237
 238         /*
 239          * no preproc cmd matching, it's a sh-cmnt.
 240          * otherwize, return token.
 241          */
 242         if (ret != 0)
 243                 return ret;
 244
 245         /* if it's not macro defination, append to T_WORD buffer. */
 246         append_string(&strbuff, str_buff, str_idx);
 247
 248         mask_char = 0;
 249         pfx_char = 0;
 250         pair_begin = 0;
 251         pair_end = 0;
 252         endstr[0] = '\n';
 253         endstr[1] = 0;
 254         endstr_len = 1;
 255
 256         /* ignore overlimited comment string. */
 257         //while (readuntil(gpcurr_src_file) == -1) str_idx = 0;
 258         readuntil(gpcurr_src_file);
 259
 260         return T_SH_CMNT;
 261 }
 262
 263
 264 char trans_char_tbl[128] = {
 265 /*      0       1       2               3       4               5       6               7       8               9       A       B       C       D       E               F */
 266         0,      0,      0,              0,      0,              0,      0,              0,      0,              0,      0,      0,      0,      0,      0,              0,      // 0
 267         0,      0,      0,              0,      0,              0,      0,              0,      0,              0,      0,      0,      0,      0,      0,              0,      // 1
 268 /*      ' '     !       "               #       $               %       &               '       (               )       *       +       ,       -       .               / */
 269         0,      0,      0,              0,      0,              0,      0,              0,      0,              0,      0,      0,      0,      0,      0,              0,      // 2
 270 /*      0       1       2               3       4               5       6               7       8               9       :       ;       <       =       >               ? */
 271         -16,-17,-18,    -19,-20,        -21,-22,        -23,-24,        -25,0,  0,      0,      0,      0,              0,      // 3
 272 /*      @       A       B               C       D               E       F               G       H               I       J       K       L       M       N               O */
 273         0,      -26,-27,        -28,-29,        -30,-31,        0,      0,              0,      0,      0,      0,      0,      0,              0,      // 4
 274 /*      P       Q       R               S       T               U       V               W       X               Y       Z       [       \       ]       ^               _ */
 275         0,      0,      0,              0,      0,              0,      0,              0,      0,              0,      0,      0,      1,      0,      0,              0,      // 5
 276 /*      `       a       b               c       d               e       f               g       h               i       j       k       l       m       n               o */
 277         0,      -26,-27,        -28,-29,        -30,'\f',       0,      0,              0,      0,      0,      0,      0,      '\n',   0,      // 6
 278 /*      p       q       r               s       t               u       v               w       x               y       z       {       |       }       ~                 */
 279         0,      0,      '\r',   0,      '\t',   0,      '\v',   0,      -127,   0,      0,      0,      0,      0,      0,              0,      // 7
 280 };
 281
 282 int str2hex (char *str, int iidx, int oidx)
 283 {
 284         int i;
 285         unsigned long long val = 0;
 286         int chars = 0;
 287         char *pbuff = &str[iidx];
 288         static char tmp[32];
 289         register char c;
 290
 291         /*
 292          * str2int
 293          * max hex string len is 32, it's a 128bit hex value string.
 294          */
 295         for (i=0; i<32; i++)
 296         {
 297                 c = trans_char_tbl[pbuff[i]>>1];
 298                 if (c < 0 || c == '\f')
 299                 {
 300                         tmp[i] = ((c<-15) ? (c+16) : 0x0f);
 301                 }
 302                 else
 303                         break;
 304         }
 305
 306         /* calc */
 307         chars = i;
 308         for (; i>=0; i--) val += tmp[i]<<(chars - i);
 309
 310         /* store value */
 311         pbuff = &str[oidx];
 312         for (i=0; i<chars>>2; i++)
 313         {
 314                 pbuff[i] = (char)(val>>i);
 315         }
 316
 317         return chars;
 318 }
 319
 320 int translate_char (char *str, int len)
 321 {
 322         int i, j;
 323         char c;
 324
 325         /*
 326          * "a"
 327          * "\a"
 328          * "\n"
 329          * "\xaa"
 330          */
 331         for (i=0; i<len; i++)
 332         {
 333                 c = str[i];
 334                 if (c == '\\')
 335                 {
 336                         i++;
 337                         {
 338                                 c = trans_char_tbl[c&0x7f];
 339                                 str[j] = (c > 0) ? c : str[i];
 340                                 if (c == -127)
 341                                 {
 342                                         i++;
 343                                         c = str2hex(str, i, j);
 344                                         j += (int)((c+1)>>1);
 345                                         i += (int)c;
 346                                         continue;
 347                                 }
 348                                 c = str[j];
 349                         }
 350                 }
 351                 str[j] = c;
 352                 j++;
 353         }
 354         str[j] = 0;
 355         return j;
 356 }
 357
 358
 359 /*
 360  * quote string proc.
 361  */
 362 int on_quote (int evt, int token, register const char *str, register unsigned int len, SRC_FILE_DESC *psrcfile)
 363 {
 364         printf("################################################ single quote\n");
 365
 366         mask_char = 0;
 367         pfx_char = 0;
 368         pair_begin = 0;
 369         pair_end = 0;
 370         endstr[0] = '\'';
 371         endstr[1] = 0;
 372         endstr_len = 1;
 373
 374         /* ignore overlimited comment string. */
 375         //while (readuntil(gpcurr_src_file) == -1) str_idx = 0;
 376         readuntil(gpcurr_src_file);
 377
 378         /* TBD: with $ flag, translate \X to corresponding char. */
 379         if (CHK_STATE_FLAG(psrcfile->pstCurrTokenState, TF_DOLLAR))
 380         {
 381                 //CLR_STATE_FLAG(psrcfile->pstCurrTokenState, TF_DOLLAR);
 382                 strbuff.text_size = translate_char(strbuff.text, strbuff.text_size);
 383                 strbuff.text[strbuff.text_size] = 0;
 384                 STATE_RECOVERY(psrcfile);
 385         }
 386
 387         return T_WORD_QUOTE;
 388 }
 389
 390 /*
 391  * quote string proc.
 392  */
 393 int on_bquote (int evt, int token, register const char *str, register unsigned int len, SRC_FILE_DESC *psrcfile)
 394 {
 395         printf("################################################ back quote\n");
 396         mask_char = '\\';
 397         pfx_char = 0;
 398         pair_begin = '"';
 399         pair_end = '"';
 400         endstr[0] = '`';
 401         endstr[1] = 0;
 402         endstr_len = 1;
 403
 404         /* ignore overlimited comment string. */
 405         //while (readuntil(gpcurr_src_file) == -1) str_idx = 0;
 406         readuntil(gpcurr_src_file);
 407
 408         return T_WORD_BACK_QUOTE;
 409 }
 410
 411 int on_ext_cmnt (int evt, int token, register const char *str, register unsigned int len, SRC_FILE_DESC *psrcfile)
 412 {
 413         printf("################################################ ext comment\n");
 414
 415         if (evt == EVT_ON_ENTER)
 416         {
 417                 //
 418                 return T_SKIP;
 419         }
 420         else if (evt == EVT_ON_EXIT)
 421         {
 422                 // do cmnt proc and return token.
 423                 return T_EXT_CMNT;
 424         }
 425 }
 426
 427 int on_c89_cmnt (int evt, int token, register const char *str, register unsigned int len, SRC_FILE_DESC *psrcfile)
 428 {
 429         printf("a ################################################ c89 comment\n");
 430
 431         if (evt == EVT_ON_ENTER)
 432         {
 433                 printf("enter\n");
 434                 //strcpy(str_buff, "/*");
 435                 //str_idx += 2;
 436                 append_string(&strbuff, "/*", 2);
 437                 return T_SKIP;
 438         }
 439         else if (evt == EVT_ON_EXIT)
 440         {
 441                 printf("exit\n");
 442                 // do cmnt proc and return token.
 443                 //strcat(str_buff, "*/");
 444                 //str_idx += 2;
 445                 append_string(&strbuff, "*/", 2);
 446
 447 //              printf("str_idx=%d; str_buff='%s'\n", str_idx, str_buff);
 448                 return T_C89_CMNT;
 449         }
 450
 451         ////////////////////////////////////
 452         mask_char = 0;
 453         pfx_char = 0;
 454         pair_begin = 0;
 455         pair_end = 0;
 456         endstr[0] = '*';
 457         endstr[1] = '/';
 458         endstr[2] = 0;
 459         endstr_len = 2;
 460
 461         //str_buff[0] = '/';
 462         //str_buff[1] = '*';
 463         //str_idx = 2;
 464         append_string(&strbuff, "/*", 2);
 465
 466         /* TBD: use xxx_string() function to store txt. */
 467         /* ignore overlimited comment string. */
 468         //while (readuntil(gpcurr_src_file) == -1) str_idx = 0;
 469         readuntil(gpcurr_src_file);
 470
 471         //strcat(str_buff, "*/");
 472         append_string(&strbuff, "*/", 2);
 473 //      printf("str_idx = %d\n", str_idx);
 474 //      if (str_idx != 0)
 475 //              printf(str_buff);
 476
 477         return T_C89_CMNT;
 478 }
 479
 480 int on_dquote (int evt, int token, register const char *str, register unsigned int len, SRC_FILE_DESC *psrcfile)
 481 {
 482         printf("################################################ dual quote\n");
 483         return 0;
 484 }
 485
 486
 487
 488
 489
 490 /*
 491  * ():
 492  * code-block, sub-script. stdout is main-script's stdout.
 493  * $():
 494  * code-block, sub-script. stdout is cmd-line buffer.
 495  */
 496 int on_cb_bracket1 (int evt, int token, register const char *str, register unsigned int len, SRC_FILE_DESC *psrcfile)
 497 {
 498         printf("################################################ bracket1 ()\n");
 499
 500         /* subscript */
 501
 502         /* func param list */
 503
 504         return 0;
 505 }
 506
 507 int on_bracket2 (int evt, int token, register const char *str, register unsigned int len, SRC_FILE_DESC *psrcfile)
 508 {
 509         printf("################################################ bracket2 (())\n");
 510
 511         if (evt == EVT_ON_ENTER)
 512         {
 513                 if (CHK_STATE_FLAG(psrcfile->pstCurrTokenState, TF_DOLLAR))
 514                 {
 515                         STATE_RECOVERY(psrcfile);
 516                         STATE_SWITCH(psrcfile, STATE_EXPR);
 517                         return 0;
 518                 }
 519         }
 520         else if (evt == EVT_ON_EXIT)
 521         {
 522                 if (CHK_STATE_FLAG(psrcfile->pstCurrTokenState, TF_DOLLAR))
 523                 {
 524                         STATE_RECOVERY(psrcfile);
 525                         return 0;
 526                 }
 527         }
 528
 529         return 0;
 530 }
 531
 532 int on_dbracket1 (int evt, int token, register const char *str, register unsigned int len, SRC_FILE_DESC *psrcfile)
 533 {
 534         printf("################################################ dbracket1 [[]]\n");
 535
 536         if (evt == EVT_ON_ENTER)
 537         {
 538                 if (CHK_STATE_FLAG(psrcfile->pstCurrTokenState, TF_DOLLAR))
 539                 {
 540                         STATE_RECOVERY(psrcfile);
 541                         STATE_SWITCH(psrcfile, STATE_EXPR);
 542                         return 0;
 543                 }
 544         }
 545         else if (evt == EVT_ON_EXIT)
 546         {
 547                 if (STATE_CHK(psrcfile, STATE_EXPR))
 548                 {
 549                         STATE_RECOVERY(psrcfile);
 550                         return 0;
 551                 }
 552         }
 553
 554         return 0;
 555 }
 556
 557
 558
 559
 560
 561
 562
 563
 564
 565 char *state_token_str = NULL;
 566 TOKEN_ID_PROC pfn_pair_token_proc = NULL;
 567 int token_state = 0;
 568
 569 /*
 570  * TBD: improve
 571  * @ use three func to process word/pair/other token.
 572  */
 573 int id_str_matching (char *str, int str_len, const struct token_id *pstTokenIdTbl, short *ptoken_len_tbl)
 574 {
 575         int i, k, h;
 576         int len = str_len;
 577         char *pstr;
 578         short *plentbl;
 579         int str_idx_bak = str_idx;
 580
 581         if (!pstTokenIdTbl) return 0;
 582
 583 //      printf("%s: %s(%s)(%d)\n", __FILE__, __FUNCTION__, str, str_len);
 584 //      printf("pstTokenIdTbl = 0x%08x\n", pstTokenIdTbl);
 585 //      printf("ptoken_len_tbl = 0x%08x\n", ptoken_len_tbl);
 586 //      printf("len = 0x%08x\n", len);
 587
 588         for (k=0; len>0; k++)
 589         {
 590                 plentbl = ptoken_len_tbl;
 591 //              printf("str=%s(%d)\n", str, k);
 592                 for (i=0; i<MAX_TOKEN_ID_TBL_SIZE; i++)
 593                 {
 594 //                      printf("XXXXXXXXXXXXXXXXXXXXXXXXXX\n");
 595                         if (pstTokenIdTbl[i].name == NULL &&
 596                                 pstTokenIdTbl[i].tokenstr == NULL &&
 597                                 pstTokenIdTbl[i].token == 0 &&
 598                                 pstTokenIdTbl[i].flags == 0 &&
 599                                 pstTokenIdTbl[i].proc == NULL)
 600                                 break;
 601 //                      printf("pstTokenIdTbl[%d].name = %s(0x%08x)(%s)\n", i, "token", pstTokenIdTbl[i].name, pstTokenIdTbl[i].tokenstr);
 602                         register int j;
 603                         register int token_id_len = *plentbl;
 604                         plentbl++;
 605
 606                         if (len < token_id_len) continue;
 607
 608                         /* TBD: normally, this addr area 'chars[3]' should not be used for user accessing. */
 609                         pstr = (pstTokenIdTbl[i].chars[3]) ? pstTokenIdTbl[i].name : (char *)&pstTokenIdTbl[i].name;
 610 //                      printf("@%d.[%s]: %s(%d)(%d)\n", i, pstTokenIdTbl[i].tokenstr, pstr, token_id_len, pstTokenIdTbl[i].token);
 611
 612                         for (j=0; j<token_id_len; j++)
 613                         {
 614                                 if (pstr[j] != str[j])
 615                                         break;
 616                         }
 617                         if (j == token_id_len)
 618                         {
 619 //                              printf("@%d.[%s]: %s(%d)(%d)[%s]\n", i, pstTokenIdTbl[i].tokenstr, pstr, token_id_len, pstTokenIdTbl[i].token, str);
 620
 621                                 if (pstr[token_id_len] != 0)
 622                                 {
 623                                         state_token_str = &pstr[token_id_len];
 624                                         pfn_pair_token_proc = pstTokenIdTbl[i].proc;
 625                                 }
 626
 627                                 pfn_token_proc = (TOKEN_ID_PROC)pstTokenIdTbl[i].proc;
 628                                 token_len = token_id_len;
 629                                 token_str = pstr;
 630                                 token_state = pstTokenIdTbl[i].state;
 631                                 if (str_idx != 0)
 632                                 {
 633                                         recent_token = pstTokenIdTbl[i].token;
 634                                         return T_WORD;
 635                                 }
 636
 637                                 return pstTokenIdTbl[i].token;
 638                         }
 639                         else if (pstr[token_id_len])
 640                         {
 641                                 for (h=0,j=token_id_len; j<(token_id_len<<1); h++,j++)
 642                                 {
 643                                         if (pstr[j] != str[h])
 644                                                 break;
 645                                 }
 646                                 if (j == (token_id_len<<1))
 647                                 {
 648 //                                      printf("&%d.[%s]: %s(%d)(%d)\n", i, pstTokenIdTbl[i].tokenstr, &pstr[token_id_len], token_id_len, pstTokenIdTbl[i].token+1);
 649
 650                                         pfn_token_proc = (TOKEN_ID_PROC)pstTokenIdTbl[i].proc;
 651                                         token_len = token_id_len;
 652                                         token_str = &pstr[token_id_len];
 653                                         token_state = pstTokenIdTbl[i].state;
 654                                         if (str_idx != 0)
 655                                         {
 656                                                 recent_token = pstTokenIdTbl[i].token+1;
 657                                                 return T_WORD;
 658                                         }
 659
 660                                         return pstTokenIdTbl[i].token+1;
 661                                 }
 662                         }
 663                 }
 664                 str_buff[str_idx] = *str;
 665                 str_idx++;
 666                 if (str_idx >= 8192)
 667                         fprintf(stderr, "err: word string buffer is over limited.\n");
 668                 str++;
 669                 len--;
 670         }
 671
 672         str_idx = str_idx_bak;
 673         return 0;
 674 }
 675
 676 /*
 677  * TBD: improve
 678  * @ only matching whole word.
 679  */
 680 int rsvwd_str_matching (char *str, int str_len, const struct token_id *pstTokenIdTbl, short *ptoken_len_tbl)
 681 {
 682         int i, k, h;
 683         int len = str_len;
 684         char *pstr;
 685         short *plentbl;
 686         int str_idx_bak = str_idx;
 687
 688         if (!pstTokenIdTbl) return 0;
 689
 690 //      printf("%s: %s(%s)(%d)\n", __FILE__, __FUNCTION__, str, str_len);
 691 //      printf("pstTokenIdTbl = 0x%08x\n", pstTokenIdTbl);
 692 //      printf("ptoken_len_tbl = 0x%08x\n", ptoken_len_tbl);
 693 //      printf("len = 0x%08x\n", len);
 694
 695         for (k=0; len>0; k++)
 696         {
 697                 plentbl = ptoken_len_tbl;
 698 //              printf("str=%s(%d)\n", str, k);
 699                 for (i=0; i<MAX_TOKEN_ID_TBL_SIZE; i++)
 700                 {
 701 //                      printf("XXXXXXXXXXXXXXXXXXXXXXXXXX\n");
 702                         if (pstTokenIdTbl[i].name == NULL &&
 703                                 pstTokenIdTbl[i].tokenstr == NULL &&
 704                                 pstTokenIdTbl[i].token == 0 &&
 705                                 pstTokenIdTbl[i].flags == 0 &&
 706                                 pstTokenIdTbl[i].proc == NULL)
 707                                 break;
 708 //                      printf("pstTokenIdTbl[%d].name = %s(0x%08x)(%s)\n", i, "token", pstTokenIdTbl[i].name, pstTokenIdTbl[i].tokenstr);
 709                         register int j;
 710                         register int token_id_len = *plentbl;
 711                         plentbl++;
 712
 713                         if (len < token_id_len) continue;
 714
 715                         /* TBD: normally, this addr area 'chars[3]' should not be used for user accessing. */
 716                         pstr = (pstTokenIdTbl[i].chars[3]) ? pstTokenIdTbl[i].name : (char *)&pstTokenIdTbl[i].name;
 717 //                      printf("@%d.[%s]: %s(%d)(%d)\n", i, pstTokenIdTbl[i].tokenstr, pstr, token_id_len, pstTokenIdTbl[i].token);
 718
 719                         for (j=0; j<token_id_len; j++)
 720                         {
 721                                 if (pstr[j] != str[j])
 722                                         break;
 723                         }
 724                         if (j == token_id_len)
 725                         {
 726 //                              printf("@%d.[%s]: %s(%d)(%d)[%s]\n", i, pstTokenIdTbl[i].tokenstr, pstr, token_id_len, pstTokenIdTbl[i].token, str);
 727
 728                                 if (pstr[token_id_len] != 0)
 729                                 {
 730                                         state_token_str = &pstr[token_id_len];
 731                                         pfn_pair_token_proc = pstTokenIdTbl[i].proc;
 732                                 }
 733
 734                                 pfn_token_proc = (TOKEN_ID_PROC)pstTokenIdTbl[i].proc;
 735                                 token_len = token_id_len;
 736                                 token_str = pstr;
 737                                 token_state = pstTokenIdTbl[i].state;
 738                                 if (str_idx != 0)
 739                                 {
 740                                         recent_token = pstTokenIdTbl[i].token;
 741                                         return T_WORD;
 742                                 }
 743
 744                                 return pstTokenIdTbl[i].token;
 745                         }
 746                         else if (pstr[token_id_len])
 747                         {
 748                                 for (h=0,j=token_id_len; j<(token_id_len<<1); h++,j++)
 749                                 {
 750                                         if (pstr[j] != str[h])
 751                                                 break;
 752                                 }
 753                                 if (j == (token_id_len<<1))
 754                                 {
 755 //                                      printf("&%d.[%s]: %s(%d)(%d)\n", i, pstTokenIdTbl[i].tokenstr, &pstr[token_id_len], token_id_len, pstTokenIdTbl[i].token+1);
 756
 757                                         pfn_token_proc = (TOKEN_ID_PROC)pstTokenIdTbl[i].proc;
 758                                         token_len = token_id_len;
 759                                         token_str = &pstr[token_id_len];
 760                                         token_state = pstTokenIdTbl[i].state;
 761                                         if (str_idx != 0)
 762                                         {
 763                                                 recent_token = pstTokenIdTbl[i].token+1;
 764                                                 return T_WORD;
 765                                         }
 766
 767                                         return pstTokenIdTbl[i].token+1;
 768                                 }
 769                         }
 770                 }
 771                 str_buff[str_idx] = *str;
 772                 str_idx++;
 773                 if (str_idx >= 8192)
 774                         fprintf(stderr, "err: word string buffer is over limited.\n");
 775                 str++;
 776                 len--;
 777         }
 778
 779         str_idx = str_idx_bak;
 780         return 0;
 781 }
 782
 783 int mask_flag = 0;
 784
 785 /*
 786  * general proc for text/string. it store string into text buffer.
 787  * it can be used in:
 788  * @ comment string.
 789  * @ general string.
 790  * @ here-doc.
 791  */
 792 int Txt_TokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState)
 793 {
 794         if (blank_flag == 1)
 795         {
 796                 //strcat(str_buff, blank_str);
 797                 //str_idx += strlen(blank_str);
 798                 append_string(&strbuff, blank_str, -1);
 799         }
 800
 801         //strcat(str_buff, str);
 802         append_string(&strbuff, str, -1);
 803         pstTokenState->token_len = strlen(str);
 804         //str_idx += pstTokenState->token_len;
 805
 806 //      printf("Txt_TokenProc(%s) (%s) %d, %d\n", str_buff, str, pstTokenState->token_len, str_idx);
 807         return T_SKIP;
 808 }
 809
 810 int String_PunctTokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState)
 811 {
 812         return 0;
 813 }
 814
 815
 816 /*
 817  * general proc for text/string. it store string into text buffer.
 818  * it can be used in:
 819  * @ comment string.
 820  * @ general string.
 821  * @ here-doc.
 822  */
 823 int Macro_TokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState)
 824 {
 825         int i;
 826         int ret;
 827         int str_idx_bak;
 828         char MacroCMDBuff[16];
 829
 830 //      printf("Macro_TokenProc(%s)\n", str);
 831
 832         /* matching first following word for marco cmd */
 833         for (i=0; i< strbuff.text_size && i<sizeof(MacroCMDBuff); i++)
 834         {
 835                 if (strbuff.text[i] == ' ') break;
 836                 MacroCMDBuff[i] = strbuff.text[i];
 837         }
 838         MacroCMDBuff[i] = 0;
 839
 840 //      printf("%s(%d)\n", MacroCMDBuff, i);
 841
 842         str_idx_bak = str_idx;
 843         str_idx = 0;
 844
 845         ret = rsvwd_str_matching (
 846                         MacroCMDBuff,
 847                         i,
 848                         macro_array,
 849                         au16MacroTokenLen);
 850
 851         str_idx = str_idx_bak;
 852
 853         return ret;
 854 }
 855
 856 /*
 857  * set mask flag/state, the following char will be treated as a WORD char.
 858  */
 859 int Def_MaskTokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState)
 860 {
 861         printf("OnMaskTokenProc(%s)\n", str);
 862
 863         mask_flag = 1;
 864         return T_SKIP;
 865 }
 866
 867 /*
 868  * lookup reserved-word table, maching correspoding token.
 869  * or return as a normal T_WORD.
 870  */
 871 int Def_WordTokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState)
 872 {
 873         int len = strlen(str);
 874         int ret = 0;
 875
 876         printf("OnWordTokenProc(%s)\n", str);
 877
 878         /* xxx: macro define replacing. */
 879
 880         pfn_token_proc = NULL;
 881
 882         if (mask_flag == 1)
 883         {
 884                 mask_flag = 0;
 885                 //str_buff[str_idx] = *str;
 886                 //str_idx++;
 887                 //str_buff[str_idx] = 0;
 888                 append_string(&strbuff, str, 1);
 889                 str++;
 890         }
 891
 892         //if (str_idx == 0)
 893         if (is_empty_string(&strbuff))
 894         {
 895                 /* T_WORD buffer is empty, do token matching work. */
 896                 ret = rsvwd_str_matching (
 897                                 str,
 898                                 len,
 899                                 pstTokenState->pstStateInfo->astTokenTypeTbl[TYPE_WORD].pstTokenTbl,
 900                                 pstTokenState->pstStateInfo->api16TokenNameLen[TYPE_WORD]);
 901
 902                 pstTokenState->pfnTokenProc = pfn_token_proc;
 903 //              pstTokenState->recent_token = recent_token;
 904 //              recent_token = 0;
 905                 pstTokenState->token_len = token_len;
 906                 token_len = 0;
 907                 pstTokenState->token_str = token_str;
 908
 909                 /* return valid token */
 910                 if (ret != 0) return ret;
 911         }
 912
 913         /* otherwize, append it to T_WORD buffer. */
 914         //strcpy(str_buff, str);
 915         //str_idx = strlen(str);
 916         append_string(&strbuff, str, -1);
 917         return T_WORD;
 918 }
 919
 920 #define UINT_MAX                        (-1 & (~0x80000000))
 921
 922 int state_token = 0;
 923
 924 /*
 925  * there are three kinds of punct string.
 926  * @ prefix-char, this char/str let the program entering a new state.
 927  * @ pair-char, also can trig new state.
 928  * @ operation-char, for grammar element token.
 929  * @ normal-char, T_WORD.
 930  * in this function, it recognize and split string, to get one token by looking
 931  * up pfx-char/pair-char/opr-char table.
 932  */
 933 int Def_PunctTokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState)
 934 {
 935         int len=strlen(str);
 936         int ret = 0;
 937         int opr, pfx, pair;
 938         int opr_token, pfx_token, pair_token;
 939         int opr_idx, pfx_idx, pair_idx;
 940         int opr_len, pfx_len, pair_len;
 941         TOKEN_ID_PROC opr_proc = NULL, pfx_proc = NULL, pair_proc = NULL;
 942         unsigned int tmp;
 943         char *tmp_token_str[3];
 944
 945         printf("OnPunctTokenProc(%s)\n", str);
 946
 947         pfn_token_proc = NULL;
 948         recent_token = 0;
 949         token_len = 0;
 950
 951         if (mask_flag == 1)
 952         {
 953                 mask_flag = 0;
 954                 //str_buff[str_idx] = *str;
 955                 //str_idx++;
 956                 //str_buff[str_idx] = 0;
 957                 append_string(&strbuff, str, 1);
 958                 str++;
 959         }
 960
 961         pfx = id_str_matching (
 962                         str,
 963                         len,
 964                         pstTokenState->pstStateInfo->astTokenTypeTbl[TYPE_PFX].pstTokenTbl,
 965                         pstTokenState->pstStateInfo->api16TokenNameLen[TYPE_PFX]);
 966         pfx_token = recent_token;
 967         pfx_len = token_len;
 968         pfx_idx = str_idx;
 969         recent_token = 0;
 970         str_idx = 0;
 971         pfx_proc = pfn_token_proc;
 972         if (pfx != 0)
 973                 tmp_token_str[0] = token_str;
 974 //      printf("pfx = %d\n", pfx);
 975 //      printf("str_idx = %d\n", str_idx);
 976 //      printf("pfx_len = %d\n", pfx_len);
 977
 978         /* TBD: backup */
 979         pair = id_str_matching (
 980                         str,
 981                         len,
 982                         pstTokenState->pstStateInfo->astTokenTypeTbl[TYPE_PAIR].pstTokenTbl,
 983                         pstTokenState->pstStateInfo->api16TokenNameLen[TYPE_PAIR]);
 984         pair_token = recent_token;
 985         pair_len = token_len;
 986         pair_idx = str_idx;
 987         recent_token = 0;
 988         str_idx = 0;
 989         pair_proc = pfn_token_proc;
 990         if (pair != 0)
 991                 tmp_token_str[1] = token_str;
 992 //      printf("pair = %d\n", pair);
 993 //      printf("str_idx = %d\n", str_idx);
 994 //      printf("pair_len = %d\n", pair_len);
 995
 996         opr = id_str_matching (
 997                         str,
 998                         len,
 999                         pstTokenState->pstStateInfo->astTokenTypeTbl[TYPE_PUNCT].pstTokenTbl,
1000                         pstTokenState->pstStateInfo->api16TokenNameLen[TYPE_PUNCT]);
1001         opr_token = recent_token;
1002         opr_len = token_len;
1003         opr_idx = str_idx;
1004         recent_token = 0;
1005         str_idx = 0;
1006         opr_proc = pfn_token_proc;
1007         if (opr != 0)
1008                 tmp_token_str[2] = token_str;
1009 //      printf("opr = %d\n", opr);
1010 //      printf("str_idx = %d\n", str_idx);
1011 //      printf("opr_len = %d\n", opr_len);
1012
1013         tmp = -1;
1014         if (pfx == 0)
1015         {
1016                 pfx_idx = UINT_MAX;
1017                 pfx_len = 0;
1018         }
1019         if (pair == 0)
1020         {
1021                 pair_idx = UINT_MAX;
1022                 pair_len = 0;
1023         }
1024         if (opr == 0)
1025         {
1026                 opr_idx = UINT_MAX;
1027                 opr_len = 0;
1028         }
1029
1030 //      printf("pfx = %d,%d,%d\n", pfx, pfx_idx, pfx_len);
1031 //      printf("pair = %d,%d,%d\n", pair, pair_idx, pair_len);
1032 //      printf("opr = %d,%d,%d\n", opr, opr_idx, opr_len);
1033         if (pfx != 0 &&
1034                 pfx_idx <= pair_idx && pfx_idx <= opr_idx &&
1035                 pfx_len >= pair_len && pfx_len >= opr_len)
1036         {
1037 //              printf("xxxxxxxxxxxxxxxx1(%d)\n", pfx);
1038                 recent_token = pfx_token;
1039                 token_len = pfx_len;
1040                 str_idx = pfx_idx;
1041                 pfn_token_proc = pfx_proc;
1042                 pstTokenState->token_str = tmp_token_str[0];
1043                 ret = pfx;
1044         }
1045         if (pair != 0 &&
1046                 pair_idx <= pfx_idx && pair_idx <= opr_idx &&
1047                 pair_len >= pfx_len && pair_len >= opr_len)
1048         {
1049                 /* entering paired token, and need to be switched to new state. */
1050                 if (token_state != 0)
1051                 {
1052                         if (state_token_str)
1053                         {
1054                                 pstTokenState->pfnPairTokenProc = pfn_pair_token_proc;
1055                                 pstTokenState->state_token_str = state_token_str;
1056                                 pstTokenState->paired_token = pair+1;
1057                         }
1058
1059                         pstTokenState->paired_state = token_state;
1060                 }
1061 //              printf("xxxxxxxxxxxxxxxx2 (%s) pstTokenState->paired_token (%d)\n", pstTokenState->state_token_str, pstTokenState->paired_token);
1062                 recent_token = pair_token;
1063                 token_len = pair_len;
1064                 str_idx = pair_idx;
1065                 pfn_token_proc = pair_proc;
1066                 pstTokenState->token_str = tmp_token_str[1];
1067                 ret = pair;
1068         }
1069         if (opr != 0 &&
1070                 opr_idx <= pfx_idx && opr_idx <= pair_idx &&
1071                 opr_len >= pfx_len && opr_len >= pair_len)
1072         {
1073 //              printf("xxxxxxxxxxxxxxxx3(%d)\n", opr);
1074                 recent_token = opr_token;
1075                 token_len = opr_len;
1076                 str_idx = opr_idx;
1077                 pfn_token_proc = opr_proc;
1078                 pstTokenState->token_str = tmp_token_str[2];
1079                 ret = opr;
1080         }
1081 //      printf("matching ret = %d\n", ret);
1082
1083         pstTokenState->pfnTokenProc = pfn_token_proc;
1084         pstTokenState->recent_token = recent_token;
1085         recent_token = 0;
1086         pstTokenState->token_len = token_len;
1087         token_len = 0;
1088         state_token_str = 0;
1089         token_state = 0;
1090
1091         if (ret == pfx && ret != 0)
1092         {
1093                 state_token = ret;
1094 //              ret = T_SKIP;
1095         }
1096
1097         /* no token matching, append it to T_WORD buffer. */
1098         if (!ret)
1099         {
1100                 //strcpy(str_buff, str);
1101                 //str_idx = strlen(str);
1102                 append_string(&strbuff, str, -1);
1103
1104 //              printf("return T_WORD(%s)\n", str);
1105                 return T_WORD;
1106         }
1107
1108         /* token matched, append string before token to T_WORD buffer */
1109         if (str_idx != 0)
1110         {
1111                 //memcpy(str_buff, str, str_idx);
1112                 //str_buff[str_idx] = 0;
1113                 append_string(&strbuff, str, str_idx);
1114                 /*
1115                  * append T_WORD part string len for token_len,
1116                  * it's used for readed token string length.
1117                  */
1118                 pstTokenState->token_len += str_idx;
1119                 str_idx = 0;
1120                 pstTokenState->recent_token = ret;
1121                 ret = T_WORD;
1122         }
1123
1124 //      printf("return ret = %d\n", ret);
1125
1126         return ret;
1127 }
1128
1129 int Def_SpaceTokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState)
1130 {
1131         printf("OnSpaceTokenProc(%s)\n", str);
1132
1133         if (mask_flag == 1)
1134         {
1135                 mask_flag = 0;
1136                 //str_buff[str_idx] = *str;
1137                 //str_idx++;
1138                 //str_buff[str_idx] = 0;
1139                 append_string(&strbuff, str, 1);
1140                 str++;
1141         }
1142
1143         if (str[0])
1144         {
1145                 blank_flag = 1;
1146                 strcpy(blank_str, str);
1147                 //str_idx = strlen(str);
1148         }
1149         pstTokenState->token_len = strlen(str) + 1;
1150
1151         return T_SKIP;
1152 }
1153
1154 int Def_NewlineTokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState)
1155 {
1156         printf("OnNewlineTokenProc(%s)\n", str);
1157
1158         if (mask_flag == 1)
1159         {
1160                 mask_flag = 0;
1161                 //str_buff[str_idx] = *str;
1162                 //str_idx++;
1163                 //str_buff[str_idx] = 0;
1164                 append_string(&strbuff, str, 1);
1165                 str++;
1166                 if (str[0] == 0)
1167                         return T_SKIP;
1168         }
1169
1170         pstTokenState->token_len = 1;
1171         pstTokenState->nl_cnt = strlen(str) - 1;
1172 //      printf("NL(%d)\n", pstTokenState->nl_cnt);
1173
1174         /* in cmd state, new line reset token id num */
1175         // XXX:
1176         /*
1177         if (STATE_VCHK(pstTokenState, STATE_CMD))
1178                 pstTokenState->iTokenIdNum = 0;
1179         */
1180
1181         return T_EOL;
1182 }
1183
1184 int Def_EOFTokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState)
1185 {
1186         printf("OnEOFTokenProc(%s)\n", str);
1187
1188         return 0;
1189 }
1190
1191 /*
1192  *
1193  */
1194 int Envar_WordTokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState)
1195 {
1196         int ret = Def_WordTokenProc(str, type, pstTokenState);
1197
1198         /* store to prev state. */
1199         /*
1200         if (STATE_VCHK(pstTokenState, STATE_DOLLAR))
1201         {
1202                 STATE_RECOVERY(gpcurr_src_file);
1203         }
1204         */
1205
1206         /* envar expanssion */
1207         if (0)
1208         {
1209                 /* ungetc() to buffer */
1210                 return T_SKIP;
1211         }
1212         else
1213         {
1214                 return T_WORD;
1215         }
1216 }
1217
1218 /*
1219  *
1220  */
1221 int Envar_PunctTokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState)
1222 {
1223         int ret = Def_PunctTokenProc(str, type, pstTokenState);
1224
1225         /* store to prev state. */
1226         /*
1227         if (STATE_VCHK(pstTokenState, STATE_DOLLAR))
1228         {
1229                 STATE_RECOVERY(gpcurr_src_file);
1230         }
1231         */
1232
1233         /* $( */
1234         if (ret == T_LBRACKET1)
1235         {
1236                 STATE_SWITCH(gpcurr_src_file, STATE_CODE);
1237                 return T_SKIP;
1238         }
1239         /* $(( or $[ */
1240         else if (ret == T_DLBRACKET1 || ret == T_LBRACKET2)
1241         {
1242                 STATE_SWITCH(gpcurr_src_file, STATE_EXPR);
1243                 return T_SKIP;
1244         }
1245         /* ${ */
1246         else if (ret == T_LBRACKET3)
1247         {
1248                 /* XXX: envar expanssion */
1249                 if (0)
1250                 {
1251                         return T_SKIP;
1252                 }
1253                 else
1254                 {
1255                         return T_WORD;
1256                 }
1257         }
1258
1259         /* XXX: expanssion */
1260         if (0)
1261         {
1262                 return T_SKIP;
1263         }
1264         else
1265         {
1266                 return T_WORD;
1267         }
1268
1269         return ret;
1270 }
1271
1272 int Block_PunctTokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState)
1273 {
1274         int ret = Def_WordTokenProc(str, type, pstTokenState);
1275
1276         Txt_TokenProc(str, type, pstTokenState);
1277
1278         /* TBD: append string to buffer */
1279
1280         return ret;
1281 }
1282
1283 /*
1284  *
1285  */
1286 int Cmnt_WordTokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState)
1287 {
1288         //int ret = Def_WordTokenProc(str, type, pstTokenState);
1289
1290         /* TBD: append string to buffer */
1291
1292         return Txt_TokenProc(str, type, pstTokenState);
1293
1294         //return ret;
1295 }
1296
1297 /*
1298  *
1299  */
1300 int Cmnt_PunctTokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState)
1301 {
1302         //int ret = Def_PunctTokenProc(str, type, pstTokenState);
1303
1304         /* TBD: append string to buffer */
1305
1306         return Txt_TokenProc(str, type, pstTokenState);
1307
1308         //return ret;
1309 }
1310
1311 #include "codeblock.c"
1312 #include "envar.c"